From c38a13bf73bf4cf3a5a89ab7c0b904621a8d6fda Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 2 May 2023 17:02:13 +0200 Subject: [PATCH 01/32] Initial failing browser db test --- ocrdmonitor/dbmodel.py | 27 ++++ ocrdmonitor/server/settings.py | 1 + pdm.lock | 153 +++++++++++++++++- pyproject.toml | 1 + .../server/test_workspace_endpoint.py | 37 ++++- 5 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 ocrdmonitor/dbmodel.py diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py new file mode 100644 index 0000000..f263763 --- /dev/null +++ b/ocrdmonitor/dbmodel.py @@ -0,0 +1,27 @@ +import pymongo +from motor.motor_asyncio import AsyncIOMotorClient +from beanie import Document, init_beanie + + +class BrowserProcess(Document): + owner: str + process_id: str + workspace: str + + class Settings: + indexes = [ + pymongo.IndexModel( + [ + ("owner", pymongo.ASCENDING), + ("workspace", pymongo.ASCENDING), + ] + ) + ] + + +async def init(connection_str: str) -> None: + client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) + await init_beanie( + database=client.browsers, + document_models=[BrowserProcess], # type: ignore + ) diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index 2ccfc27..8a09a86 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -36,6 +36,7 @@ class OcrdBrowserSettings(BaseModel): workspace_dir: Path mode: Literal["native", "docker"] = "native" port_range: tuple[int, int] + # db_connection_string: str def factory(self) -> OcrdBrowserFactory: port_range_set = set(range(*self.port_range)) diff --git a/pdm.lock b/pdm.lock index 927db5f..f6c4e50 100644 --- a/pdm.lock +++ b/pdm.lock @@ -23,6 +23,19 @@ version = "22.2.0" requires_python = ">=3.6" summary = "Classes Without Boilerplate" +[[package]] +name = "beanie" +version = "1.18.0" +requires_python = ">=3.7,<4.0" +summary = "Asynchronous Python ODM for MongoDB" +dependencies = [ + "click>=7", + "lazy-model>=0.0.3", + "motor<4.0,>=2.5", + "pydantic>=1.10.0", + "toml", +] + [[package]] name = "beautifulsoup4" version = "4.12.0" @@ -94,6 +107,12 @@ name = "distlib" version = "0.3.6" summary = "Distribution utilities" +[[package]] +name = "dnspython" +version = "2.3.0" +requires_python = ">=3.7,<4.0" +summary = "DNS toolkit" + [[package]] name = "docker" version = "6.0.1" @@ -174,12 +193,30 @@ dependencies = [ "MarkupSafe>=2.0", ] +[[package]] +name = "lazy-model" +version = "0.0.5" +requires_python = ">=3.7,<4.0" +summary = "" +dependencies = [ + "pydantic>=1.9.0", +] + [[package]] name = "markupsafe" version = "2.1.2" requires_python = ">=3.7" summary = "Safely add untrusted strings to HTML/XML markup." +[[package]] +name = "motor" +version = "3.1.2" +requires_python = ">=3.7" +summary = "Non-blocking MongoDB driver for Tornado or asyncio" +dependencies = [ + "pymongo<5,>=4.1", +] + [[package]] name = "mypy" version = "1.1.1" @@ -252,6 +289,15 @@ dependencies = [ "python-dotenv>=0.10.4", ] +[[package]] +name = "pymongo" +version = "4.3.3" +requires_python = ">=3.7" +summary = "Python driver for MongoDB " +dependencies = [ + "dnspython<3.0.0,>=1.16.0", +] + [[package]] name = "pytest" version = "7.2.2" @@ -344,6 +390,12 @@ dependencies = [ "wrapt", ] +[[package]] +name = "toml" +version = "0.10.2" +requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Python Library for Tom's Obvious, Minimal Language" + [[package]] name = "types-beautifulsoup4" version = "4.12.0.0" @@ -422,8 +474,9 @@ requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" summary = "Module for decorators, wrappers and monkey patching." [metadata] -lock_version = "4.1" -content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b07050d5f4e" +lock_version = "4.2" +groups = ["default", "dev", "nox"] +content_hash = "sha256:9fccfa483f45b614a2680a3e849e3f890096aeb9ba0b41824e49540ebfd88325" [metadata.files] "anyio 3.6.2" = [ @@ -438,6 +491,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/21/31/3f468da74c7de4fcf9b25591e682856389b3400b4b62f201e65f15ea3e07/attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, {url = "https://files.pythonhosted.org/packages/fb/6e/6f83bf616d2becdf333a1640f1d463fef3150e2e926b7010cb0f81c95e88/attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, ] +"beanie 1.18.0" = [ + {url = "https://files.pythonhosted.org/packages/54/95/dbf00d5776e1a6f761262003e9abbc14706952e16018e8d6449a845a4c4d/beanie-1.18.0.tar.gz", hash = "sha256:1e1205d41176fe5b010cf04964c827841fe9c2c9cffc5ba5a29f66fb2f9c2e68"}, + {url = "https://files.pythonhosted.org/packages/d9/8b/a371c3fb5db326675720f66ac65f22f94f0b4c3830ccc65e0acecd352d2a/beanie-1.18.0-py3-none-any.whl", hash = "sha256:31f8eff8fe436e420766df457dc5bfd6025e927c9f8914adbc59ebff37fe6d7c"}, +] "beautifulsoup4 4.12.0" = [ {url = "https://files.pythonhosted.org/packages/c5/4c/b5b7d6e1d4406973fb7f4e5df81c6f07890fa82548ac3b945deed1df9d48/beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"}, {url = "https://files.pythonhosted.org/packages/ee/a7/06b189a2e280e351adcef25df532af3c59442123187e228b960ab3238687/beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"}, @@ -570,6 +627,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/58/07/815476ae605bcc5f95c87a62b95e74a1bce0878bc7a3119bc2bf4178f175/distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, {url = "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, ] +"dnspython 2.3.0" = [ + {url = "https://files.pythonhosted.org/packages/12/86/d305e87555430ff4630d729420d97dece3b16efcbf2b7d7e974d11b0d86c/dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {url = "https://files.pythonhosted.org/packages/91/8b/522301c50ca1f78b09c2ca116ffb0fd797eadf6a76085d376c01f9dd3429/dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] "docker 6.0.1" = [ {url = "https://files.pythonhosted.org/packages/79/26/6609b51ecb418e12d1534d00b888ce7e108f38b47dc6cd589598d5c6aaa2/docker-6.0.1.tar.gz", hash = "sha256:896c4282e5c7af5c45e8b683b0b0c33932974fe6e50fc6906a0a83616ab3da97"}, {url = "https://files.pythonhosted.org/packages/d5/b3/a5e41798a6d4b92880998e0d9e6980e57c5d039f7f7144f87627a6b19084/docker-6.0.1-py3-none-any.whl", hash = "sha256:dbcb3bd2fa80dca0788ed908218bf43972772009b881ed1e20dfc29a65e49782"}, @@ -606,6 +667,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/7a/ff/75c28576a1d900e87eb6335b063fab47a8ef3c8b4d88524c4bf78f670cce/Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, {url = "https://files.pythonhosted.org/packages/bc/c3/f068337a370801f372f2f8f6bad74a5c140f6fda3d9de154052708dd3c65/Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, ] +"lazy-model 0.0.5" = [ + {url = "https://files.pythonhosted.org/packages/0c/dd/0ef5eaa54e502c3e3229420f0c619bdd5a556417bab1ed74c1d8b23dd3e6/lazy-model-0.0.5.tar.gz", hash = "sha256:2d98f9dfe275012477555a439dceb56364793a0f266758d1a33267d68e8fbc76"}, + {url = "https://files.pythonhosted.org/packages/16/19/7d72b219dd73dd3e5c2212fcc54d58b255bd62d9469da39c830f8a41cc70/lazy_model-0.0.5-py3-none-any.whl", hash = "sha256:8b4fc5eac99029f84b11b21e81a6894911a475f25e53227b7e44833e62e26553"}, +] "markupsafe 2.1.2" = [ {url = "https://files.pythonhosted.org/packages/02/2c/18d55e5df6a9ea33709d6c33e08cb2e07d39e20ad05d8c6fbf9c9bcafd54/MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, {url = "https://files.pythonhosted.org/packages/04/cf/9464c3c41b7cdb8df660cda75676697e7fb49ce1be7691a1162fc88da078/MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, @@ -658,6 +723,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/ea/60/2400ba59cf2465fa136487ee7299f52121a9d04b2cf8539ad43ad10e70e8/MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, {url = "https://files.pythonhosted.org/packages/f9/aa/ebcd114deab08f892b1d70badda4436dbad1747f9e5b72cffb3de4c7129d/MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, ] +"motor 3.1.2" = [ + {url = "https://files.pythonhosted.org/packages/82/96/ae017cd62761d2fd2cc1eabfc902c3b4e3768fe994fc6a2f474694a56910/motor-3.1.2.tar.gz", hash = "sha256:80c08477c09e70db4f85c99d484f2bafa095772f1d29b3ccb253270f9041da9a"}, + {url = "https://files.pythonhosted.org/packages/f9/c3/22a695d0e6c373d0a33036de7fdc084068d896e948d11b691c88b6c1672f/motor-3.1.2-py3-none-any.whl", hash = "sha256:4bfc65230853ad61af447088527c1197f91c20ee957cfaea3144226907335716"}, +] "mypy 1.1.1" = [ {url = "https://files.pythonhosted.org/packages/2a/28/8485aad67750b3374443d28bad3eed947737cf425a640ea4be4ac70a7827/mypy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce61663faf7a8e5ec6f456857bfbcec2901fbdb3ad958b778403f63b9e606a1b"}, {url = "https://files.pythonhosted.org/packages/30/da/808ceaf2bcf23a9e90156c7b11b41add8dd5a009ee48159ec820d04d97bd/mypy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9401e33814cec6aec8c03a9548e9385e0e228fc1b8b0a37b9ea21038e64cdd8a"}, @@ -748,6 +817,82 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/f5/09/3f2ad426d20d2d353432f1c76290fa3c9863e2c04e05382ccca2aeade4c3/pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, {url = "https://files.pythonhosted.org/packages/f5/56/64028e205064748d6015a1afd6111c06f2b90982636850a3e157a7180ed5/pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, ] +"pymongo 4.3.3" = [ + {url = "https://files.pythonhosted.org/packages/01/10/e7157fcda1db4f759c858f8d9dc001112eb630136894056bb29f332137c3/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:bb869707d8e30645ed6766e44098600ca6cdf7989c22a3ea2b7966bb1d98d4b2"}, + {url = "https://files.pythonhosted.org/packages/05/17/185c96a98d3d91ad3cdfbc9bc91ad8bea697cfaf1b3ca314f52006f71d2b/pymongo-4.3.3-cp310-cp310-win32.whl", hash = "sha256:dc0cff74cd36d7e1edba91baa09622c35a8a57025f2f2b7a41e3f83b1db73186"}, + {url = "https://files.pythonhosted.org/packages/0e/8f/1009913e8ad51390966811e0163ed6df2dfa43a6f632ac35f53e51b2321b/pymongo-4.3.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e2961b05f9c04a53da8bfc72f1910b6aec7205fcf3ac9c036d24619979bbee4b"}, + {url = "https://files.pythonhosted.org/packages/0e/9f/a4986f0a86fc017599bf4c8912c01005a27c536acd221041234e0cb9739a/pymongo-4.3.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4ed00f96e147f40b565fe7530d1da0b0f3ab803d5dd5b683834500fa5d195ec4"}, + {url = "https://files.pythonhosted.org/packages/10/58/cdf21baff3328e6ba3b960918cd48302c3973e97ea4dcfbdf6ae5bf18408/pymongo-4.3.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6fcfbf435eebf8a1765c6d1f46821740ebe9f54f815a05c8fc30d789ef43cb12"}, + {url = "https://files.pythonhosted.org/packages/11/a3/8f7b87dbb9fd496f14c596bb02487fdb44dbb58e3c39da3f0eb0199b1523/pymongo-4.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdb87309de97c63cb9a69132e1cb16be470e58cffdfbad68fdd1dc292b22a840"}, + {url = "https://files.pythonhosted.org/packages/22/18/68b8a63f289df40df27623c99779acd9eb6c007a4546700e676e07d7c2d6/pymongo-4.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08fc250b5552ee97ceeae0f52d8b04f360291285fc7437f13daa516ce38fdbc6"}, + {url = "https://files.pythonhosted.org/packages/26/38/33270a35e265c1936ab3ea6863c02b9e3292ca013df9bd1e5ab1ed231ec7/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5effd87c7d363890259eac16c56a4e8da307286012c076223997f8cc4a8c435b"}, + {url = "https://files.pythonhosted.org/packages/29/3f/230c83a6be6e037f4558c9b7a2b8dc6de55ebc68662b0a13f9ff800614f7/pymongo-4.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eac0a143ef4f28f49670bf89cb15847eb80b375d55eba401ca2f777cd425f338"}, + {url = "https://files.pythonhosted.org/packages/2c/5c/ab73b2fc15fd9930f07bce865c3f0d98fe90211b92889831a746d61d3830/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f3621a46cdc7a9ba8080422262398a91762a581d27e0647746588d3f995c88"}, + {url = "https://files.pythonhosted.org/packages/2c/c7/302a0fa990e5c2e1b137b94a5dfc174437a77872990c6c05b21779fe9502/pymongo-4.3.3-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:c1a70c51da9fa95bd75c167edb2eb3f3c4d27bc4ddd29e588f21649d014ec0b7"}, + {url = "https://files.pythonhosted.org/packages/2e/d8/d35fcd7fd6d9b55ab6b317182884938d34c64c91dce9ff5cf3548ca5cd30/pymongo-4.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:81d1a7303bd02ca1c5be4aacd4db73593f573ba8e0c543c04c6da6275fd7a47e"}, + {url = "https://files.pythonhosted.org/packages/2f/f0/33804cfc9113e0405063f0a777d213d9c006512cb06681a258ae559b3a8c/pymongo-4.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3055510fdfdb1775bc8baa359783022f70bb553f2d46e153c094dfcb08578ff"}, + {url = "https://files.pythonhosted.org/packages/38/68/928d7ce22719cfa255fb973b34aed6f04ac3ea89049ce69e3b092c30a60f/pymongo-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1074f1a6f23e28b983c96142f2d45be03ec55d93035b471c26889a7ad2365db3"}, + {url = "https://files.pythonhosted.org/packages/39/22/e5acdce322f6aed2c6b06b8afae19c0fdf01031db1f7dbaeb34df60396c1/pymongo-4.3.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:016c412118e1c23fef3a1eada4f83ae6e8844fd91986b2e066fc1b0013cdd9ae"}, + {url = "https://files.pythonhosted.org/packages/39/97/3a04c850755723d64555ae29fdec2d4eafe9f2a12c22d4dc5e41e846423d/pymongo-4.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704d939656e21b073bfcddd7228b29e0e8a93dd27b54240eaafc0b9a631629a6"}, + {url = "https://files.pythonhosted.org/packages/3c/30/3d7e6336cfc795655a7193d77853972c5b502f58e1992205ad1b9bd28128/pymongo-4.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b6163dac53ef1e5d834297810c178050bd0548a4136cd4e0f56402185916ca"}, + {url = "https://files.pythonhosted.org/packages/40/e3/dda96a2280058e08bc0dabeddf86bd3513e601f579134f2107680585636b/pymongo-4.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7fac06a539daef4fcf5d8288d0d21b412f9b750454cd5a3cf90484665db442a"}, + {url = "https://files.pythonhosted.org/packages/42/0c/d2ad12aec55acdc4099134a8c87912d8fe01e2e1e5969b5d6c3485b99284/pymongo-4.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:54c377893f2cbbffe39abcff5ff2e917b082c364521fa079305f6f064e1a24a9"}, + {url = "https://files.pythonhosted.org/packages/45/2f/70f2e110a77dcb5490fe000aa380397968a09b8528f878aa1eadc0b11920/pymongo-4.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:74731c9e423c93cbe791f60c27030b6af6a948cef67deca079da6cd1bb583a8e"}, + {url = "https://files.pythonhosted.org/packages/48/b3/048d832794acb914cf8cf396089a29301ee79417e18f068f38a1eace9408/pymongo-4.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:cafa52873ae12baa512a8721afc20de67a36886baae6a5f394ddef0ce9391f91"}, + {url = "https://files.pythonhosted.org/packages/49/de/9005f70242f651fe4758a162eedbda13c9e55713083c345574c17cf8aa8f/pymongo-4.3.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9b87b23570565a6ddaa9244d87811c2ee9cffb02a753c8a2da9c077283d85845"}, + {url = "https://files.pythonhosted.org/packages/4a/92/9c11924649a557d95283882a4bcb67cfc32d6cb1528064a53c0bdb0540d7/pymongo-4.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9c2885b4a8e6e39db5662d8b02ca6dcec796a45e48c2de12552841f061692ba"}, + {url = "https://files.pythonhosted.org/packages/4b/ce/c6c6875dc14410952d3ff2e7960fb0498b1d9e70c483e5ce788c01fad54e/pymongo-4.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0640b4e9d008e13956b004d1971a23377b3d45491f87082161c92efb1e6c0d6"}, + {url = "https://files.pythonhosted.org/packages/4e/ca/6c1cb5c69715c13312852d91cb62c175e7da58c91c428447db1a2364c646/pymongo-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3a51901066696c4af38c6c63a1f0aeffd5e282367ff475de8c191ec9609b56d"}, + {url = "https://files.pythonhosted.org/packages/4f/2c/2da01e59e47cec96df562f0fe8ed6e1dd8b01a0ff8acd6d8ea1b59aaf82a/pymongo-4.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d86c35d94b5499689354ccbc48438a79f449481ee6300f3e905748edceed78e7"}, + {url = "https://files.pythonhosted.org/packages/4f/a9/32799279229f74f4d477f6c122dbbb4173f7d6d158bb9f7adf582c2ada20/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6dd1cf2995fdbd64fc0802313e8323f5fa18994d51af059b5b8862b73b5e53f0"}, + {url = "https://files.pythonhosted.org/packages/60/2f/6b18e099cfabf8fbe86ec201f53afa73a8b80e2e9dcbdef52429492d236e/pymongo-4.3.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:8fd6e191b92a10310f5a6cfe10d6f839d79d192fb02480bda325286bd1c7b385"}, + {url = "https://files.pythonhosted.org/packages/63/0e/ac6759051f18adf5506fe0c458bc12d03d9e94d2dc83087b21dc21888154/pymongo-4.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:d5571b6978750601f783cea07fb6b666837010ca57e5cefa389c1d456f6222e2"}, + {url = "https://files.pythonhosted.org/packages/63/74/51b2ec1b760169cbb19637913b86b6851dd9a57f95fe67adb7b0d1037469/pymongo-4.3.3-cp310-cp310-manylinux2014_ppc64le.whl", hash = "sha256:ffcc8394123ea8d43fff8e5d000095fe7741ce3f8988366c5c919c4f5eb179d3"}, + {url = "https://files.pythonhosted.org/packages/63/c8/a6e9f789cfbafc8293b5d94b0fa66b7a8854c6e74a04a74bc7585381ddd8/pymongo-4.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be1d2ce7e269215c3ee9a215e296b7a744aff4f39233486d2c4d77f5f0c561a6"}, + {url = "https://files.pythonhosted.org/packages/66/b6/8e554ee180a28aa3f99200eb1ab60ab180fbea1a55f47166a6da2fd93299/pymongo-4.3.3-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:d07d06dba5b5f7d80f9cc45501456e440f759fe79f9895922ed486237ac378a8"}, + {url = "https://files.pythonhosted.org/packages/69/d4/9cd99a5d98353b6c10595ec969c087d63a93ce60741b52463a9fcb2114ad/pymongo-4.3.3-cp39-cp39-win32.whl", hash = "sha256:dc24d245026a72d9b4953729d31813edd4bd4e5c13622d96e27c284942d33f24"}, + {url = "https://files.pythonhosted.org/packages/6a/6c/246b69b8fc3071e9ff1f42480fbc29835b95e910655604b66bef0a282e78/pymongo-4.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c184ec5be465c0319440734491e1aa4709b5f3ba75fdfc9dbbc2ae715a7f6829"}, + {url = "https://files.pythonhosted.org/packages/71/c7/c129dcde11ec97fe485cfc7a837284a0300bc4647a1bcb1e63f1ce050732/pymongo-4.3.3-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:4d00b91c77ceb064c9b0459f0d6ea5bfdbc53ea9e17cf75731e151ef25a830c7"}, + {url = "https://files.pythonhosted.org/packages/74/7a/140e4c739319c3ee1163aa65bc91414ddf5b3c6376af19375e2dead1fbb5/pymongo-4.3.3-cp310-cp310-manylinux2014_s390x.whl", hash = "sha256:943f208840777f34312c103a2d1caab02d780c4e9be26b3714acf6c4715ba7e1"}, + {url = "https://files.pythonhosted.org/packages/74/a8/fe9d9c1f7d3a12b3d5c2b26fb267671a02f42b68ddd69d20105b6d87798b/pymongo-4.3.3-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:c09956606c08c4a7c6178a04ba2dd9388fcc5db32002ade9c9bc865ab156ab6d"}, + {url = "https://files.pythonhosted.org/packages/76/05/de90f39846ec83fe9e2099c7993266bb1a154f3a0777e78121f56fe08ee7/pymongo-4.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef888f48eb9203ee1e04b9fb27429017b290fb916f1e7826c2f7808c88798394"}, + {url = "https://files.pythonhosted.org/packages/7d/33/aa74d9e5067bdd7b68cbe54ea5cad427883131d100c20d6a31ff0625a214/pymongo-4.3.3-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:8a06a0c02f5606330e8f2e2f3b7949877ca7e4024fa2bff5a4506bec66c49ec7"}, + {url = "https://files.pythonhosted.org/packages/81/37/c5c765526adb3f452ea4033d5d4e960514d53857b32c85fc2dfcac7aad86/pymongo-4.3.3-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:b38a96b3eed8edc515b38257f03216f382c4389d022a8834667e2bc63c0c0c31"}, + {url = "https://files.pythonhosted.org/packages/81/5d/6d34f7b3cffe3efe38cac65de60beba7f1a14b8f5b64d27354bce33b924d/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:47f7aa217b25833cd6f0e72b0d224be55393c2692b4f5e0561cb3beeb10296e9"}, + {url = "https://files.pythonhosted.org/packages/81/f1/5d56b0ffdda842298334135ac181032ee4624bc57101a538d67ba8958695/pymongo-4.3.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6c2216d8b6a6d019c6f4b1ad55f890e5e77eb089309ffc05b6911c09349e7474"}, + {url = "https://files.pythonhosted.org/packages/89/24/52d65bbb0cf038d73b49c9d1f6b251500d807ed2579aaa55cf5d788513be/pymongo-4.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fd7bb378d82b88387dc10227cfd964f6273eb083e05299e9b97cbe075da12d11"}, + {url = "https://files.pythonhosted.org/packages/8b/8f/93649909ec1ba88fee224884884b4e10ac26c0ca00c58f1781036476d30d/pymongo-4.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:599d3f6fbef31933b96e2d906b0f169b3371ff79ea6aaf6ecd76c947a3508a3d"}, + {url = "https://files.pythonhosted.org/packages/92/45/47134bdc3d628fa02945545c9d0cca1d7b349c507734860cf3614da77cb0/pymongo-4.3.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0c466710871d0026c190fc4141e810cf9d9affbf4935e1d273fbdc7d7cda6143"}, + {url = "https://files.pythonhosted.org/packages/93/da/d58cdba6e4c896300d1c939119c0911948a7edd94e10cc75048142e56160/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2fdc855149efe7cdcc2a01ca02bfa24761c640203ea94df467f3baf19078be"}, + {url = "https://files.pythonhosted.org/packages/96/48/8baccdb480d0ceb2799d1b6d2da780b6f174635c64f82fa27bc8fbb9d660/pymongo-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c051fe37c96b9878f37fa58906cb53ecd13dcb7341d3a85f1e2e2f6b10782d9"}, + {url = "https://files.pythonhosted.org/packages/97/9f/0156a752e50cfbc767a182c80a7e94174a772a94cb72a52f2660fc373c77/pymongo-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b16250238de8dafca225647608dddc7bbb5dce3dd53b4d8e63c1cc287394c2f"}, + {url = "https://files.pythonhosted.org/packages/98/4d/4423858f2587a3c15c9b40a70e3672e0902667874f24e77e5388d848715d/pymongo-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7761cacb8745093062695b11574effea69db636c2fd0a9269a1f0183712927b4"}, + {url = "https://files.pythonhosted.org/packages/9a/31/482f7401e7bbbeb66ab6b4ac263e2b50435f4329cce1e72378972d48f6b5/pymongo-4.3.3.tar.gz", hash = "sha256:34e95ffb0a68bffbc3b437f2d1f25fc916fef3df5cdeed0992da5f42fae9b807"}, + {url = "https://files.pythonhosted.org/packages/a0/53/f8b2099b2d8dcec0e4070455b6b7a9ea5088ee07a745b0c6a711d55a5357/pymongo-4.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:fc28e8d85d392a06434e9a934908d97e2cf453d69488d2bcd0bfb881497fd975"}, + {url = "https://files.pythonhosted.org/packages/a3/c6/ff88fce93529c9418c80854ecaf013254ab0b1d59f8f4fa2702419352d18/pymongo-4.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:b8a03af1ce79b902a43f5f694c4ca8d92c2a4195db0966f08f266549e2fc49bc"}, + {url = "https://files.pythonhosted.org/packages/a9/8c/5ae0d794ff1771dd2a298f1a7d0889455a65874a98120171a002c8cb741a/pymongo-4.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52896e22115c97f1c829db32aa2760b0d61839cfe08b168c2b1d82f31dbc5f55"}, + {url = "https://files.pythonhosted.org/packages/b5/a2/a566780a2baeb108ae4b7e87add2c022090a39f728e9c808dee4c8b1efde/pymongo-4.3.3-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:b0cfe925610f2fd59555bb7fc37bd739e4b197d33f2a8b2fae7b9c0c6640318c"}, + {url = "https://files.pythonhosted.org/packages/c4/3d/51e3ed544c4d4a0dbcafe197d582f9e922e73ea185bd5a19486c7c297308/pymongo-4.3.3-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:34b040e095e1671df0c095ec0b04fc4ebb19c4c160f87c2b55c079b16b1a6b00"}, + {url = "https://files.pythonhosted.org/packages/c4/b5/e2d246016d15c949736c9a4b4da4ec8e2045b661504be4b749c34188c2a5/pymongo-4.3.3-cp311-cp311-win32.whl", hash = "sha256:524d78673518dcd352a91541ecd2839c65af92dc883321c2109ef6e5cd22ef23"}, + {url = "https://files.pythonhosted.org/packages/c5/be/64441bc6f65ddca82ecb1231348c89257272c023a44658d59f044877a498/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e758f0e734e1e90357ae01ec9c6daf19ff60a051192fe110d8fb25c62600e"}, + {url = "https://files.pythonhosted.org/packages/c6/1f/cd1d6d21620125693cd6d21eb9264b885df553f3c51cb778b06fe96d6abd/pymongo-4.3.3-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:711bc52cb98e7892c03e9b669bebd89c0a890a90dbc6d5bb2c47f30239bac6e9"}, + {url = "https://files.pythonhosted.org/packages/c9/02/77f30505aa009f329ec935a2e0e856889e19568c1c6c7af4dbffe894c27e/pymongo-4.3.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:dca34367a4e77fcab0693e603a959878eaf2351585e7d752cac544bc6b2dee46"}, + {url = "https://files.pythonhosted.org/packages/d4/4d/cdfde31b4545d2f0aaabae9a9acd0dda6384f3d02b4f7a4b6a483f4bf749/pymongo-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5134d33286c045393c7beb51be29754647cec5ebc051cf82799c5ce9820a2ca2"}, + {url = "https://files.pythonhosted.org/packages/d6/58/a39537805ca205b4b65503765ca110224a409e777c1825fd6c8108ec9fd0/pymongo-4.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:341221e2f2866a5960e6f8610f4cbac0bb13097f3b1a289aa55aba984fc0d969"}, + {url = "https://files.pythonhosted.org/packages/dc/bc/1d69ee98cc0b50278f9c6044666a5dbd8b296e8bd3af733066f6bb8bc597/pymongo-4.3.3-cp37-cp37m-win32.whl", hash = "sha256:49210feb0be8051a64d71691f0acbfbedc33e149f0a5d6e271fddf6a12493fed"}, + {url = "https://files.pythonhosted.org/packages/df/2c/572e43db59a870b8df3332b94bd29ee7246bcba8cbb071b61174ecd1c834/pymongo-4.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:316498b642c00401370b2156b5233b256f9b33799e0a8d9d0b8a7da217a20fca"}, + {url = "https://files.pythonhosted.org/packages/e1/ea/ca13d38405cea315683b085cfcf661cf48be9a9a786dcead86d9454fcc18/pymongo-4.3.3-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:39b03045c71f761aee96a12ebfbc2f4be89e724ff6f5e31c2574c1a0e2add8bd"}, + {url = "https://files.pythonhosted.org/packages/e2/7c/a076b118f1b7aea6c8dc548d45441801b86486bf67765589112e28ea188d/pymongo-4.3.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:c6258a3663780ae47ba73d43eb63c79c40ffddfb764e09b56df33be2f9479837"}, + {url = "https://files.pythonhosted.org/packages/e7/e1/e2c577333ee346b411db65d9f62c746eca8b1062c55afdb5d2fb8ebc23fe/pymongo-4.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa7e202feb683dad74f00dea066690448d0cfa310f8a277db06ec8eb466601b5"}, + {url = "https://files.pythonhosted.org/packages/ee/2a/223a77aab2d1d9f2ca86b1db60578f25ebd2f1c0f558fcf46d05457865d1/pymongo-4.3.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cd6a4afb20fb3c26a7bfd4611a0bbb24d93cbd746f5eb881f114b5e38fd55501"}, + {url = "https://files.pythonhosted.org/packages/f0/25/5331b822a0e2486efe75c741fa9dcb500b67ecfb0223f26179afa60f1c17/pymongo-4.3.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3b93043b14ba7eb08c57afca19751658ece1cfa2f0b7b1fb5c7a41452fbb8482"}, + {url = "https://files.pythonhosted.org/packages/f2/4a/68ab4706a992fd7b01ec53a9e2138733972895b260578e544221845770dd/pymongo-4.3.3-cp310-cp310-manylinux1_i686.whl", hash = "sha256:66413c50d510e5bcb0afc79880d1693a2185bcea003600ed898ada31338c004e"}, + {url = "https://files.pythonhosted.org/packages/f3/87/f2ccd99ea5184d9a9013acca92f3060e29253038df8003148b1a643e6165/pymongo-4.3.3-cp310-cp310-manylinux2014_i686.whl", hash = "sha256:695939036a320f4329ccf1627edefbbb67cc7892b8222d297b0dd2313742bfee"}, + {url = "https://files.pythonhosted.org/packages/f4/d6/3088b63536c74c4e9cf687916712843e7d4abfc981eca3e264ec801372af/pymongo-4.3.3-cp38-cp38-win32.whl", hash = "sha256:a6cd6f1db75eb07332bd3710f58f5fce4967eadbf751bad653842750a61bda62"}, + {url = "https://files.pythonhosted.org/packages/f8/ef/bd801e889305bc48ca3210569ea613d66a52c717578a465ac2792cec709a/pymongo-4.3.3-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:01f7cbe88d22440b6594c955e37312d932fd632ffed1a86d0c361503ca82cc9d"}, + {url = "https://files.pythonhosted.org/packages/fa/6a/bf5391534a10cfb4a2b4a9e6697f17115fc460da8041ec67835c23d2ff59/pymongo-4.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a966d5304b7d90c45c404914e06bbf02c5bf7e99685c6c12f0047ef2aa837142"}, + {url = "https://files.pythonhosted.org/packages/fc/28/1b934e5839bf12b022782561c803ee63149737d6c5f9627d3299cd28516f/pymongo-4.3.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:7d43ac9c7eeda5100fb0a7152fab7099c9cf9e5abd3bb36928eb98c7d7a339c6"}, +] "pytest 7.2.2" = [ {url = "https://files.pythonhosted.org/packages/b2/68/5321b5793bd506961bd40bdbdd0674e7de4fb873ee7cab33dd27283ad513/pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, {url = "https://files.pythonhosted.org/packages/b9/29/311895d9cd3f003dd58e8fdea36dd895ba2da5c0c90601836f7de79f76fe/pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, @@ -799,6 +944,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 "testcontainers 3.7.1" = [ {url = "https://files.pythonhosted.org/packages/b3/37/38c595414d764cb1d9f3a0c907878c4146a21505ab974c63bcf3d8145807/testcontainers-3.7.1-py2.py3-none-any.whl", hash = "sha256:7f48cef4bf0ccd78f1a4534d4b701a003a3bace851f24eae58a32f9e3f0aeba0"}, ] +"toml 0.10.2" = [ + {url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] "types-beautifulsoup4 4.12.0.0" = [ {url = "https://files.pythonhosted.org/packages/92/0b/39afb220c7d8328c5c887007e17c950eda2c2e9300132b69e923e81ff033/types_beautifulsoup4-4.12.0.0-py3-none-any.whl", hash = "sha256:43c23852a6ef0053632b9a308fc3488831c0f3e02c0f4b4478a28703217cf683"}, {url = "https://files.pythonhosted.org/packages/a4/23/9a9131dedfbd64354fabedef74c8b69092afa4c65720b8fb35df18ded18b/types-beautifulsoup4-4.12.0.0.tar.gz", hash = "sha256:3859e70d3118d65d12ebfca109304de4bf52383e6f99f941c114fd1153bb6cc1"}, diff --git a/pyproject.toml b/pyproject.toml index 3606818..3ea9c17 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "websockets>=10.4", "uvicorn>=0.19.0", "httpx>=0.23.3", + "beanie>=1.18.0", ] requires-python = ">=3.11" license = { text = "MIT" } diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index e901414..5b070dc 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -6,17 +6,28 @@ import pytest_asyncio from fastapi.testclient import TestClient from httpx import Response +from testcontainers.mongodb import MongoDbContainer from ocrdbrowser import ChannelClosed +from ocrdmonitor import dbmodel +from ocrdmonitor.server.app import create_app from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures import WORKSPACE_DIR, patch_factory -from tests.testdoubles import BrowserFake +from tests.ocrdmonitor.server.fixtures import ( + WORKSPACE_DIR, + create_settings, + patch_factory, +) +from tests.testdoubles import ( + Browser_Heading, + BrowserFake, + BrowserSpy, + BrowserTestDouble, +) from tests.testdoubles._browserfactory import ( BrowserTestDoubleFactory, IteratingBrowserTestDoubleFactory, SingletonBrowserTestDoubleFactory, ) -from tests.testdoubles import Browser_Heading, BrowserSpy, BrowserTestDouble class DisconnectingChannel: @@ -174,3 +185,23 @@ def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( result = app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 502 + + +@pytest.mark.asyncio +async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with MongoDbContainer() as container: + await dbmodel.init(container.get_connection_url()) + + async with patch_factory(IteratingBrowserTestDoubleFactory()): + settings = create_settings() + app = TestClient(create_app(settings)) + + _ = view_workspace(app, "a_workspace") + + found_browsers = await dbmodel.BrowserProcess.find_all( + dbmodel.BrowserProcess.workspace == "a_workspace" + ).count() + + assert found_browsers == 1 From ca7f4df7b9235ea8af8dc89ffe967e3c9860d191 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Wed, 3 May 2023 16:59:34 +0200 Subject: [PATCH 02/32] Passes first DB test. Needs refactoring! --- ocrdbrowser/_browser.py | 3 ++ ocrdbrowser/_docker.py | 3 ++ ocrdbrowser/_subprocess.py | 6 ++++ ocrdmonitor/dbmodel.py | 2 ++ ocrdmonitor/server/workspaces.py | 6 ++++ pyproject.toml | 10 ++---- .../server/test_workspace_endpoint.py | 31 ++++++++++++------- tests/testdoubles/_backgroundprocess.py | 7 +++++ tests/testdoubles/_browserfake.py | 3 ++ tests/testdoubles/_browserspy.py | 7 ++++- 10 files changed, 59 insertions(+), 19 deletions(-) diff --git a/ocrdbrowser/_browser.py b/ocrdbrowser/_browser.py index aa9d713..7336171 100644 --- a/ocrdbrowser/_browser.py +++ b/ocrdbrowser/_browser.py @@ -6,6 +6,9 @@ class OcrdBrowser(Protocol): + def process_id(self) -> str: + ... + def address(self) -> str: ... diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index 4858eb1..78cc76b 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -30,6 +30,9 @@ def __init__(self, host: str, port: Port, owner: str, workspace: str) -> None: self._workspace = path.abspath(workspace) self.id: str | None = None + def process_id(self) -> str: + return self._container_name() + def address(self) -> str: return f"{self._host}:{self._port}" diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 832ea09..8110325 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -20,6 +20,12 @@ def __init__(self, localport: Port, owner: str, workspace: str) -> None: self._workspace = workspace self._process: Optional[asyncio.subprocess.Process] = None + def process_id(self) -> str: + if not self._process: + raise AttributeError + + return str(self._process.pid) + def address(self) -> str: # as long as we do not have a reverse proxy on BW_PORT, # we must map the local port range to the exposed range diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index f263763..168d78d 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -1,3 +1,4 @@ +import asyncio import pymongo from motor.motor_asyncio import AsyncIOMotorClient from beanie import Document, init_beanie @@ -21,6 +22,7 @@ class Settings: async def init(connection_str: str) -> None: client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) + client.get_io_loop = asyncio.get_event_loop await init_beanie( database=client.browsers, document_models=[BrowserProcess], # type: ignore diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index 0dfabab..e6111aa 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -5,6 +5,7 @@ import ocrdbrowser +from ocrdmonitor import dbmodel import ocrdmonitor.server.proxy as proxy from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect from fastapi.templating import Jinja2Templates @@ -40,6 +41,11 @@ async def browser(request: Request, workspace: Path) -> Response: if (session_id, workspace) not in redirects: browser = await launch_browser(session_id, workspace) redirects.add(session_id, workspace, browser) + await dbmodel.BrowserProcess( # type: ignore + process_id=browser.process_id(), + owner=browser.owner(), + workspace=browser.workspace(), + ).insert() return response diff --git a/pyproject.toml b/pyproject.toml index 3ea9c17..31141b0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,24 +32,20 @@ dev = [ "types-requests>=2.28.11.15", "types-beautifulsoup4>=4.12.0.0", ] -nox = [ - "nox>=2022.11.21", -] +nox = ["nox>=2022.11.21"] [tool.mypy] plugins = ["pydantic.mypy"] [tool.pytest.ini_options] -markers = [ - "integration: mark test as integration test", -] +markers = ["integration: mark test as integration test"] [tool.pdm.scripts] test = "pytest tests -m 'not integration'" test-integration = "pytest tests" [[tool.mypy.overrides]] -module = "testcontainers.*" +module = ["testcontainers.*", "motor.motor_asyncio"] ignore_missing_imports = true [build-system] diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 5b070dc..8534d0a 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -11,6 +11,7 @@ from ocrdbrowser import ChannelClosed from ocrdmonitor import dbmodel from ocrdmonitor.server.app import create_app +from ocrdmonitor.server.settings import OcrdBrowserSettings from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.fixtures import ( WORKSPACE_DIR, @@ -187,21 +188,29 @@ def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( assert result.status_code == 502 +@pytest_asyncio.fixture(autouse=True) +async def db() -> AsyncIterator[None]: + with MongoDbContainer() as container: + await dbmodel.init(container.get_connection_url()) + yield + + @pytest.mark.asyncio async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( monkeypatch: pytest.MonkeyPatch, ) -> None: - with MongoDbContainer() as container: - await dbmodel.init(container.get_connection_url()) - - async with patch_factory(IteratingBrowserTestDoubleFactory()): - settings = create_settings() - app = TestClient(create_app(settings)) + async with patch_factory(IteratingBrowserTestDoubleFactory()): + settings = create_settings() + app = TestClient(create_app(settings)) - _ = view_workspace(app, "a_workspace") + _ = view_workspace(app, "a_workspace") - found_browsers = await dbmodel.BrowserProcess.find_all( - dbmodel.BrowserProcess.workspace == "a_workspace" - ).count() + found_browsers = ( + await dbmodel.BrowserProcess.find( + dbmodel.BrowserProcess.workspace == str(WORKSPACE_DIR / "a_workspace") + ) + .find(dbmodel.BrowserProcess.process_id == "1234") + .to_list() + ) - assert found_browsers == 1 + assert len(found_browsers) == 1 diff --git a/tests/testdoubles/_backgroundprocess.py b/tests/testdoubles/_backgroundprocess.py index 159682e..a2dc529 100644 --- a/tests/testdoubles/_backgroundprocess.py +++ b/tests/testdoubles/_backgroundprocess.py @@ -25,6 +25,13 @@ def __exit__(self, *args: Any, **kwargs: Any) -> None: def is_running(self) -> bool: return self._process is not None and self._process.is_alive() + @property + def pid(self) -> int | None: + if not self._process: + return None + + return self._process.pid + def launch(self) -> None: if self.is_running: return diff --git a/tests/testdoubles/_browserfake.py b/tests/testdoubles/_browserfake.py index 6c81ff0..03d32be 100644 --- a/tests/testdoubles/_browserfake.py +++ b/tests/testdoubles/_browserfake.py @@ -21,6 +21,9 @@ def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self._workspace = workspace self._browser = broadway_fake(workspace) + def process_id(self) -> str: + return str(self._browser.pid) + def address(self) -> str: return "http://localhost:7000" diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 212e081..57e816d 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -47,13 +47,15 @@ async def open_channel(self) -> AsyncGenerator[Channel, None]: class BrowserSpy: def __init__( self, + process_id: str = "1234", owner: str = "", workspace_path: str = "", address: str = "http://unreachable.example.com", running: bool = False, ) -> None: - self.is_running = running self._address = address + self._process_id = process_id + self.is_running = running self.owner_name = owner self.workspace_path = workspace_path self._client = BrowserClientStub() @@ -67,6 +69,9 @@ def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self.owner_name = owner self.workspace_path = workspace + def process_id(self) -> str: + return self._process_id + def address(self) -> str: return self._address From 554297224c6d648027e2c955c117a46deb8e1e46 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Thu, 4 May 2023 16:50:12 +0200 Subject: [PATCH 03/32] Begin implementing repository --- ocrdmonitor/browserprocess.py | 28 ++++++++++ ocrdmonitor/dbmodel.py | 52 ++++++++++++++++++- .../server/test_workspace_endpoint.py | 20 +++---- 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 ocrdmonitor/browserprocess.py diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py new file mode 100644 index 0000000..801d059 --- /dev/null +++ b/ocrdmonitor/browserprocess.py @@ -0,0 +1,28 @@ +from typing import Collection, Protocol + + +class BrowserProcess(Protocol): + @property + def process_id(self) -> str: + ... + + @property + def workspace(self) -> str: + ... + + @property + def owner(self) -> str: + ... + + +class BrowserProcessRepository(Protocol): + async def insert(self, browser: BrowserProcess) -> None: + ... + + async def delete(self, browser: BrowserProcess) -> None: + ... + + async def find( + self, owner: str, workspace: str, process_id: str | None = None + ) -> Collection[BrowserProcess]: + ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 168d78d..e28f102 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -1,7 +1,12 @@ import asyncio +from typing import Any, Collection, Mapping + import pymongo -from motor.motor_asyncio import AsyncIOMotorClient from beanie import Document, init_beanie +from beanie.odm.queries.find import FindMany +from motor.motor_asyncio import AsyncIOMotorClient + +from ocrdmonitor.browserprocess import BrowserProcess as BrowserProcessProtocol class BrowserProcess(Document): @@ -20,6 +25,51 @@ class Settings: ] +class MongoBrowserProcessRepository: + async def insert(self, browser: BrowserProcessProtocol) -> None: + await BrowserProcess( + owner=browser.owner, + process_id=browser.process_id, + workspace=browser.workspace, + ).insert() + + async def delete(self, browser: BrowserProcessProtocol) -> None: + await BrowserProcess( + owner=browser.owner, + process_id=browser.process_id, + workspace=browser.workspace, + ).delete() + + async def find( + self, + *, + owner: str | None = None, + workspace: str | None = None, + process_id: str | None = None, + ) -> Collection[BrowserProcess]: + results: FindMany[BrowserProcess] | None = None + + def find( + results: FindMany[BrowserProcess] | None, + *predicates: Mapping[str, Any] | bool, + ) -> FindMany[BrowserProcess]: + if results is None: + return BrowserProcess.find(*predicates) + + return results.find(*predicates) + + if owner is not None: + results = find(results, BrowserProcess.owner == owner) + + if workspace is not None: + results = find(results, BrowserProcess.workspace == workspace) + + if process_id is not None: + results = find(results, BrowserProcess.process_id == process_id) + + return await results.to_list() if results is not None else [] + + async def init(connection_str: str) -> None: client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) client.get_io_loop = asyncio.get_event_loop diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 8534d0a..70f4ce2 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -196,21 +196,15 @@ async def db() -> AsyncIterator[None]: @pytest.mark.asyncio +@pytest.mark.usefixtures("iterating_factory") async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( - monkeypatch: pytest.MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, app: TestClient ) -> None: - async with patch_factory(IteratingBrowserTestDoubleFactory()): - settings = create_settings() - app = TestClient(create_app(settings)) - - _ = view_workspace(app, "a_workspace") - - found_browsers = ( - await dbmodel.BrowserProcess.find( - dbmodel.BrowserProcess.workspace == str(WORKSPACE_DIR / "a_workspace") - ) - .find(dbmodel.BrowserProcess.process_id == "1234") - .to_list() + repo = dbmodel.MongoBrowserProcessRepository() + _ = view_workspace(app, "a_workspace") + + found_browsers = await repo.find( + workspace=str(WORKSPACE_DIR / "a_workspace"), process_id="1234" ) assert len(found_browsers) == 1 From a525a1d062130891e5da30500ecc6505a46590a6 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 2 May 2023 17:02:13 +0200 Subject: [PATCH 04/32] Initial failing browser db test --- ocrdmonitor/dbmodel.py | 27 ++++ ocrdmonitor/server/settings.py | 1 + pdm.lock | 153 +++++++++++++++++- pyproject.toml | 1 + .../server/test_workspace_endpoint.py | 37 ++++- 5 files changed, 214 insertions(+), 5 deletions(-) create mode 100644 ocrdmonitor/dbmodel.py diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py new file mode 100644 index 0000000..f263763 --- /dev/null +++ b/ocrdmonitor/dbmodel.py @@ -0,0 +1,27 @@ +import pymongo +from motor.motor_asyncio import AsyncIOMotorClient +from beanie import Document, init_beanie + + +class BrowserProcess(Document): + owner: str + process_id: str + workspace: str + + class Settings: + indexes = [ + pymongo.IndexModel( + [ + ("owner", pymongo.ASCENDING), + ("workspace", pymongo.ASCENDING), + ] + ) + ] + + +async def init(connection_str: str) -> None: + client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) + await init_beanie( + database=client.browsers, + document_models=[BrowserProcess], # type: ignore + ) diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index 2ccfc27..8a09a86 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -36,6 +36,7 @@ class OcrdBrowserSettings(BaseModel): workspace_dir: Path mode: Literal["native", "docker"] = "native" port_range: tuple[int, int] + # db_connection_string: str def factory(self) -> OcrdBrowserFactory: port_range_set = set(range(*self.port_range)) diff --git a/pdm.lock b/pdm.lock index 927db5f..f6c4e50 100644 --- a/pdm.lock +++ b/pdm.lock @@ -23,6 +23,19 @@ version = "22.2.0" requires_python = ">=3.6" summary = "Classes Without Boilerplate" +[[package]] +name = "beanie" +version = "1.18.0" +requires_python = ">=3.7,<4.0" +summary = "Asynchronous Python ODM for MongoDB" +dependencies = [ + "click>=7", + "lazy-model>=0.0.3", + "motor<4.0,>=2.5", + "pydantic>=1.10.0", + "toml", +] + [[package]] name = "beautifulsoup4" version = "4.12.0" @@ -94,6 +107,12 @@ name = "distlib" version = "0.3.6" summary = "Distribution utilities" +[[package]] +name = "dnspython" +version = "2.3.0" +requires_python = ">=3.7,<4.0" +summary = "DNS toolkit" + [[package]] name = "docker" version = "6.0.1" @@ -174,12 +193,30 @@ dependencies = [ "MarkupSafe>=2.0", ] +[[package]] +name = "lazy-model" +version = "0.0.5" +requires_python = ">=3.7,<4.0" +summary = "" +dependencies = [ + "pydantic>=1.9.0", +] + [[package]] name = "markupsafe" version = "2.1.2" requires_python = ">=3.7" summary = "Safely add untrusted strings to HTML/XML markup." +[[package]] +name = "motor" +version = "3.1.2" +requires_python = ">=3.7" +summary = "Non-blocking MongoDB driver for Tornado or asyncio" +dependencies = [ + "pymongo<5,>=4.1", +] + [[package]] name = "mypy" version = "1.1.1" @@ -252,6 +289,15 @@ dependencies = [ "python-dotenv>=0.10.4", ] +[[package]] +name = "pymongo" +version = "4.3.3" +requires_python = ">=3.7" +summary = "Python driver for MongoDB " +dependencies = [ + "dnspython<3.0.0,>=1.16.0", +] + [[package]] name = "pytest" version = "7.2.2" @@ -344,6 +390,12 @@ dependencies = [ "wrapt", ] +[[package]] +name = "toml" +version = "0.10.2" +requires_python = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" +summary = "Python Library for Tom's Obvious, Minimal Language" + [[package]] name = "types-beautifulsoup4" version = "4.12.0.0" @@ -422,8 +474,9 @@ requires_python = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" summary = "Module for decorators, wrappers and monkey patching." [metadata] -lock_version = "4.1" -content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b07050d5f4e" +lock_version = "4.2" +groups = ["default", "dev", "nox"] +content_hash = "sha256:9fccfa483f45b614a2680a3e849e3f890096aeb9ba0b41824e49540ebfd88325" [metadata.files] "anyio 3.6.2" = [ @@ -438,6 +491,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/21/31/3f468da74c7de4fcf9b25591e682856389b3400b4b62f201e65f15ea3e07/attrs-22.2.0.tar.gz", hash = "sha256:c9227bfc2f01993c03f68db37d1d15c9690188323c067c641f1a35ca58185f99"}, {url = "https://files.pythonhosted.org/packages/fb/6e/6f83bf616d2becdf333a1640f1d463fef3150e2e926b7010cb0f81c95e88/attrs-22.2.0-py3-none-any.whl", hash = "sha256:29e95c7f6778868dbd49170f98f8818f78f3dc5e0e37c0b1f474e3561b240836"}, ] +"beanie 1.18.0" = [ + {url = "https://files.pythonhosted.org/packages/54/95/dbf00d5776e1a6f761262003e9abbc14706952e16018e8d6449a845a4c4d/beanie-1.18.0.tar.gz", hash = "sha256:1e1205d41176fe5b010cf04964c827841fe9c2c9cffc5ba5a29f66fb2f9c2e68"}, + {url = "https://files.pythonhosted.org/packages/d9/8b/a371c3fb5db326675720f66ac65f22f94f0b4c3830ccc65e0acecd352d2a/beanie-1.18.0-py3-none-any.whl", hash = "sha256:31f8eff8fe436e420766df457dc5bfd6025e927c9f8914adbc59ebff37fe6d7c"}, +] "beautifulsoup4 4.12.0" = [ {url = "https://files.pythonhosted.org/packages/c5/4c/b5b7d6e1d4406973fb7f4e5df81c6f07890fa82548ac3b945deed1df9d48/beautifulsoup4-4.12.0.tar.gz", hash = "sha256:c5fceeaec29d09c84970e47c65f2f0efe57872f7cff494c9691a26ec0ff13234"}, {url = "https://files.pythonhosted.org/packages/ee/a7/06b189a2e280e351adcef25df532af3c59442123187e228b960ab3238687/beautifulsoup4-4.12.0-py3-none-any.whl", hash = "sha256:2130a5ad7f513200fae61a17abb5e338ca980fa28c439c0571014bc0217e9591"}, @@ -570,6 +627,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/58/07/815476ae605bcc5f95c87a62b95e74a1bce0878bc7a3119bc2bf4178f175/distlib-0.3.6.tar.gz", hash = "sha256:14bad2d9b04d3a36127ac97f30b12a19268f211063d8f8ee4f47108896e11b46"}, {url = "https://files.pythonhosted.org/packages/76/cb/6bbd2b10170ed991cf64e8c8b85e01f2fb38f95d1bc77617569e0b0b26ac/distlib-0.3.6-py2.py3-none-any.whl", hash = "sha256:f35c4b692542ca110de7ef0bea44d73981caeb34ca0b9b6b2e6d7790dda8f80e"}, ] +"dnspython 2.3.0" = [ + {url = "https://files.pythonhosted.org/packages/12/86/d305e87555430ff4630d729420d97dece3b16efcbf2b7d7e974d11b0d86c/dnspython-2.3.0-py3-none-any.whl", hash = "sha256:89141536394f909066cabd112e3e1a37e4e654db00a25308b0f130bc3152eb46"}, + {url = "https://files.pythonhosted.org/packages/91/8b/522301c50ca1f78b09c2ca116ffb0fd797eadf6a76085d376c01f9dd3429/dnspython-2.3.0.tar.gz", hash = "sha256:224e32b03eb46be70e12ef6d64e0be123a64e621ab4c0822ff6d450d52a540b9"}, +] "docker 6.0.1" = [ {url = "https://files.pythonhosted.org/packages/79/26/6609b51ecb418e12d1534d00b888ce7e108f38b47dc6cd589598d5c6aaa2/docker-6.0.1.tar.gz", hash = "sha256:896c4282e5c7af5c45e8b683b0b0c33932974fe6e50fc6906a0a83616ab3da97"}, {url = "https://files.pythonhosted.org/packages/d5/b3/a5e41798a6d4b92880998e0d9e6980e57c5d039f7f7144f87627a6b19084/docker-6.0.1-py3-none-any.whl", hash = "sha256:dbcb3bd2fa80dca0788ed908218bf43972772009b881ed1e20dfc29a65e49782"}, @@ -606,6 +667,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/7a/ff/75c28576a1d900e87eb6335b063fab47a8ef3c8b4d88524c4bf78f670cce/Jinja2-3.1.2.tar.gz", hash = "sha256:31351a702a408a9e7595a8fc6150fc3f43bb6bf7e319770cbc0db9df9437e852"}, {url = "https://files.pythonhosted.org/packages/bc/c3/f068337a370801f372f2f8f6bad74a5c140f6fda3d9de154052708dd3c65/Jinja2-3.1.2-py3-none-any.whl", hash = "sha256:6088930bfe239f0e6710546ab9c19c9ef35e29792895fed6e6e31a023a182a61"}, ] +"lazy-model 0.0.5" = [ + {url = "https://files.pythonhosted.org/packages/0c/dd/0ef5eaa54e502c3e3229420f0c619bdd5a556417bab1ed74c1d8b23dd3e6/lazy-model-0.0.5.tar.gz", hash = "sha256:2d98f9dfe275012477555a439dceb56364793a0f266758d1a33267d68e8fbc76"}, + {url = "https://files.pythonhosted.org/packages/16/19/7d72b219dd73dd3e5c2212fcc54d58b255bd62d9469da39c830f8a41cc70/lazy_model-0.0.5-py3-none-any.whl", hash = "sha256:8b4fc5eac99029f84b11b21e81a6894911a475f25e53227b7e44833e62e26553"}, +] "markupsafe 2.1.2" = [ {url = "https://files.pythonhosted.org/packages/02/2c/18d55e5df6a9ea33709d6c33e08cb2e07d39e20ad05d8c6fbf9c9bcafd54/MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, {url = "https://files.pythonhosted.org/packages/04/cf/9464c3c41b7cdb8df660cda75676697e7fb49ce1be7691a1162fc88da078/MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, @@ -658,6 +723,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/ea/60/2400ba59cf2465fa136487ee7299f52121a9d04b2cf8539ad43ad10e70e8/MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, {url = "https://files.pythonhosted.org/packages/f9/aa/ebcd114deab08f892b1d70badda4436dbad1747f9e5b72cffb3de4c7129d/MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, ] +"motor 3.1.2" = [ + {url = "https://files.pythonhosted.org/packages/82/96/ae017cd62761d2fd2cc1eabfc902c3b4e3768fe994fc6a2f474694a56910/motor-3.1.2.tar.gz", hash = "sha256:80c08477c09e70db4f85c99d484f2bafa095772f1d29b3ccb253270f9041da9a"}, + {url = "https://files.pythonhosted.org/packages/f9/c3/22a695d0e6c373d0a33036de7fdc084068d896e948d11b691c88b6c1672f/motor-3.1.2-py3-none-any.whl", hash = "sha256:4bfc65230853ad61af447088527c1197f91c20ee957cfaea3144226907335716"}, +] "mypy 1.1.1" = [ {url = "https://files.pythonhosted.org/packages/2a/28/8485aad67750b3374443d28bad3eed947737cf425a640ea4be4ac70a7827/mypy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce61663faf7a8e5ec6f456857bfbcec2901fbdb3ad958b778403f63b9e606a1b"}, {url = "https://files.pythonhosted.org/packages/30/da/808ceaf2bcf23a9e90156c7b11b41add8dd5a009ee48159ec820d04d97bd/mypy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9401e33814cec6aec8c03a9548e9385e0e228fc1b8b0a37b9ea21038e64cdd8a"}, @@ -748,6 +817,82 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 {url = "https://files.pythonhosted.org/packages/f5/09/3f2ad426d20d2d353432f1c76290fa3c9863e2c04e05382ccca2aeade4c3/pydantic-1.10.6-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bbd5c531b22928e63d0cb1868dee76123456e1de2f1cb45879e9e7a3f3f1779b"}, {url = "https://files.pythonhosted.org/packages/f5/56/64028e205064748d6015a1afd6111c06f2b90982636850a3e157a7180ed5/pydantic-1.10.6-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:60184e80aac3b56933c71c48d6181e630b0fbc61ae455a63322a66a23c14731a"}, ] +"pymongo 4.3.3" = [ + {url = "https://files.pythonhosted.org/packages/01/10/e7157fcda1db4f759c858f8d9dc001112eb630136894056bb29f332137c3/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:bb869707d8e30645ed6766e44098600ca6cdf7989c22a3ea2b7966bb1d98d4b2"}, + {url = "https://files.pythonhosted.org/packages/05/17/185c96a98d3d91ad3cdfbc9bc91ad8bea697cfaf1b3ca314f52006f71d2b/pymongo-4.3.3-cp310-cp310-win32.whl", hash = "sha256:dc0cff74cd36d7e1edba91baa09622c35a8a57025f2f2b7a41e3f83b1db73186"}, + {url = "https://files.pythonhosted.org/packages/0e/8f/1009913e8ad51390966811e0163ed6df2dfa43a6f632ac35f53e51b2321b/pymongo-4.3.3-cp39-cp39-manylinux2014_aarch64.whl", hash = "sha256:e2961b05f9c04a53da8bfc72f1910b6aec7205fcf3ac9c036d24619979bbee4b"}, + {url = "https://files.pythonhosted.org/packages/0e/9f/a4986f0a86fc017599bf4c8912c01005a27c536acd221041234e0cb9739a/pymongo-4.3.3-cp38-cp38-manylinux2014_x86_64.whl", hash = "sha256:4ed00f96e147f40b565fe7530d1da0b0f3ab803d5dd5b683834500fa5d195ec4"}, + {url = "https://files.pythonhosted.org/packages/10/58/cdf21baff3328e6ba3b960918cd48302c3973e97ea4dcfbdf6ae5bf18408/pymongo-4.3.3-cp37-cp37m-manylinux1_i686.whl", hash = "sha256:6fcfbf435eebf8a1765c6d1f46821740ebe9f54f815a05c8fc30d789ef43cb12"}, + {url = "https://files.pythonhosted.org/packages/11/a3/8f7b87dbb9fd496f14c596bb02487fdb44dbb58e3c39da3f0eb0199b1523/pymongo-4.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cdb87309de97c63cb9a69132e1cb16be470e58cffdfbad68fdd1dc292b22a840"}, + {url = "https://files.pythonhosted.org/packages/22/18/68b8a63f289df40df27623c99779acd9eb6c007a4546700e676e07d7c2d6/pymongo-4.3.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:08fc250b5552ee97ceeae0f52d8b04f360291285fc7437f13daa516ce38fdbc6"}, + {url = "https://files.pythonhosted.org/packages/26/38/33270a35e265c1936ab3ea6863c02b9e3292ca013df9bd1e5ab1ed231ec7/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5effd87c7d363890259eac16c56a4e8da307286012c076223997f8cc4a8c435b"}, + {url = "https://files.pythonhosted.org/packages/29/3f/230c83a6be6e037f4558c9b7a2b8dc6de55ebc68662b0a13f9ff800614f7/pymongo-4.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:eac0a143ef4f28f49670bf89cb15847eb80b375d55eba401ca2f777cd425f338"}, + {url = "https://files.pythonhosted.org/packages/2c/5c/ab73b2fc15fd9930f07bce865c3f0d98fe90211b92889831a746d61d3830/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:12f3621a46cdc7a9ba8080422262398a91762a581d27e0647746588d3f995c88"}, + {url = "https://files.pythonhosted.org/packages/2c/c7/302a0fa990e5c2e1b137b94a5dfc174437a77872990c6c05b21779fe9502/pymongo-4.3.3-cp39-cp39-manylinux2014_ppc64le.whl", hash = "sha256:c1a70c51da9fa95bd75c167edb2eb3f3c4d27bc4ddd29e588f21649d014ec0b7"}, + {url = "https://files.pythonhosted.org/packages/2e/d8/d35fcd7fd6d9b55ab6b317182884938d34c64c91dce9ff5cf3548ca5cd30/pymongo-4.3.3-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:81d1a7303bd02ca1c5be4aacd4db73593f573ba8e0c543c04c6da6275fd7a47e"}, + {url = "https://files.pythonhosted.org/packages/2f/f0/33804cfc9113e0405063f0a777d213d9c006512cb06681a258ae559b3a8c/pymongo-4.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f3055510fdfdb1775bc8baa359783022f70bb553f2d46e153c094dfcb08578ff"}, + {url = "https://files.pythonhosted.org/packages/38/68/928d7ce22719cfa255fb973b34aed6f04ac3ea89049ce69e3b092c30a60f/pymongo-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1074f1a6f23e28b983c96142f2d45be03ec55d93035b471c26889a7ad2365db3"}, + {url = "https://files.pythonhosted.org/packages/39/22/e5acdce322f6aed2c6b06b8afae19c0fdf01031db1f7dbaeb34df60396c1/pymongo-4.3.3-cp39-cp39-manylinux1_i686.whl", hash = "sha256:016c412118e1c23fef3a1eada4f83ae6e8844fd91986b2e066fc1b0013cdd9ae"}, + {url = "https://files.pythonhosted.org/packages/39/97/3a04c850755723d64555ae29fdec2d4eafe9f2a12c22d4dc5e41e846423d/pymongo-4.3.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704d939656e21b073bfcddd7228b29e0e8a93dd27b54240eaafc0b9a631629a6"}, + {url = "https://files.pythonhosted.org/packages/3c/30/3d7e6336cfc795655a7193d77853972c5b502f58e1992205ad1b9bd28128/pymongo-4.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55b6163dac53ef1e5d834297810c178050bd0548a4136cd4e0f56402185916ca"}, + {url = "https://files.pythonhosted.org/packages/40/e3/dda96a2280058e08bc0dabeddf86bd3513e601f579134f2107680585636b/pymongo-4.3.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e7fac06a539daef4fcf5d8288d0d21b412f9b750454cd5a3cf90484665db442a"}, + {url = "https://files.pythonhosted.org/packages/42/0c/d2ad12aec55acdc4099134a8c87912d8fe01e2e1e5969b5d6c3485b99284/pymongo-4.3.3-cp37-cp37m-win_amd64.whl", hash = "sha256:54c377893f2cbbffe39abcff5ff2e917b082c364521fa079305f6f064e1a24a9"}, + {url = "https://files.pythonhosted.org/packages/45/2f/70f2e110a77dcb5490fe000aa380397968a09b8528f878aa1eadc0b11920/pymongo-4.3.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:74731c9e423c93cbe791f60c27030b6af6a948cef67deca079da6cd1bb583a8e"}, + {url = "https://files.pythonhosted.org/packages/48/b3/048d832794acb914cf8cf396089a29301ee79417e18f068f38a1eace9408/pymongo-4.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:cafa52873ae12baa512a8721afc20de67a36886baae6a5f394ddef0ce9391f91"}, + {url = "https://files.pythonhosted.org/packages/49/de/9005f70242f651fe4758a162eedbda13c9e55713083c345574c17cf8aa8f/pymongo-4.3.3-cp310-cp310-manylinux2014_aarch64.whl", hash = "sha256:9b87b23570565a6ddaa9244d87811c2ee9cffb02a753c8a2da9c077283d85845"}, + {url = "https://files.pythonhosted.org/packages/4a/92/9c11924649a557d95283882a4bcb67cfc32d6cb1528064a53c0bdb0540d7/pymongo-4.3.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:a9c2885b4a8e6e39db5662d8b02ca6dcec796a45e48c2de12552841f061692ba"}, + {url = "https://files.pythonhosted.org/packages/4b/ce/c6c6875dc14410952d3ff2e7960fb0498b1d9e70c483e5ce788c01fad54e/pymongo-4.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c0640b4e9d008e13956b004d1971a23377b3d45491f87082161c92efb1e6c0d6"}, + {url = "https://files.pythonhosted.org/packages/4e/ca/6c1cb5c69715c13312852d91cb62c175e7da58c91c428447db1a2364c646/pymongo-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d3a51901066696c4af38c6c63a1f0aeffd5e282367ff475de8c191ec9609b56d"}, + {url = "https://files.pythonhosted.org/packages/4f/2c/2da01e59e47cec96df562f0fe8ed6e1dd8b01a0ff8acd6d8ea1b59aaf82a/pymongo-4.3.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d86c35d94b5499689354ccbc48438a79f449481ee6300f3e905748edceed78e7"}, + {url = "https://files.pythonhosted.org/packages/4f/a9/32799279229f74f4d477f6c122dbbb4173f7d6d158bb9f7adf582c2ada20/pymongo-4.3.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:6dd1cf2995fdbd64fc0802313e8323f5fa18994d51af059b5b8862b73b5e53f0"}, + {url = "https://files.pythonhosted.org/packages/60/2f/6b18e099cfabf8fbe86ec201f53afa73a8b80e2e9dcbdef52429492d236e/pymongo-4.3.3-cp39-cp39-manylinux1_x86_64.whl", hash = "sha256:8fd6e191b92a10310f5a6cfe10d6f839d79d192fb02480bda325286bd1c7b385"}, + {url = "https://files.pythonhosted.org/packages/63/0e/ac6759051f18adf5506fe0c458bc12d03d9e94d2dc83087b21dc21888154/pymongo-4.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:d5571b6978750601f783cea07fb6b666837010ca57e5cefa389c1d456f6222e2"}, + {url = "https://files.pythonhosted.org/packages/63/74/51b2ec1b760169cbb19637913b86b6851dd9a57f95fe67adb7b0d1037469/pymongo-4.3.3-cp310-cp310-manylinux2014_ppc64le.whl", hash = "sha256:ffcc8394123ea8d43fff8e5d000095fe7741ce3f8988366c5c919c4f5eb179d3"}, + {url = "https://files.pythonhosted.org/packages/63/c8/a6e9f789cfbafc8293b5d94b0fa66b7a8854c6e74a04a74bc7585381ddd8/pymongo-4.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:be1d2ce7e269215c3ee9a215e296b7a744aff4f39233486d2c4d77f5f0c561a6"}, + {url = "https://files.pythonhosted.org/packages/66/b6/8e554ee180a28aa3f99200eb1ab60ab180fbea1a55f47166a6da2fd93299/pymongo-4.3.3-cp38-cp38-manylinux2014_i686.whl", hash = "sha256:d07d06dba5b5f7d80f9cc45501456e440f759fe79f9895922ed486237ac378a8"}, + {url = "https://files.pythonhosted.org/packages/69/d4/9cd99a5d98353b6c10595ec969c087d63a93ce60741b52463a9fcb2114ad/pymongo-4.3.3-cp39-cp39-win32.whl", hash = "sha256:dc24d245026a72d9b4953729d31813edd4bd4e5c13622d96e27c284942d33f24"}, + {url = "https://files.pythonhosted.org/packages/6a/6c/246b69b8fc3071e9ff1f42480fbc29835b95e910655604b66bef0a282e78/pymongo-4.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c184ec5be465c0319440734491e1aa4709b5f3ba75fdfc9dbbc2ae715a7f6829"}, + {url = "https://files.pythonhosted.org/packages/71/c7/c129dcde11ec97fe485cfc7a837284a0300bc4647a1bcb1e63f1ce050732/pymongo-4.3.3-cp37-cp37m-manylinux2014_s390x.whl", hash = "sha256:4d00b91c77ceb064c9b0459f0d6ea5bfdbc53ea9e17cf75731e151ef25a830c7"}, + {url = "https://files.pythonhosted.org/packages/74/7a/140e4c739319c3ee1163aa65bc91414ddf5b3c6376af19375e2dead1fbb5/pymongo-4.3.3-cp310-cp310-manylinux2014_s390x.whl", hash = "sha256:943f208840777f34312c103a2d1caab02d780c4e9be26b3714acf6c4715ba7e1"}, + {url = "https://files.pythonhosted.org/packages/74/a8/fe9d9c1f7d3a12b3d5c2b26fb267671a02f42b68ddd69d20105b6d87798b/pymongo-4.3.3-cp37-cp37m-manylinux2014_i686.whl", hash = "sha256:c09956606c08c4a7c6178a04ba2dd9388fcc5db32002ade9c9bc865ab156ab6d"}, + {url = "https://files.pythonhosted.org/packages/76/05/de90f39846ec83fe9e2099c7993266bb1a154f3a0777e78121f56fe08ee7/pymongo-4.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ef888f48eb9203ee1e04b9fb27429017b290fb916f1e7826c2f7808c88798394"}, + {url = "https://files.pythonhosted.org/packages/7d/33/aa74d9e5067bdd7b68cbe54ea5cad427883131d100c20d6a31ff0625a214/pymongo-4.3.3-cp39-cp39-manylinux2014_s390x.whl", hash = "sha256:8a06a0c02f5606330e8f2e2f3b7949877ca7e4024fa2bff5a4506bec66c49ec7"}, + {url = "https://files.pythonhosted.org/packages/81/37/c5c765526adb3f452ea4033d5d4e960514d53857b32c85fc2dfcac7aad86/pymongo-4.3.3-cp39-cp39-manylinux2014_i686.whl", hash = "sha256:b38a96b3eed8edc515b38257f03216f382c4389d022a8834667e2bc63c0c0c31"}, + {url = "https://files.pythonhosted.org/packages/81/5d/6d34f7b3cffe3efe38cac65de60beba7f1a14b8f5b64d27354bce33b924d/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:47f7aa217b25833cd6f0e72b0d224be55393c2692b4f5e0561cb3beeb10296e9"}, + {url = "https://files.pythonhosted.org/packages/81/f1/5d56b0ffdda842298334135ac181032ee4624bc57101a538d67ba8958695/pymongo-4.3.3-cp39-cp39-manylinux2014_x86_64.whl", hash = "sha256:6c2216d8b6a6d019c6f4b1ad55f890e5e77eb089309ffc05b6911c09349e7474"}, + {url = "https://files.pythonhosted.org/packages/89/24/52d65bbb0cf038d73b49c9d1f6b251500d807ed2579aaa55cf5d788513be/pymongo-4.3.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:fd7bb378d82b88387dc10227cfd964f6273eb083e05299e9b97cbe075da12d11"}, + {url = "https://files.pythonhosted.org/packages/8b/8f/93649909ec1ba88fee224884884b4e10ac26c0ca00c58f1781036476d30d/pymongo-4.3.3-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:599d3f6fbef31933b96e2d906b0f169b3371ff79ea6aaf6ecd76c947a3508a3d"}, + {url = "https://files.pythonhosted.org/packages/92/45/47134bdc3d628fa02945545c9d0cca1d7b349c507734860cf3614da77cb0/pymongo-4.3.3-cp38-cp38-manylinux2014_aarch64.whl", hash = "sha256:0c466710871d0026c190fc4141e810cf9d9affbf4935e1d273fbdc7d7cda6143"}, + {url = "https://files.pythonhosted.org/packages/93/da/d58cdba6e4c896300d1c939119c0911948a7edd94e10cc75048142e56160/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c2fdc855149efe7cdcc2a01ca02bfa24761c640203ea94df467f3baf19078be"}, + {url = "https://files.pythonhosted.org/packages/96/48/8baccdb480d0ceb2799d1b6d2da780b6f174635c64f82fa27bc8fbb9d660/pymongo-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7c051fe37c96b9878f37fa58906cb53ecd13dcb7341d3a85f1e2e2f6b10782d9"}, + {url = "https://files.pythonhosted.org/packages/97/9f/0156a752e50cfbc767a182c80a7e94174a772a94cb72a52f2660fc373c77/pymongo-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7b16250238de8dafca225647608dddc7bbb5dce3dd53b4d8e63c1cc287394c2f"}, + {url = "https://files.pythonhosted.org/packages/98/4d/4423858f2587a3c15c9b40a70e3672e0902667874f24e77e5388d848715d/pymongo-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:7761cacb8745093062695b11574effea69db636c2fd0a9269a1f0183712927b4"}, + {url = "https://files.pythonhosted.org/packages/9a/31/482f7401e7bbbeb66ab6b4ac263e2b50435f4329cce1e72378972d48f6b5/pymongo-4.3.3.tar.gz", hash = "sha256:34e95ffb0a68bffbc3b437f2d1f25fc916fef3df5cdeed0992da5f42fae9b807"}, + {url = "https://files.pythonhosted.org/packages/a0/53/f8b2099b2d8dcec0e4070455b6b7a9ea5088ee07a745b0c6a711d55a5357/pymongo-4.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:fc28e8d85d392a06434e9a934908d97e2cf453d69488d2bcd0bfb881497fd975"}, + {url = "https://files.pythonhosted.org/packages/a3/c6/ff88fce93529c9418c80854ecaf013254ab0b1d59f8f4fa2702419352d18/pymongo-4.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:b8a03af1ce79b902a43f5f694c4ca8d92c2a4195db0966f08f266549e2fc49bc"}, + {url = "https://files.pythonhosted.org/packages/a9/8c/5ae0d794ff1771dd2a298f1a7d0889455a65874a98120171a002c8cb741a/pymongo-4.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:52896e22115c97f1c829db32aa2760b0d61839cfe08b168c2b1d82f31dbc5f55"}, + {url = "https://files.pythonhosted.org/packages/b5/a2/a566780a2baeb108ae4b7e87add2c022090a39f728e9c808dee4c8b1efde/pymongo-4.3.3-cp37-cp37m-manylinux2014_ppc64le.whl", hash = "sha256:b0cfe925610f2fd59555bb7fc37bd739e4b197d33f2a8b2fae7b9c0c6640318c"}, + {url = "https://files.pythonhosted.org/packages/c4/3d/51e3ed544c4d4a0dbcafe197d582f9e922e73ea185bd5a19486c7c297308/pymongo-4.3.3-cp38-cp38-manylinux2014_s390x.whl", hash = "sha256:34b040e095e1671df0c095ec0b04fc4ebb19c4c160f87c2b55c079b16b1a6b00"}, + {url = "https://files.pythonhosted.org/packages/c4/b5/e2d246016d15c949736c9a4b4da4ec8e2045b661504be4b749c34188c2a5/pymongo-4.3.3-cp311-cp311-win32.whl", hash = "sha256:524d78673518dcd352a91541ecd2839c65af92dc883321c2109ef6e5cd22ef23"}, + {url = "https://files.pythonhosted.org/packages/c5/be/64441bc6f65ddca82ecb1231348c89257272c023a44658d59f044877a498/pymongo-4.3.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c29e758f0e734e1e90357ae01ec9c6daf19ff60a051192fe110d8fb25c62600e"}, + {url = "https://files.pythonhosted.org/packages/c6/1f/cd1d6d21620125693cd6d21eb9264b885df553f3c51cb778b06fe96d6abd/pymongo-4.3.3-cp38-cp38-manylinux2014_ppc64le.whl", hash = "sha256:711bc52cb98e7892c03e9b669bebd89c0a890a90dbc6d5bb2c47f30239bac6e9"}, + {url = "https://files.pythonhosted.org/packages/c9/02/77f30505aa009f329ec935a2e0e856889e19568c1c6c7af4dbffe894c27e/pymongo-4.3.3-cp38-cp38-manylinux1_i686.whl", hash = "sha256:dca34367a4e77fcab0693e603a959878eaf2351585e7d752cac544bc6b2dee46"}, + {url = "https://files.pythonhosted.org/packages/d4/4d/cdfde31b4545d2f0aaabae9a9acd0dda6384f3d02b4f7a4b6a483f4bf749/pymongo-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:5134d33286c045393c7beb51be29754647cec5ebc051cf82799c5ce9820a2ca2"}, + {url = "https://files.pythonhosted.org/packages/d6/58/a39537805ca205b4b65503765ca110224a409e777c1825fd6c8108ec9fd0/pymongo-4.3.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:341221e2f2866a5960e6f8610f4cbac0bb13097f3b1a289aa55aba984fc0d969"}, + {url = "https://files.pythonhosted.org/packages/dc/bc/1d69ee98cc0b50278f9c6044666a5dbd8b296e8bd3af733066f6bb8bc597/pymongo-4.3.3-cp37-cp37m-win32.whl", hash = "sha256:49210feb0be8051a64d71691f0acbfbedc33e149f0a5d6e271fddf6a12493fed"}, + {url = "https://files.pythonhosted.org/packages/df/2c/572e43db59a870b8df3332b94bd29ee7246bcba8cbb071b61174ecd1c834/pymongo-4.3.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:316498b642c00401370b2156b5233b256f9b33799e0a8d9d0b8a7da217a20fca"}, + {url = "https://files.pythonhosted.org/packages/e1/ea/ca13d38405cea315683b085cfcf661cf48be9a9a786dcead86d9454fcc18/pymongo-4.3.3-cp37-cp37m-macosx_10_6_intel.whl", hash = "sha256:39b03045c71f761aee96a12ebfbc2f4be89e724ff6f5e31c2574c1a0e2add8bd"}, + {url = "https://files.pythonhosted.org/packages/e2/7c/a076b118f1b7aea6c8dc548d45441801b86486bf67765589112e28ea188d/pymongo-4.3.3-cp37-cp37m-manylinux2014_x86_64.whl", hash = "sha256:c6258a3663780ae47ba73d43eb63c79c40ffddfb764e09b56df33be2f9479837"}, + {url = "https://files.pythonhosted.org/packages/e7/e1/e2c577333ee346b411db65d9f62c746eca8b1062c55afdb5d2fb8ebc23fe/pymongo-4.3.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa7e202feb683dad74f00dea066690448d0cfa310f8a277db06ec8eb466601b5"}, + {url = "https://files.pythonhosted.org/packages/ee/2a/223a77aab2d1d9f2ca86b1db60578f25ebd2f1c0f558fcf46d05457865d1/pymongo-4.3.3-cp38-cp38-manylinux1_x86_64.whl", hash = "sha256:cd6a4afb20fb3c26a7bfd4611a0bbb24d93cbd746f5eb881f114b5e38fd55501"}, + {url = "https://files.pythonhosted.org/packages/f0/25/5331b822a0e2486efe75c741fa9dcb500b67ecfb0223f26179afa60f1c17/pymongo-4.3.3-cp37-cp37m-manylinux2014_aarch64.whl", hash = "sha256:3b93043b14ba7eb08c57afca19751658ece1cfa2f0b7b1fb5c7a41452fbb8482"}, + {url = "https://files.pythonhosted.org/packages/f2/4a/68ab4706a992fd7b01ec53a9e2138733972895b260578e544221845770dd/pymongo-4.3.3-cp310-cp310-manylinux1_i686.whl", hash = "sha256:66413c50d510e5bcb0afc79880d1693a2185bcea003600ed898ada31338c004e"}, + {url = "https://files.pythonhosted.org/packages/f3/87/f2ccd99ea5184d9a9013acca92f3060e29253038df8003148b1a643e6165/pymongo-4.3.3-cp310-cp310-manylinux2014_i686.whl", hash = "sha256:695939036a320f4329ccf1627edefbbb67cc7892b8222d297b0dd2313742bfee"}, + {url = "https://files.pythonhosted.org/packages/f4/d6/3088b63536c74c4e9cf687916712843e7d4abfc981eca3e264ec801372af/pymongo-4.3.3-cp38-cp38-win32.whl", hash = "sha256:a6cd6f1db75eb07332bd3710f58f5fce4967eadbf751bad653842750a61bda62"}, + {url = "https://files.pythonhosted.org/packages/f8/ef/bd801e889305bc48ca3210569ea613d66a52c717578a465ac2792cec709a/pymongo-4.3.3-cp310-cp310-manylinux2014_x86_64.whl", hash = "sha256:01f7cbe88d22440b6594c955e37312d932fd632ffed1a86d0c361503ca82cc9d"}, + {url = "https://files.pythonhosted.org/packages/fa/6a/bf5391534a10cfb4a2b4a9e6697f17115fc460da8041ec67835c23d2ff59/pymongo-4.3.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a966d5304b7d90c45c404914e06bbf02c5bf7e99685c6c12f0047ef2aa837142"}, + {url = "https://files.pythonhosted.org/packages/fc/28/1b934e5839bf12b022782561c803ee63149737d6c5f9627d3299cd28516f/pymongo-4.3.3-cp37-cp37m-manylinux1_x86_64.whl", hash = "sha256:7d43ac9c7eeda5100fb0a7152fab7099c9cf9e5abd3bb36928eb98c7d7a339c6"}, +] "pytest 7.2.2" = [ {url = "https://files.pythonhosted.org/packages/b2/68/5321b5793bd506961bd40bdbdd0674e7de4fb873ee7cab33dd27283ad513/pytest-7.2.2-py3-none-any.whl", hash = "sha256:130328f552dcfac0b1cec75c12e3f005619dc5f874f0a06e8ff7263f0ee6225e"}, {url = "https://files.pythonhosted.org/packages/b9/29/311895d9cd3f003dd58e8fdea36dd895ba2da5c0c90601836f7de79f76fe/pytest-7.2.2.tar.gz", hash = "sha256:c99ab0c73aceb050f68929bc93af19ab6db0558791c6a0715723abe9d0ade9d4"}, @@ -799,6 +944,10 @@ content_hash = "sha256:41531e3c394e45bd34b4d98fb20ef889335291b731a867850f266b070 "testcontainers 3.7.1" = [ {url = "https://files.pythonhosted.org/packages/b3/37/38c595414d764cb1d9f3a0c907878c4146a21505ab974c63bcf3d8145807/testcontainers-3.7.1-py2.py3-none-any.whl", hash = "sha256:7f48cef4bf0ccd78f1a4534d4b701a003a3bace851f24eae58a32f9e3f0aeba0"}, ] +"toml 0.10.2" = [ + {url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b"}, + {url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, +] "types-beautifulsoup4 4.12.0.0" = [ {url = "https://files.pythonhosted.org/packages/92/0b/39afb220c7d8328c5c887007e17c950eda2c2e9300132b69e923e81ff033/types_beautifulsoup4-4.12.0.0-py3-none-any.whl", hash = "sha256:43c23852a6ef0053632b9a308fc3488831c0f3e02c0f4b4478a28703217cf683"}, {url = "https://files.pythonhosted.org/packages/a4/23/9a9131dedfbd64354fabedef74c8b69092afa4c65720b8fb35df18ded18b/types-beautifulsoup4-4.12.0.0.tar.gz", hash = "sha256:3859e70d3118d65d12ebfca109304de4bf52383e6f99f941c114fd1153bb6cc1"}, diff --git a/pyproject.toml b/pyproject.toml index c245e85..97ab010 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,6 +17,7 @@ dependencies = [ "websockets>=10.4", "uvicorn>=0.19.0", "httpx>=0.23.3", + "beanie>=1.18.0", ] requires-python = ">=3.11" license = { text = "MIT" } diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index e901414..5b070dc 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -6,17 +6,28 @@ import pytest_asyncio from fastapi.testclient import TestClient from httpx import Response +from testcontainers.mongodb import MongoDbContainer from ocrdbrowser import ChannelClosed +from ocrdmonitor import dbmodel +from ocrdmonitor.server.app import create_app from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures import WORKSPACE_DIR, patch_factory -from tests.testdoubles import BrowserFake +from tests.ocrdmonitor.server.fixtures import ( + WORKSPACE_DIR, + create_settings, + patch_factory, +) +from tests.testdoubles import ( + Browser_Heading, + BrowserFake, + BrowserSpy, + BrowserTestDouble, +) from tests.testdoubles._browserfactory import ( BrowserTestDoubleFactory, IteratingBrowserTestDoubleFactory, SingletonBrowserTestDoubleFactory, ) -from tests.testdoubles import Browser_Heading, BrowserSpy, BrowserTestDouble class DisconnectingChannel: @@ -174,3 +185,23 @@ def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( result = app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 502 + + +@pytest.mark.asyncio +async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( + monkeypatch: pytest.MonkeyPatch, +) -> None: + with MongoDbContainer() as container: + await dbmodel.init(container.get_connection_url()) + + async with patch_factory(IteratingBrowserTestDoubleFactory()): + settings = create_settings() + app = TestClient(create_app(settings)) + + _ = view_workspace(app, "a_workspace") + + found_browsers = await dbmodel.BrowserProcess.find_all( + dbmodel.BrowserProcess.workspace == "a_workspace" + ).count() + + assert found_browsers == 1 From 3f0e307184aad33604992a7fcf5331f53fafa9f3 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Wed, 3 May 2023 16:59:34 +0200 Subject: [PATCH 05/32] Passes first DB test. Needs refactoring! --- ocrdbrowser/_browser.py | 3 ++ ocrdbrowser/_docker.py | 3 ++ ocrdbrowser/_subprocess.py | 6 ++++ ocrdmonitor/dbmodel.py | 2 ++ ocrdmonitor/server/workspaces.py | 6 ++++ pyproject.toml | 6 ++-- .../server/test_workspace_endpoint.py | 31 ++++++++++++------- tests/testdoubles/_backgroundprocess.py | 7 +++++ tests/testdoubles/_browserfake.py | 3 ++ tests/testdoubles/_browserspy.py | 7 ++++- 10 files changed, 58 insertions(+), 16 deletions(-) diff --git a/ocrdbrowser/_browser.py b/ocrdbrowser/_browser.py index aa9d713..7336171 100644 --- a/ocrdbrowser/_browser.py +++ b/ocrdbrowser/_browser.py @@ -6,6 +6,9 @@ class OcrdBrowser(Protocol): + def process_id(self) -> str: + ... + def address(self) -> str: ... diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index 4858eb1..78cc76b 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -30,6 +30,9 @@ def __init__(self, host: str, port: Port, owner: str, workspace: str) -> None: self._workspace = path.abspath(workspace) self.id: str | None = None + def process_id(self) -> str: + return self._container_name() + def address(self) -> str: return f"{self._host}:{self._port}" diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 832ea09..8110325 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -20,6 +20,12 @@ def __init__(self, localport: Port, owner: str, workspace: str) -> None: self._workspace = workspace self._process: Optional[asyncio.subprocess.Process] = None + def process_id(self) -> str: + if not self._process: + raise AttributeError + + return str(self._process.pid) + def address(self) -> str: # as long as we do not have a reverse proxy on BW_PORT, # we must map the local port range to the exposed range diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index f263763..168d78d 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -1,3 +1,4 @@ +import asyncio import pymongo from motor.motor_asyncio import AsyncIOMotorClient from beanie import Document, init_beanie @@ -21,6 +22,7 @@ class Settings: async def init(connection_str: str) -> None: client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) + client.get_io_loop = asyncio.get_event_loop await init_beanie( database=client.browsers, document_models=[BrowserProcess], # type: ignore diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index 0dfabab..e6111aa 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -5,6 +5,7 @@ import ocrdbrowser +from ocrdmonitor import dbmodel import ocrdmonitor.server.proxy as proxy from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect from fastapi.templating import Jinja2Templates @@ -40,6 +41,11 @@ async def browser(request: Request, workspace: Path) -> Response: if (session_id, workspace) not in redirects: browser = await launch_browser(session_id, workspace) redirects.add(session_id, workspace, browser) + await dbmodel.BrowserProcess( # type: ignore + process_id=browser.process_id(), + owner=browser.owner(), + workspace=browser.workspace(), + ).insert() return response diff --git a/pyproject.toml b/pyproject.toml index 97ab010..24ef407 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,9 +32,7 @@ dev = [ "types-requests>=2.28.11.15", "types-beautifulsoup4>=4.12.0.0", ] -nox = [ - "nox>=2022.11.21", -] +nox = ["nox>=2022.11.21"] [tool.mypy] plugins = ["pydantic.mypy"] @@ -50,7 +48,7 @@ test = "pytest tests -m 'not integration'" test-integration = "pytest tests" [[tool.mypy.overrides]] -module = "testcontainers.*" +module = ["testcontainers.*", "motor.motor_asyncio"] ignore_missing_imports = true [build-system] diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 5b070dc..8534d0a 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -11,6 +11,7 @@ from ocrdbrowser import ChannelClosed from ocrdmonitor import dbmodel from ocrdmonitor.server.app import create_app +from ocrdmonitor.server.settings import OcrdBrowserSettings from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.fixtures import ( WORKSPACE_DIR, @@ -187,21 +188,29 @@ def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( assert result.status_code == 502 +@pytest_asyncio.fixture(autouse=True) +async def db() -> AsyncIterator[None]: + with MongoDbContainer() as container: + await dbmodel.init(container.get_connection_url()) + yield + + @pytest.mark.asyncio async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( monkeypatch: pytest.MonkeyPatch, ) -> None: - with MongoDbContainer() as container: - await dbmodel.init(container.get_connection_url()) - - async with patch_factory(IteratingBrowserTestDoubleFactory()): - settings = create_settings() - app = TestClient(create_app(settings)) + async with patch_factory(IteratingBrowserTestDoubleFactory()): + settings = create_settings() + app = TestClient(create_app(settings)) - _ = view_workspace(app, "a_workspace") + _ = view_workspace(app, "a_workspace") - found_browsers = await dbmodel.BrowserProcess.find_all( - dbmodel.BrowserProcess.workspace == "a_workspace" - ).count() + found_browsers = ( + await dbmodel.BrowserProcess.find( + dbmodel.BrowserProcess.workspace == str(WORKSPACE_DIR / "a_workspace") + ) + .find(dbmodel.BrowserProcess.process_id == "1234") + .to_list() + ) - assert found_browsers == 1 + assert len(found_browsers) == 1 diff --git a/tests/testdoubles/_backgroundprocess.py b/tests/testdoubles/_backgroundprocess.py index 159682e..a2dc529 100644 --- a/tests/testdoubles/_backgroundprocess.py +++ b/tests/testdoubles/_backgroundprocess.py @@ -25,6 +25,13 @@ def __exit__(self, *args: Any, **kwargs: Any) -> None: def is_running(self) -> bool: return self._process is not None and self._process.is_alive() + @property + def pid(self) -> int | None: + if not self._process: + return None + + return self._process.pid + def launch(self) -> None: if self.is_running: return diff --git a/tests/testdoubles/_browserfake.py b/tests/testdoubles/_browserfake.py index 3fcd9b9..b3c89c5 100644 --- a/tests/testdoubles/_browserfake.py +++ b/tests/testdoubles/_browserfake.py @@ -21,6 +21,9 @@ def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self._workspace = workspace self._browser = broadway_fake(workspace) + def process_id(self) -> str: + return str(self._browser.pid) + def address(self) -> str: return f"http://{FAKE_HOST_ADDRESS}" diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 212e081..57e816d 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -47,13 +47,15 @@ async def open_channel(self) -> AsyncGenerator[Channel, None]: class BrowserSpy: def __init__( self, + process_id: str = "1234", owner: str = "", workspace_path: str = "", address: str = "http://unreachable.example.com", running: bool = False, ) -> None: - self.is_running = running self._address = address + self._process_id = process_id + self.is_running = running self.owner_name = owner self.workspace_path = workspace_path self._client = BrowserClientStub() @@ -67,6 +69,9 @@ def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self.owner_name = owner self.workspace_path = workspace + def process_id(self) -> str: + return self._process_id + def address(self) -> str: return self._address From c908ad817bad26f28d7e52b36e16514cbaf6d171 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Thu, 4 May 2023 16:50:12 +0200 Subject: [PATCH 06/32] Begin implementing repository --- ocrdmonitor/browserprocess.py | 28 ++++++++++ ocrdmonitor/dbmodel.py | 52 ++++++++++++++++++- .../server/test_workspace_endpoint.py | 20 +++---- 3 files changed, 86 insertions(+), 14 deletions(-) create mode 100644 ocrdmonitor/browserprocess.py diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py new file mode 100644 index 0000000..801d059 --- /dev/null +++ b/ocrdmonitor/browserprocess.py @@ -0,0 +1,28 @@ +from typing import Collection, Protocol + + +class BrowserProcess(Protocol): + @property + def process_id(self) -> str: + ... + + @property + def workspace(self) -> str: + ... + + @property + def owner(self) -> str: + ... + + +class BrowserProcessRepository(Protocol): + async def insert(self, browser: BrowserProcess) -> None: + ... + + async def delete(self, browser: BrowserProcess) -> None: + ... + + async def find( + self, owner: str, workspace: str, process_id: str | None = None + ) -> Collection[BrowserProcess]: + ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 168d78d..e28f102 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -1,7 +1,12 @@ import asyncio +from typing import Any, Collection, Mapping + import pymongo -from motor.motor_asyncio import AsyncIOMotorClient from beanie import Document, init_beanie +from beanie.odm.queries.find import FindMany +from motor.motor_asyncio import AsyncIOMotorClient + +from ocrdmonitor.browserprocess import BrowserProcess as BrowserProcessProtocol class BrowserProcess(Document): @@ -20,6 +25,51 @@ class Settings: ] +class MongoBrowserProcessRepository: + async def insert(self, browser: BrowserProcessProtocol) -> None: + await BrowserProcess( + owner=browser.owner, + process_id=browser.process_id, + workspace=browser.workspace, + ).insert() + + async def delete(self, browser: BrowserProcessProtocol) -> None: + await BrowserProcess( + owner=browser.owner, + process_id=browser.process_id, + workspace=browser.workspace, + ).delete() + + async def find( + self, + *, + owner: str | None = None, + workspace: str | None = None, + process_id: str | None = None, + ) -> Collection[BrowserProcess]: + results: FindMany[BrowserProcess] | None = None + + def find( + results: FindMany[BrowserProcess] | None, + *predicates: Mapping[str, Any] | bool, + ) -> FindMany[BrowserProcess]: + if results is None: + return BrowserProcess.find(*predicates) + + return results.find(*predicates) + + if owner is not None: + results = find(results, BrowserProcess.owner == owner) + + if workspace is not None: + results = find(results, BrowserProcess.workspace == workspace) + + if process_id is not None: + results = find(results, BrowserProcess.process_id == process_id) + + return await results.to_list() if results is not None else [] + + async def init(connection_str: str) -> None: client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) client.get_io_loop = asyncio.get_event_loop diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 8534d0a..70f4ce2 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -196,21 +196,15 @@ async def db() -> AsyncIterator[None]: @pytest.mark.asyncio +@pytest.mark.usefixtures("iterating_factory") async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( - monkeypatch: pytest.MonkeyPatch, + monkeypatch: pytest.MonkeyPatch, app: TestClient ) -> None: - async with patch_factory(IteratingBrowserTestDoubleFactory()): - settings = create_settings() - app = TestClient(create_app(settings)) - - _ = view_workspace(app, "a_workspace") - - found_browsers = ( - await dbmodel.BrowserProcess.find( - dbmodel.BrowserProcess.workspace == str(WORKSPACE_DIR / "a_workspace") - ) - .find(dbmodel.BrowserProcess.process_id == "1234") - .to_list() + repo = dbmodel.MongoBrowserProcessRepository() + _ = view_workspace(app, "a_workspace") + + found_browsers = await repo.find( + workspace=str(WORKSPACE_DIR / "a_workspace"), process_id="1234" ) assert len(found_browsers) == 1 From aaf6fecf97eaf88b0f1c718e5f1762fae2b90607 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Fri, 5 May 2023 15:51:51 +0200 Subject: [PATCH 07/32] Refactor DB test. Make repository setup async --- .gitignore | 2 +- ocrdmonitor/browserprocess.py | 9 +-- ocrdmonitor/dbmodel.py | 16 ++--- ocrdmonitor/main.py | 3 +- ocrdmonitor/server/app.py | 3 +- ocrdmonitor/server/settings.py | 8 ++- ocrdmonitor/server/workspaces.py | 18 ++--- tests/ocrdmonitor/server/conftest.py | 2 +- tests/ocrdmonitor/server/fixtures.py | 65 +++++++++++++++++-- tests/ocrdmonitor/server/test_settings.py | 1 + .../server/test_workspace_endpoint.py | 32 ++++----- tests/testdoubles/__init__.py | 2 + .../testdoubles/_browserprocessrepository.py | 35 ++++++++++ 13 files changed, 146 insertions(+), 50 deletions(-) create mode 100644 tests/testdoubles/_browserprocessrepository.py diff --git a/.gitignore b/.gitignore index 1c78b23..76498d3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ authorized_keys __pycache__/ .python-version -.pdm.toml +.pdm-python .pdm.lock diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py index 801d059..c43aafb 100644 --- a/ocrdmonitor/browserprocess.py +++ b/ocrdmonitor/browserprocess.py @@ -2,15 +2,12 @@ class BrowserProcess(Protocol): - @property def process_id(self) -> str: ... - @property def workspace(self) -> str: ... - @property def owner(self) -> str: ... @@ -23,6 +20,10 @@ async def delete(self, browser: BrowserProcess) -> None: ... async def find( - self, owner: str, workspace: str, process_id: str | None = None + self, + *, + owner: str | None = None, + workspace: str | None = None, + process_id: str | None = None, ) -> Collection[BrowserProcess]: ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index e28f102..9e9d40a 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -27,17 +27,17 @@ class Settings: class MongoBrowserProcessRepository: async def insert(self, browser: BrowserProcessProtocol) -> None: - await BrowserProcess( - owner=browser.owner, - process_id=browser.process_id, - workspace=browser.workspace, + await BrowserProcess( # type: ignore + owner=browser.owner(), + process_id=browser.process_id(), + workspace=browser.workspace(), ).insert() async def delete(self, browser: BrowserProcessProtocol) -> None: - await BrowserProcess( - owner=browser.owner, - process_id=browser.process_id, - workspace=browser.workspace, + await BrowserProcess( # type: ignore + owner=browser.owner(), + process_id=browser.process_id(), + workspace=browser.workspace(), ).delete() async def find( diff --git a/ocrdmonitor/main.py b/ocrdmonitor/main.py index 409c26d..47a21bf 100644 --- a/ocrdmonitor/main.py +++ b/ocrdmonitor/main.py @@ -1,5 +1,6 @@ +import asyncio from ocrdmonitor.server.settings import Settings from ocrdmonitor.server.app import create_app settings = Settings() -app = create_app(settings) +app = asyncio.get_event_loop().run_until_complete(create_app(settings)) diff --git a/ocrdmonitor/server/app.py b/ocrdmonitor/server/app.py index df1a593..3c81a07 100644 --- a/ocrdmonitor/server/app.py +++ b/ocrdmonitor/server/app.py @@ -20,7 +20,7 @@ TEMPLATE_DIR = PKG_DIR / "templates" -def create_app(settings: Settings) -> FastAPI: +async def create_app(settings: Settings) -> FastAPI: app = FastAPI() templates = Jinja2Templates(TEMPLATE_DIR) app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") @@ -44,6 +44,7 @@ async def swallow_exceptions(request: Request, err: Exception) -> Response: create_workspaces( templates, settings.ocrd_browser.factory(), + await settings.ocrd_browser.repository(), settings.ocrd_browser.workspace_dir, ) ) diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index 8a09a86..6e9c003 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -12,6 +12,8 @@ OcrdBrowserFactory, SubProcessOcrdBrowserFactory, ) +from ocrdmonitor import dbmodel +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.sshremote import SSHRemote @@ -36,7 +38,11 @@ class OcrdBrowserSettings(BaseModel): workspace_dir: Path mode: Literal["native", "docker"] = "native" port_range: tuple[int, int] - # db_connection_string: str + db_connection_string: str + + async def repository(self) -> BrowserProcessRepository: + await dbmodel.init(self.db_connection_string) + return dbmodel.MongoBrowserProcessRepository() def factory(self) -> OcrdBrowserFactory: port_range_set = set(range(*self.port_range)) diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index e6111aa..0cc76a4 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -3,18 +3,22 @@ import uuid from pathlib import Path +from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect +from fastapi.templating import Jinja2Templates import ocrdbrowser -from ocrdmonitor import dbmodel import ocrdmonitor.server.proxy as proxy -from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect -from fastapi.templating import Jinja2Templates from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace +from ocrdmonitor import dbmodel +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.server.redirect import RedirectMap def create_workspaces( - templates: Jinja2Templates, factory: OcrdBrowserFactory, workspace_dir: Path + templates: Jinja2Templates, + factory: OcrdBrowserFactory, + repository: BrowserProcessRepository, + workspace_dir: Path, ) -> APIRouter: router = APIRouter(prefix="/workspaces") @@ -41,11 +45,7 @@ async def browser(request: Request, workspace: Path) -> Response: if (session_id, workspace) not in redirects: browser = await launch_browser(session_id, workspace) redirects.add(session_id, workspace, browser) - await dbmodel.BrowserProcess( # type: ignore - process_id=browser.process_id(), - owner=browser.owner(), - workspace=browser.workspace(), - ).insert() + await repository.insert(browser) return response diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index 8905966..fb7fd0d 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -1 +1 @@ -from .fixtures import app, launch_monitor +from .fixtures import app, launch_monitor, repository diff --git a/tests/ocrdmonitor/server/fixtures.py b/tests/ocrdmonitor/server/fixtures.py index 8a5b5cd..4e538f9 100644 --- a/tests/ocrdmonitor/server/fixtures.py +++ b/tests/ocrdmonitor/server/fixtures.py @@ -1,12 +1,22 @@ -from contextlib import asynccontextmanager +from contextlib import asynccontextmanager, contextmanager from pathlib import Path -from typing import AsyncIterator, Iterator +from typing import ( + AsyncContextManager, + AsyncIterator, + Callable, + ContextManager, + Iterator, +) from unittest.mock import patch import pytest +import pytest_asyncio import uvicorn from fastapi.testclient import TestClient +from testcontainers.mongodb import MongoDbContainer +from ocrdmonitor import dbmodel +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.server.app import create_app from ocrdmonitor.server.settings import ( OcrdBrowserSettings, @@ -14,7 +24,11 @@ OcrdLogViewSettings, Settings, ) -from tests.testdoubles import BackgroundProcess, BrowserTestDoubleFactory +from tests.testdoubles import ( + BackgroundProcess, + BrowserTestDoubleFactory, + InMemoryBrowserProcessRepository, +) JOB_DIR = Path(__file__).parent / "ocrd.jobs" WORKSPACE_DIR = Path("tests") / "workspaces" @@ -25,6 +39,7 @@ def create_settings() -> Settings: ocrd_browser=OcrdBrowserSettings( workspace_dir=WORKSPACE_DIR, port_range=(9000, 9100), + db_connection_string="", ), ocrd_controller=OcrdControllerSettings( job_dir=JOB_DIR, @@ -44,9 +59,47 @@ async def patch_factory( yield factory -@pytest.fixture -def app() -> TestClient: - return TestClient(create_app(create_settings())) +@asynccontextmanager +async def mongodb_repository() -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: + with MongoDbContainer() as container: + await dbmodel.init(container.get_connection_url()) + yield dbmodel.MongoBrowserProcessRepository() + + +@asynccontextmanager +async def inmemory_repository() -> AsyncIterator[InMemoryBrowserProcessRepository]: + yield InMemoryBrowserProcessRepository() + + +@pytest_asyncio.fixture( + autouse=True, + params=[ + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, pytest.mark.needs_docker), + ), + ], +) +async def repository( + monkeypatch: pytest.MonkeyPatch, + request: pytest.FixtureRequest, +) -> AsyncIterator[BrowserProcessRepository]: + repository_constructor: Callable[ + [], AsyncContextManager[BrowserProcessRepository] + ] = request.param + async with repository_constructor() as repository: + + async def async_repository(self: OcrdBrowserSettings) -> BrowserProcessRepository: + return repository + + monkeypatch.setattr(OcrdBrowserSettings, "repository", async_repository) + yield repository + + +@pytest_asyncio.fixture +async def app() -> TestClient: + return TestClient(await create_app(create_settings())) def _launch_app() -> None: diff --git a/tests/ocrdmonitor/server/test_settings.py b/tests/ocrdmonitor/server/test_settings.py index 369bb34..d71bf79 100644 --- a/tests/ocrdmonitor/server/test_settings.py +++ b/tests/ocrdmonitor/server/test_settings.py @@ -17,6 +17,7 @@ mode="native", workspace_dir="path/to/workdir", port_range=(9000, 9100), + db_connection_string="user@mongo:mongodb:1234" ), ocrd_controller=OcrdControllerSettings( job_dir="path/to/jobdir", diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 70f4ce2..d34b891 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -1,6 +1,14 @@ from __future__ import annotations -from typing import AsyncIterator, cast +from contextlib import asynccontextmanager, contextmanager +from typing import ( + AsyncContextManager, + AsyncIterator, + Callable, + ContextManager, + Iterator, + cast, +) import pytest import pytest_asyncio @@ -10,12 +18,11 @@ from ocrdbrowser import ChannelClosed from ocrdmonitor import dbmodel -from ocrdmonitor.server.app import create_app +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.server.settings import OcrdBrowserSettings from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.fixtures import ( WORKSPACE_DIR, - create_settings, patch_factory, ) from tests.testdoubles import ( @@ -23,9 +30,8 @@ BrowserFake, BrowserSpy, BrowserTestDouble, -) -from tests.testdoubles._browserfactory import ( BrowserTestDoubleFactory, + InMemoryBrowserProcessRepository, IteratingBrowserTestDoubleFactory, SingletonBrowserTestDoubleFactory, ) @@ -188,23 +194,13 @@ def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( assert result.status_code == 502 -@pytest_asyncio.fixture(autouse=True) -async def db() -> AsyncIterator[None]: - with MongoDbContainer() as container: - await dbmodel.init(container.get_connection_url()) - yield - - @pytest.mark.asyncio @pytest.mark.usefixtures("iterating_factory") -async def test__process_stored_in_db__browsing_workspace__proxies_to_browser( - monkeypatch: pytest.MonkeyPatch, app: TestClient +async def test__browsing_workspace__stores_browser_in_repository( + repository: BrowserProcessRepository, app: TestClient ) -> None: - repo = dbmodel.MongoBrowserProcessRepository() _ = view_workspace(app, "a_workspace") - found_browsers = await repo.find( - workspace=str(WORKSPACE_DIR / "a_workspace"), process_id="1234" - ) + found_browsers = await repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) assert len(found_browsers) == 1 diff --git a/tests/testdoubles/__init__.py b/tests/testdoubles/__init__.py index 9832aee..6467af9 100644 --- a/tests/testdoubles/__init__.py +++ b/tests/testdoubles/__init__.py @@ -8,6 +8,7 @@ ) from ._browserfake import BrowserFake from ._browserspy import BrowserSpy, Browser_Heading +from ._browserprocessrepository import InMemoryBrowserProcessRepository __all__ = [ "BackgroundProcess", @@ -20,4 +21,5 @@ "FAKE_HOST_ADDRESS", "SingletonBrowserTestDoubleFactory", "IteratingBrowserTestDoubleFactory", + "InMemoryBrowserProcessRepository", ] diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py new file mode 100644 index 0000000..9959918 --- /dev/null +++ b/tests/testdoubles/_browserprocessrepository.py @@ -0,0 +1,35 @@ +from typing import Collection +from ocrdmonitor.browserprocess import BrowserProcess + + +class InMemoryBrowserProcessRepository: + def __init__(self) -> None: + self._processes: list[BrowserProcess] = [] + + async def insert(self, browser: BrowserProcess) -> None: + self._processes.append(browser) + + async def delete(self, browser: BrowserProcess) -> None: + self._processes.remove(browser) + + async def find( + self, + *, + owner: str | None = None, + workspace: str | None = None, + process_id: str | None = None, + ) -> Collection[BrowserProcess]: + def match(browser: BrowserProcess) -> bool: + matches = True + if owner is not None: + matches = matches and browser.owner() == owner + + if workspace is not None: + matches = matches and browser.workspace() == workspace + + if process_id is not None: + matches = matches and browser.process_id() == process_id + + return matches + + return list(filter(match, self._processes)) From 4e625e872598f06d938c5fbc912e139049aa89a2 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Thu, 11 May 2023 17:35:55 +0200 Subject: [PATCH 08/32] Work on test setup --- ocrdbrowser/_browser.py | 2 +- ocrdmonitor/browserprocess.py | 20 ++--- ocrdmonitor/dbmodel.py | 21 +++-- ocrdmonitor/server/workspaces.py | 28 +++++-- tests/ocrdmonitor/server/conftest.py | 2 +- tests/ocrdmonitor/server/fixtures.py | 58 +++++++++----- .../server/test_workspace_endpoint.py | 76 ++++++++++++++----- .../testdoubles/_browserprocessrepository.py | 38 ++++++---- tests/testdoubles/_browserspy.py | 6 +- 9 files changed, 170 insertions(+), 81 deletions(-) diff --git a/ocrdbrowser/_browser.py b/ocrdbrowser/_browser.py index 7336171..99af25c 100644 --- a/ocrdbrowser/_browser.py +++ b/ocrdbrowser/_browser.py @@ -51,7 +51,7 @@ def open_channel(self) -> AsyncContextManager[Channel]: class OcrdBrowserFactory(Protocol): def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: ... - + BrowserProcesses = set[OcrdBrowser] diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py index c43aafb..0a3e895 100644 --- a/ocrdmonitor/browserprocess.py +++ b/ocrdmonitor/browserprocess.py @@ -1,22 +1,19 @@ from typing import Collection, Protocol +from ocrdbrowser import OcrdBrowser -class BrowserProcess(Protocol): - def process_id(self) -> str: - ... - - def workspace(self) -> str: - ... - - def owner(self) -> str: +class BrowserRestoringFactory(Protocol): + def __call__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: ... class BrowserProcessRepository(Protocol): - async def insert(self, browser: BrowserProcess) -> None: + async def insert(self, browser: OcrdBrowser) -> None: ... - async def delete(self, browser: BrowserProcess) -> None: + async def delete(self, browser: OcrdBrowser) -> None: ... async def find( @@ -24,6 +21,5 @@ async def find( *, owner: str | None = None, workspace: str | None = None, - process_id: str | None = None, - ) -> Collection[BrowserProcess]: + ) -> Collection[OcrdBrowser]: ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 9e9d40a..506c8a5 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -6,10 +6,12 @@ from beanie.odm.queries.find import FindMany from motor.motor_asyncio import AsyncIOMotorClient -from ocrdmonitor.browserprocess import BrowserProcess as BrowserProcessProtocol +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserRestoringFactory class BrowserProcess(Document): + address: str owner: str process_id: str workspace: str @@ -26,14 +28,18 @@ class Settings: class MongoBrowserProcessRepository: - async def insert(self, browser: BrowserProcessProtocol) -> None: + def __init__(self, restoring_factory: BrowserRestoringFactory) -> None: + self._restoring_factory = restoring_factory + + async def insert(self, browser: OcrdBrowser) -> None: await BrowserProcess( # type: ignore + address=browser.address(), owner=browser.owner(), process_id=browser.process_id(), workspace=browser.workspace(), ).insert() - async def delete(self, browser: BrowserProcessProtocol) -> None: + async def delete(self, browser: OcrdBrowser) -> None: await BrowserProcess( # type: ignore owner=browser.owner(), process_id=browser.process_id(), @@ -45,8 +51,7 @@ async def find( *, owner: str | None = None, workspace: str | None = None, - process_id: str | None = None, - ) -> Collection[BrowserProcess]: + ) -> Collection[OcrdBrowser]: results: FindMany[BrowserProcess] | None = None def find( @@ -64,11 +69,11 @@ def find( if workspace is not None: results = find(results, BrowserProcess.workspace == workspace) - if process_id is not None: - results = find(results, BrowserProcess.process_id == process_id) - return await results.to_list() if results is not None else [] + async def clean(self) -> None: + await BrowserProcess.delete_all() + async def init(connection_str: str) -> None: client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index c50eb01..d221c93 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -1,4 +1,5 @@ from __future__ import annotations +from typing import cast import uuid from pathlib import Path @@ -7,12 +8,10 @@ from fastapi.templating import Jinja2Templates import ocrdbrowser -from ocrdmonitor import dbmodel import ocrdmonitor.server.proxy as proxy from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace -from ocrdmonitor import dbmodel from ocrdmonitor.browserprocess import BrowserProcessRepository -from ocrdmonitor.server.redirect import RedirectMap +from ocrdmonitor.server.redirect import BrowserRedirect, RedirectMap def create_workspaces( @@ -43,8 +42,13 @@ async def browser(request: Request, workspace: Path) -> Response: response = Response() response.set_cookie("session_id", session_id) - if (session_id, workspace) not in redirects: + existing_browsers = await repository.find( + owner=session_id, workspace=str(workspace) + ) + + if not existing_browsers: browser = await launch_browser(session_id, workspace) + print("Inserting", f"{workspace=}", f"owner={session_id}") redirects.add(session_id, workspace, browser) await repository.insert(browser) @@ -59,9 +63,14 @@ def open_workspace(request: Request, workspace: str) -> Response: @router.get("/ping/{workspace:path}", name="workspaces.ping") async def ping_workspace( - request: Request, workspace: Path, session_id: str = Cookie(default=None) + workspace: Path, session_id: str = Cookie(default=None) ) -> Response: - redirect = redirects.get(session_id, workspace) + browsers = list( + await repository.find( + owner=session_id, workspace=str(workspace_dir / workspace) + ) + ) + redirect = BrowserRedirect(workspace, browsers[0]) try: await proxy.forward(redirect, str(workspace)) return Response(status_code=200) @@ -75,7 +84,12 @@ async def ping_workspace( async def workspace_reverse_proxy( request: Request, workspace: Path, session_id: str = Cookie(default=None) ) -> Response: - redirect = redirects.get(session_id, workspace) + browsers = list( + await repository.find( + owner=session_id, workspace=str(workspace_dir / workspace) + ) + ) + redirect = BrowserRedirect(workspace, browsers[0]) try: return await proxy.forward(redirect, str(workspace)) except ConnectionError: diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index fb7fd0d..0ab2285 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -1 +1 @@ -from .fixtures import app, launch_monitor, repository +from .fixtures import app, launch_monitor, auto_repository diff --git a/tests/ocrdmonitor/server/fixtures.py b/tests/ocrdmonitor/server/fixtures.py index 4e538f9..612771d 100644 --- a/tests/ocrdmonitor/server/fixtures.py +++ b/tests/ocrdmonitor/server/fixtures.py @@ -1,10 +1,12 @@ -from contextlib import asynccontextmanager, contextmanager +import asyncio +from contextlib import asynccontextmanager from pathlib import Path from typing import ( + Any, AsyncContextManager, AsyncIterator, + Awaitable, Callable, - ContextManager, Iterator, ) from unittest.mock import patch @@ -15,8 +17,9 @@ from fastapi.testclient import TestClient from testcontainers.mongodb import MongoDbContainer +from ocrdbrowser import OcrdBrowser from ocrdmonitor import dbmodel -from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory from ocrdmonitor.server.app import create_app from ocrdmonitor.server.settings import ( OcrdBrowserSettings, @@ -26,6 +29,7 @@ ) from tests.testdoubles import ( BackgroundProcess, + BrowserSpy, BrowserTestDoubleFactory, InMemoryBrowserProcessRepository, ) @@ -60,15 +64,37 @@ async def patch_factory( @asynccontextmanager -async def mongodb_repository() -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: +async def mongodb_repository( + restoring_factory: BrowserRestoringFactory, +) -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: with MongoDbContainer() as container: await dbmodel.init(container.get_connection_url()) - yield dbmodel.MongoBrowserProcessRepository() + yield dbmodel.MongoBrowserProcessRepository(restoring_factory) @asynccontextmanager -async def inmemory_repository() -> AsyncIterator[InMemoryBrowserProcessRepository]: - yield InMemoryBrowserProcessRepository() +async def inmemory_repository( + restoring_factory: BrowserRestoringFactory, +) -> AsyncIterator[InMemoryBrowserProcessRepository]: + yield InMemoryBrowserProcessRepository(restoring_factory) + + +def spy_restoring_factory() -> BrowserRestoringFactory: + def factory( + owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: + return BrowserSpy(owner, workspace, address, process_id) + + return factory + + +@asynccontextmanager +async def patch_repository(repository: BrowserProcessRepository) -> AsyncIterator[None]: + async def _repository(_: OcrdBrowserSettings) -> BrowserProcessRepository: + return repository + + with patch.object(OcrdBrowserSettings, "repository", _repository): + yield @pytest_asyncio.fixture( @@ -81,20 +107,16 @@ async def inmemory_repository() -> AsyncIterator[InMemoryBrowserProcessRepositor ), ], ) -async def repository( - monkeypatch: pytest.MonkeyPatch, +async def auto_repository( request: pytest.FixtureRequest, -) -> AsyncIterator[BrowserProcessRepository]: +) -> AsyncIterator[BrowserProcessRepository] | None: repository_constructor: Callable[ - [], AsyncContextManager[BrowserProcessRepository] + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], ] = request.param - async with repository_constructor() as repository: - - async def async_repository(self: OcrdBrowserSettings) -> BrowserProcessRepository: - return repository - - monkeypatch.setattr(OcrdBrowserSettings, "repository", async_repository) - yield repository + async with repository_constructor(spy_restoring_factory()) as repository: + async with patch_repository(repository): + yield repository @pytest_asyncio.fixture diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index d34b891..7f96a78 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -1,29 +1,22 @@ from __future__ import annotations -from contextlib import asynccontextmanager, contextmanager -from typing import ( - AsyncContextManager, - AsyncIterator, - Callable, - ContextManager, - Iterator, - cast, -) +from typing import AsyncIterator, cast import pytest import pytest_asyncio from fastapi.testclient import TestClient from httpx import Response -from testcontainers.mongodb import MongoDbContainer -from ocrdbrowser import ChannelClosed -from ocrdmonitor import dbmodel -from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdbrowser import ChannelClosed, OcrdBrowser +from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory +from ocrdmonitor.server.app import create_app from ocrdmonitor.server.settings import OcrdBrowserSettings from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.fixtures import ( WORKSPACE_DIR, + create_settings, patch_factory, + patch_repository, ) from tests.testdoubles import ( Browser_Heading, @@ -31,9 +24,9 @@ BrowserSpy, BrowserTestDouble, BrowserTestDoubleFactory, - InMemoryBrowserProcessRepository, IteratingBrowserTestDoubleFactory, SingletonBrowserTestDoubleFactory, + InMemoryBrowserProcessRepository, ) @@ -181,11 +174,22 @@ def test__browsed_workspace_is_ready__when_pinging__returns_ok( assert result.status_code == 200 +@pytest.mark.usefixtures("iterating_factory") def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( - singleton_browser_spy: BrowserSpy, + auto_repository: BrowserProcessRepository, app: TestClient, ) -> None: - singleton_browser_spy.configure_client(response=ConnectionError) + if isinstance(auto_repository, InMemoryBrowserProcessRepository): + + def restore( + owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: + spy = BrowserSpy(owner, workspace, address, process_id) + spy.configure_client(response=ConnectionError) + return spy + + auto_repository.restoring_factory = restore + workspace = "a_workspace" _ = view_workspace(app, workspace) @@ -197,10 +201,46 @@ def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( @pytest.mark.asyncio @pytest.mark.usefixtures("iterating_factory") async def test__browsing_workspace__stores_browser_in_repository( - repository: BrowserProcessRepository, app: TestClient + auto_repository: BrowserProcessRepository, app: TestClient ) -> None: _ = view_workspace(app, "a_workspace") - found_browsers = await repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + found_browsers = list( + await auto_repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) assert len(found_browsers) == 1 + + +@pytest.fixture +def singleton_restoring_factory() -> BrowserRestoringFactory: + spy = BrowserSpy() + + def factory( + owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: + spy.set_owner_and_workspace(owner, workspace) + return spy + + return factory + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("iterating_factory") +async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( + singleton_restoring_factory: BrowserRestoringFactory, +) -> None: + repository = InMemoryBrowserProcessRepository(singleton_restoring_factory) + async with patch_repository(repository): + app = TestClient(await create_app(create_settings())) + + browser = cast( + BrowserSpy, + singleton_restoring_factory("the-owner", "a_workspace", "", ""), + ) + browser.configure_client(response=b"RESTORED BROWSER") + await repository.insert(browser) + + response = view_workspace(app, "a_workspace") + + assert response.content == b"RESTORED BROWSER" diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index 9959918..4bd3b0e 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -1,15 +1,22 @@ from typing import Collection -from ocrdmonitor.browserprocess import BrowserProcess +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserRestoringFactory +from tests.testdoubles import BrowserSpy class InMemoryBrowserProcessRepository: - def __init__(self) -> None: - self._processes: list[BrowserProcess] = [] - - async def insert(self, browser: BrowserProcess) -> None: + def __init__( + self, restoring_factory: BrowserRestoringFactory | None = None + ) -> None: + self._processes: list[OcrdBrowser] = [] + self.restoring_factory: BrowserRestoringFactory = ( + restoring_factory or BrowserSpy + ) + + async def insert(self, browser: OcrdBrowser) -> None: self._processes.append(browser) - async def delete(self, browser: BrowserProcess) -> None: + async def delete(self, browser: OcrdBrowser) -> None: self._processes.remove(browser) async def find( @@ -17,9 +24,8 @@ async def find( *, owner: str | None = None, workspace: str | None = None, - process_id: str | None = None, - ) -> Collection[BrowserProcess]: - def match(browser: BrowserProcess) -> bool: + ) -> Collection[OcrdBrowser]: + def match(browser: OcrdBrowser) -> bool: matches = True if owner is not None: matches = matches and browser.owner() == owner @@ -27,9 +33,15 @@ def match(browser: BrowserProcess) -> bool: if workspace is not None: matches = matches and browser.workspace() == workspace - if process_id is not None: - matches = matches and browser.process_id() == process_id - return matches - return list(filter(match, self._processes)) + return [ + self.restoring_factory( + process_id=browser.process_id(), + owner=browser.owner(), + workspace=browser.workspace(), + address=browser.address(), + ) + for browser in self._processes + if match(browser) + ] diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 57e816d..9daab1f 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -47,17 +47,17 @@ async def open_channel(self) -> AsyncGenerator[Channel, None]: class BrowserSpy: def __init__( self, - process_id: str = "1234", owner: str = "", - workspace_path: str = "", + workspace: str = "", address: str = "http://unreachable.example.com", + process_id: str = "1234", running: bool = False, ) -> None: self._address = address self._process_id = process_id self.is_running = running self.owner_name = owner - self.workspace_path = workspace_path + self.workspace_path = workspace self._client = BrowserClientStub() def configure_client( From 91ca4dfb3f7e406dbf1a022b1885b89eb639a568 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 16 May 2023 23:25:51 +0200 Subject: [PATCH 09/32] Refactor fixtures. Begin refactor towards running browser --- ocrdbrowser/__init__.py | 4 +- ocrdbrowser/_browser.py | 24 ++++- ocrdbrowser/_docker.py | 38 +++++++- ocrdbrowser/_subprocess.py | 10 +- ocrdmonitor/dbmodel.py | 14 ++- ocrdmonitor/server/workspaces.py | 1 - pyproject.toml | 6 +- tests/ocrdmonitor/server/conftest.py | 9 +- tests/ocrdmonitor/server/fixtures/__init__.py | 0 tests/ocrdmonitor/server/fixtures/app.py | 51 ++++++++++ tests/ocrdmonitor/server/fixtures/factory.py | 15 +++ .../{fixtures.py => fixtures/repository.py} | 92 ++++-------------- tests/ocrdmonitor/server/test_job_endpoint.py | 3 +- .../server/test_workspace_endpoint.py | 96 ++++++++++++------- tests/testdoubles/_browserfake.py | 5 +- tests/testdoubles/_browserspy.py | 5 +- 16 files changed, 247 insertions(+), 126 deletions(-) create mode 100644 tests/ocrdmonitor/server/fixtures/__init__.py create mode 100644 tests/ocrdmonitor/server/fixtures/app.py create mode 100644 tests/ocrdmonitor/server/fixtures/factory.py rename tests/ocrdmonitor/server/{fixtures.py => fixtures/repository.py} (50%) diff --git a/ocrdbrowser/__init__.py b/ocrdbrowser/__init__.py index 5528a80..5f1deee 100644 --- a/ocrdbrowser/__init__.py +++ b/ocrdbrowser/__init__.py @@ -5,6 +5,7 @@ OcrdBrowser, OcrdBrowserClient, OcrdBrowserFactory, + RunningOcrdBrowser, filter_owned, in_other_workspaces, in_same_workspace, @@ -12,10 +13,10 @@ stop_all, stop_owned_in_workspace, ) +from ._client import HttpBrowserClient from ._docker import DockerOcrdBrowserFactory from ._port import NoPortsAvailableError from ._subprocess import SubProcessOcrdBrowserFactory -from ._client import HttpBrowserClient __all__ = [ "Channel", @@ -26,6 +27,7 @@ "OcrdBrowser", "OcrdBrowserClient", "OcrdBrowserFactory", + "RunningOcrdBrowser", "SubProcessOcrdBrowserFactory", "filter_owned", "launch", diff --git a/ocrdbrowser/_browser.py b/ocrdbrowser/_browser.py index 99af25c..ade74ac 100644 --- a/ocrdbrowser/_browser.py +++ b/ocrdbrowser/_browser.py @@ -5,6 +5,26 @@ from typing import AsyncContextManager, Protocol +class RunningOcrdBrowser(Protocol): + def process_id(self) -> str: + ... + + def address(self) -> str: + ... + + def owner(self) -> str: + ... + + def workspace(self) -> str: + ... + + def client(self) -> OcrdBrowserClient: + ... + + async def stop(self) -> None: + ... + + class OcrdBrowser(Protocol): def process_id(self) -> str: ... @@ -21,7 +41,7 @@ def workspace(self) -> str: def client(self) -> OcrdBrowserClient: ... - async def start(self) -> None: + async def start(self) -> RunningOcrdBrowser: ... async def stop(self) -> None: @@ -51,7 +71,7 @@ def open_channel(self) -> AsyncContextManager[Channel]: class OcrdBrowserFactory(Protocol): def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: ... - + BrowserProcesses = set[OcrdBrowser] diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index 78cc76b..68129c5 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -5,7 +5,7 @@ import os.path as path from typing import Any -from ._browser import OcrdBrowser, OcrdBrowserClient +from ._browser import OcrdBrowser, OcrdBrowserClient, RunningOcrdBrowser from ._port import Port from ._client import HttpBrowserClient @@ -42,11 +42,12 @@ def workspace(self) -> str: def owner(self) -> str: return self._owner - async def start(self) -> None: + async def start(self) -> RunningOcrdBrowser: cmd = await _run_command( _docker_run, self._container_name(), self._workspace, self._port.get() ) self.id = str(cmd.stdout).strip() + return self async def stop(self) -> None: cmd = await _run_command( @@ -69,6 +70,39 @@ def _container_name(self) -> str: return f"ocrd-browser-{self.owner()}-{workspace}" +class RunningDockerOcrdBrowser: + def __init__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> None: + self._owner = owner + self._workspace = workspace + self._address = address + self._process_id = process_id + + def process_id(self) -> str: + return self._process_id + + def address(self) -> str: + return self._address + + def workspace(self) -> str: + return self._workspace + + def owner(self) -> str: + return self._owner + + async def stop(self) -> None: + cmd = await _run_command(_docker_stop, self._process_id) + + if cmd.returncode != 0: + logging.info( + f"Stopping container {self._process_id} returned exit code {cmd.returncode}" + ) + + def client(self) -> OcrdBrowserClient: + return HttpBrowserClient(self._address) + + class DockerOcrdBrowserFactory: def __init__(self, host: str, available_ports: set[int]) -> None: self._host = host diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 8110325..4d16fd5 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -6,7 +6,7 @@ from shutil import which from typing import Optional -from ._browser import OcrdBrowser, OcrdBrowserClient +from ._browser import OcrdBrowser, OcrdBrowserClient, RunningOcrdBrowser from ._port import Port from ._client import HttpBrowserClient @@ -27,10 +27,6 @@ def process_id(self) -> str: return str(self._process.pid) def address(self) -> str: - # as long as we do not have a reverse proxy on BW_PORT, - # we must map the local port range to the exposed range - # (we use 8085 as fixed start of the internal port range, - # and map to the runtime corresponding external port) localport = self._localport.get() return "http://localhost:" + str(localport) @@ -40,7 +36,7 @@ def workspace(self) -> str: def owner(self) -> str: return self._owner - async def start(self) -> None: + async def start(self) -> RunningOcrdBrowser: browse_ocrd = which("browse-ocrd") if not browse_ocrd: raise FileNotFoundError("Could not find browse-ocrd executable") @@ -67,6 +63,8 @@ async def start(self) -> None: env=environment, ) + return self + async def stop(self) -> None: if self._process: try: diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 506c8a5..2cafce5 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -69,7 +69,19 @@ def find( if workspace is not None: results = find(results, BrowserProcess.workspace == workspace) - return await results.to_list() if results is not None else [] + return ( + [ + self._restoring_factory( + browser.owner, + browser.workspace, + browser.address, + browser.process_id, + ) + for browser in await results.to_list() + ] + if results + else [] + ) async def clean(self) -> None: await BrowserProcess.delete_all() diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index d221c93..d6feeca 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -1,5 +1,4 @@ from __future__ import annotations -from typing import cast import uuid from pathlib import Path diff --git a/pyproject.toml b/pyproject.toml index 24ef407..42f67f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,10 +37,14 @@ nox = ["nox>=2022.11.21"] [tool.mypy] plugins = ["pydantic.mypy"] +[tool.ruff] +line-length = 100 + [tool.pytest.ini_options] markers = [ "integration: mark test as integration test", - "needs_docker: marks tests that need access to Docker in order to run" + "needs_docker: marks tests that need access to Docker in order to run", + "no_auto_repository: marks test to not automatically use the auto_repository fixture" ] [tool.pdm.scripts] diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index 0ab2285..1e5e8db 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -1 +1,8 @@ -from .fixtures import app, launch_monitor, auto_repository +from .fixtures.app import app, create_settings # noqa: F401 +from .fixtures.factory import patch_factory # noqa: F401 +from .fixtures.repository import ( + auto_repository, # noqa: F401 + inmemory_repository, # noqa: F401 + mongodb_repository, # noqa: F401 + patch_repository, # noqa: F401 +) diff --git a/tests/ocrdmonitor/server/fixtures/__init__.py b/tests/ocrdmonitor/server/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/ocrdmonitor/server/fixtures/app.py b/tests/ocrdmonitor/server/fixtures/app.py new file mode 100644 index 0000000..13a2ee5 --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/app.py @@ -0,0 +1,51 @@ +from pathlib import Path +from typing import Iterator + +import pytest +import pytest_asyncio +import uvicorn +from fastapi.testclient import TestClient + +from ocrdmonitor.server.app import create_app +from ocrdmonitor.server.settings import ( + OcrdBrowserSettings, + OcrdControllerSettings, + OcrdLogViewSettings, + Settings, +) +from tests.testdoubles import BackgroundProcess + +JOB_DIR = Path(__file__).parent / "ocrd.jobs" +WORKSPACE_DIR = Path("tests") / "workspaces" + + +def create_settings() -> Settings: + return Settings( + ocrd_browser=OcrdBrowserSettings( + workspace_dir=WORKSPACE_DIR, + port_range=(9000, 9100), + db_connection_string="", + ), + ocrd_controller=OcrdControllerSettings( + job_dir=JOB_DIR, + host="", + user="", + ), + ocrd_logview=OcrdLogViewSettings(port=8022), + ) + + +@pytest_asyncio.fixture +async def app() -> TestClient: + return TestClient(await create_app(create_settings())) + + +def _launch_app() -> None: + app = create_app(create_settings()) + uvicorn.run(app, port=3000) + + +@pytest.fixture +def launch_monitor() -> Iterator[None]: + with BackgroundProcess(_launch_app): + yield diff --git a/tests/ocrdmonitor/server/fixtures/factory.py b/tests/ocrdmonitor/server/fixtures/factory.py new file mode 100644 index 0000000..68f8815 --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/factory.py @@ -0,0 +1,15 @@ +from contextlib import asynccontextmanager +from typing import AsyncIterator +from unittest.mock import patch + +from ocrdmonitor.server.settings import OcrdBrowserSettings +from tests.testdoubles import BrowserTestDoubleFactory + + +@asynccontextmanager +async def patch_factory( + factory: BrowserTestDoubleFactory, +) -> AsyncIterator[BrowserTestDoubleFactory]: + async with factory: + with patch.object(OcrdBrowserSettings, "factory", lambda _: factory): + yield factory diff --git a/tests/ocrdmonitor/server/fixtures.py b/tests/ocrdmonitor/server/fixtures/repository.py similarity index 50% rename from tests/ocrdmonitor/server/fixtures.py rename to tests/ocrdmonitor/server/fixtures/repository.py index 612771d..8638a06 100644 --- a/tests/ocrdmonitor/server/fixtures.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -1,67 +1,20 @@ -import asyncio from contextlib import asynccontextmanager -from pathlib import Path -from typing import ( - Any, - AsyncContextManager, - AsyncIterator, - Awaitable, - Callable, - Iterator, -) +from typing import AsyncContextManager, AsyncIterator, Callable from unittest.mock import patch import pytest import pytest_asyncio -import uvicorn -from fastapi.testclient import TestClient from testcontainers.mongodb import MongoDbContainer from ocrdbrowser import OcrdBrowser from ocrdmonitor import dbmodel from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory -from ocrdmonitor.server.app import create_app -from ocrdmonitor.server.settings import ( - OcrdBrowserSettings, - OcrdControllerSettings, - OcrdLogViewSettings, - Settings, -) +from ocrdmonitor.server.settings import OcrdBrowserSettings from tests.testdoubles import ( - BackgroundProcess, BrowserSpy, - BrowserTestDoubleFactory, InMemoryBrowserProcessRepository, ) -JOB_DIR = Path(__file__).parent / "ocrd.jobs" -WORKSPACE_DIR = Path("tests") / "workspaces" - - -def create_settings() -> Settings: - return Settings( - ocrd_browser=OcrdBrowserSettings( - workspace_dir=WORKSPACE_DIR, - port_range=(9000, 9100), - db_connection_string="", - ), - ocrd_controller=OcrdControllerSettings( - job_dir=JOB_DIR, - host="", - user="", - ), - ocrd_logview=OcrdLogViewSettings(port=8022), - ) - - -@asynccontextmanager -async def patch_factory( - factory: BrowserTestDoubleFactory, -) -> AsyncIterator[BrowserTestDoubleFactory]: - async with factory: - with patch.object(OcrdBrowserSettings, "factory", lambda _: factory): - yield factory - @asynccontextmanager async def mongodb_repository( @@ -110,26 +63,21 @@ async def _repository(_: OcrdBrowserSettings) -> BrowserProcessRepository: async def auto_repository( request: pytest.FixtureRequest, ) -> AsyncIterator[BrowserProcessRepository] | None: - repository_constructor: Callable[ - [BrowserRestoringFactory], - AsyncContextManager[BrowserProcessRepository], - ] = request.param - async with repository_constructor(spy_restoring_factory()) as repository: - async with patch_repository(repository): - yield repository - - -@pytest_asyncio.fixture -async def app() -> TestClient: - return TestClient(await create_app(create_settings())) - - -def _launch_app() -> None: - app = create_app(create_settings()) - uvicorn.run(app, port=3000) - - -@pytest.fixture -def launch_monitor() -> Iterator[None]: - with BackgroundProcess(_launch_app): - yield + """ + This fixture will be used automatically for all tests, + as a repository for browser processes is pretty much always needed. + + It can be turned off by marking a test with 'pytest.mark.no_auto_repository' + """ + if "no_auto_repository" in request.keywords: + # NOTE: we're yielding 0 here, because pytest_asyncio + # raises a StopIterationError if we return or yield None + yield 0 + else: + repository_constructor: Callable[ + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], + ] = request.param + async with repository_constructor(spy_restoring_factory()) as repository: + async with patch_repository(repository): + yield repository diff --git a/tests/ocrdmonitor/server/test_job_endpoint.py b/tests/ocrdmonitor/server/test_job_endpoint.py index 58fa120..6a34cfe 100644 --- a/tests/ocrdmonitor/server/test_job_endpoint.py +++ b/tests/ocrdmonitor/server/test_job_endpoint.py @@ -8,12 +8,13 @@ import pytest from fastapi.testclient import TestClient from httpx import Response + from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.ocrdjob import OcrdJob from ocrdmonitor.processstatus import ProcessState, ProcessStatus from ocrdmonitor.server.settings import OcrdControllerSettings from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures import JOB_DIR +from tests.ocrdmonitor.server.fixtures.app import JOB_DIR from tests.ocrdmonitor.test_jobs import JOB_TEMPLATE, jobfile_content_for diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 7f96a78..948fe1f 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -1,6 +1,6 @@ from __future__ import annotations -from typing import AsyncIterator, cast +from typing import AsyncContextManager, AsyncIterator, Callable, cast import pytest import pytest_asyncio @@ -10,12 +10,12 @@ from ocrdbrowser import ChannelClosed, OcrdBrowser from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory from ocrdmonitor.server.app import create_app -from ocrdmonitor.server.settings import OcrdBrowserSettings from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures import ( - WORKSPACE_DIR, - create_settings, - patch_factory, +from tests.ocrdmonitor.server.fixtures.app import WORKSPACE_DIR, create_settings +from tests.ocrdmonitor.server.fixtures.factory import patch_factory +from tests.ocrdmonitor.server.fixtures.repository import ( + inmemory_repository, + mongodb_repository, patch_repository, ) from tests.testdoubles import ( @@ -26,7 +26,6 @@ BrowserTestDoubleFactory, IteratingBrowserTestDoubleFactory, SingletonBrowserTestDoubleFactory, - InMemoryBrowserProcessRepository, ) @@ -174,26 +173,40 @@ def test__browsed_workspace_is_ready__when_pinging__returns_ok( assert result.status_code == 200 +@pytest.mark.asyncio @pytest.mark.usefixtures("iterating_factory") -def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( - auto_repository: BrowserProcessRepository, - app: TestClient, +@pytest.mark.parametrize( + "repository", + ( + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, pytest.mark.needs_docker), + ), + ), +) +async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( + singleton_restoring_factory: BrowserRestoringFactory, + repository: Callable[ + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], + ], ) -> None: - if isinstance(auto_repository, InMemoryBrowserProcessRepository): + async with repository(singleton_restoring_factory) as repo: + async with patch_repository(repo): + app = TestClient(await create_app(create_settings())) - def restore( - owner: str, workspace: str, address: str, process_id: str - ) -> OcrdBrowser: - spy = BrowserSpy(owner, workspace, address, process_id) - spy.configure_client(response=ConnectionError) - return spy + browser = cast( + BrowserSpy, + singleton_restoring_factory("the-owner", "a_workspace", "", ""), + ) + browser.configure_client(response=ConnectionError) + await repo.insert(browser) - auto_repository.restoring_factory = restore + workspace = "a_workspace" + _ = view_workspace(app, workspace) - workspace = "a_workspace" - _ = view_workspace(app, workspace) - - result = app.get(f"/workspaces/ping/{workspace}") + result = app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 502 @@ -227,20 +240,35 @@ def factory( @pytest.mark.asyncio @pytest.mark.usefixtures("iterating_factory") +@pytest.mark.parametrize( + "repository", + ( + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, pytest.mark.needs_docker), + ), + ), +) +@pytest.mark.no_auto_repository async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( singleton_restoring_factory: BrowserRestoringFactory, + repository: Callable[ + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], + ], ) -> None: - repository = InMemoryBrowserProcessRepository(singleton_restoring_factory) - async with patch_repository(repository): - app = TestClient(await create_app(create_settings())) - - browser = cast( - BrowserSpy, - singleton_restoring_factory("the-owner", "a_workspace", "", ""), - ) - browser.configure_client(response=b"RESTORED BROWSER") - await repository.insert(browser) - - response = view_workspace(app, "a_workspace") + async with repository(singleton_restoring_factory) as repo: + async with patch_repository(repo): + app = TestClient(await create_app(create_settings())) + + browser = cast( + BrowserSpy, + singleton_restoring_factory("the-owner", "a_workspace", "", ""), + ) + browser.configure_client(response=b"RESTORED BROWSER") + await repo.insert(browser) + + response = view_workspace(app, "a_workspace") assert response.content == b"RESTORED BROWSER" diff --git a/tests/testdoubles/_browserfake.py b/tests/testdoubles/_browserfake.py index b3c89c5..f60d37e 100644 --- a/tests/testdoubles/_browserfake.py +++ b/tests/testdoubles/_browserfake.py @@ -3,7 +3,7 @@ import asyncio -from ocrdbrowser import HttpBrowserClient, OcrdBrowserClient +from ocrdbrowser import HttpBrowserClient, OcrdBrowserClient, RunningOcrdBrowser from ._backgroundprocess import BackgroundProcess from ._broadwayfake import broadway_fake, FAKE_HOST_ADDRESS @@ -36,9 +36,10 @@ def workspace(self) -> str: def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) - async def start(self) -> None: + async def start(self) -> RunningOcrdBrowser: self._running = True await asyncio.to_thread(self._browser.launch) + return self async def stop(self) -> None: self._running = False diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 9daab1f..1dd09af 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -4,7 +4,7 @@ from textwrap import dedent from typing import AsyncGenerator, Type -from ocrdbrowser import Channel, OcrdBrowserClient +from ocrdbrowser import Channel, OcrdBrowserClient, RunningOcrdBrowser Browser_Heading = "OCRD BROWSER" @@ -84,8 +84,9 @@ def owner(self) -> str: def client(self) -> OcrdBrowserClient: return self._client - async def start(self) -> None: + async def start(self) -> RunningOcrdBrowser: self.is_running = True + return self async def stop(self) -> None: self.is_running = False From 6a321ebb06cf7e05a9f8476276976eb309b099ca Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Fri, 19 May 2023 15:17:36 +0200 Subject: [PATCH 10/32] Factory responsible for starting browsers. --- ocrdbrowser/__init__.py | 14 ----- ocrdbrowser/_browser.py | 87 +--------------------------- ocrdbrowser/_docker.py | 83 +++++++++----------------- ocrdbrowser/_subprocess.py | 79 ++++++++++++------------- ocrdmonitor/server/workspaces.py | 4 +- tests/ocrdbrowser/test_launch.py | 51 ---------------- tests/testdoubles/_browserfactory.py | 9 ++- tests/testdoubles/_browserfake.py | 5 +- tests/testdoubles/_browserspy.py | 5 +- 9 files changed, 78 insertions(+), 259 deletions(-) delete mode 100644 tests/ocrdbrowser/test_launch.py diff --git a/ocrdbrowser/__init__.py b/ocrdbrowser/__init__.py index 5f1deee..6c10fc6 100644 --- a/ocrdbrowser/__init__.py +++ b/ocrdbrowser/__init__.py @@ -5,13 +5,6 @@ OcrdBrowser, OcrdBrowserClient, OcrdBrowserFactory, - RunningOcrdBrowser, - filter_owned, - in_other_workspaces, - in_same_workspace, - launch, - stop_all, - stop_owned_in_workspace, ) from ._client import HttpBrowserClient from ._docker import DockerOcrdBrowserFactory @@ -27,13 +20,6 @@ "OcrdBrowser", "OcrdBrowserClient", "OcrdBrowserFactory", - "RunningOcrdBrowser", "SubProcessOcrdBrowserFactory", - "filter_owned", - "launch", - "in_other_workspaces", - "in_same_workspace", - "stop_all", - "stop_owned_in_workspace", "workspace", ] diff --git a/ocrdbrowser/_browser.py b/ocrdbrowser/_browser.py index ade74ac..f6f9baf 100644 --- a/ocrdbrowser/_browser.py +++ b/ocrdbrowser/_browser.py @@ -5,26 +5,6 @@ from typing import AsyncContextManager, Protocol -class RunningOcrdBrowser(Protocol): - def process_id(self) -> str: - ... - - def address(self) -> str: - ... - - def owner(self) -> str: - ... - - def workspace(self) -> str: - ... - - def client(self) -> OcrdBrowserClient: - ... - - async def stop(self) -> None: - ... - - class OcrdBrowser(Protocol): def process_id(self) -> str: ... @@ -41,9 +21,6 @@ def workspace(self) -> str: def client(self) -> OcrdBrowserClient: ... - async def start(self) -> RunningOcrdBrowser: - ... - async def stop(self) -> None: ... @@ -69,67 +46,5 @@ def open_channel(self) -> AsyncContextManager[Channel]: class OcrdBrowserFactory(Protocol): - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: ... - - -BrowserProcesses = set[OcrdBrowser] - - -async def launch( - workspace_path: str, - owner: str, - browser_factory: OcrdBrowserFactory, - running_browsers: BrowserProcesses | None = None, -) -> OcrdBrowser: - running_browsers = running_browsers or set() - owned_processes = filter_owned(owner, running_browsers) - in_workspace = in_same_workspace(workspace_path, owned_processes) - - if in_workspace: - return in_workspace.pop() - - return await start_process(browser_factory, workspace_path, owner) - - -def in_same_workspace( - workspace_path: str, browser_processes: BrowserProcesses -) -> BrowserProcesses: - workspace_path = path.abspath(workspace_path) - return { - p for p in browser_processes if path.abspath(p.workspace()) == workspace_path - } - - -def in_other_workspaces( - workspace_path: str, browser_processes: BrowserProcesses -) -> BrowserProcesses: - workspace_path = path.abspath(workspace_path) - return {p for p in browser_processes if p.workspace() != workspace_path} - - -def filter_owned(owner: str, running_processes: BrowserProcesses) -> BrowserProcesses: - return {p for p in running_processes if p.owner() == owner} - - -async def stop_all(owned_processes: BrowserProcesses) -> None: - async with asyncio.TaskGroup() as group: - for p in owned_processes: - group.create_task(p.stop()) - - -async def stop_owned_in_workspace( - owner: str, workspace: str, browsers: set[OcrdBrowser] -) -> set[OcrdBrowser]: - owned = filter_owned(owner, browsers) - in_workspace = in_same_workspace(workspace, owned) - await stop_all(in_workspace) - return in_workspace - - -async def start_process( - process_factory: OcrdBrowserFactory, workspace_path: str, owner: str -) -> OcrdBrowser: - process = process_factory(owner, workspace_path) - await process.start() - return process diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index 68129c5..0896fc9 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -5,7 +5,7 @@ import os.path as path from typing import Any -from ._browser import OcrdBrowser, OcrdBrowserClient, RunningOcrdBrowser +from ._browser import OcrdBrowser, OcrdBrowserClient from ._port import Port from ._client import HttpBrowserClient @@ -23,61 +23,13 @@ async def _run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process: class DockerOcrdBrowser: - def __init__(self, host: str, port: Port, owner: str, workspace: str) -> None: - self._host = host - self._port = port - self._owner = owner - self._workspace = path.abspath(workspace) - self.id: str | None = None - - def process_id(self) -> str: - return self._container_name() - - def address(self) -> str: - return f"{self._host}:{self._port}" - - def workspace(self) -> str: - return self._workspace - - def owner(self) -> str: - return self._owner - - async def start(self) -> RunningOcrdBrowser: - cmd = await _run_command( - _docker_run, self._container_name(), self._workspace, self._port.get() - ) - self.id = str(cmd.stdout).strip() - return self - - async def stop(self) -> None: - cmd = await _run_command( - _docker_stop, self._container_name(), self.workspace(), self._port.get() - ) - - if cmd.returncode != 0: - logging.info( - f"Stopping container {self.id} returned exit code {cmd.returncode}" - ) - - self._port.release() - self.id = None - - def client(self) -> OcrdBrowserClient: - return HttpBrowserClient(self.address()) - - def _container_name(self) -> str: - workspace = path.basename(self.workspace()) - return f"ocrd-browser-{self.owner()}-{workspace}" - - -class RunningDockerOcrdBrowser: def __init__( self, owner: str, workspace: str, address: str, process_id: str ) -> None: self._owner = owner self._workspace = workspace self._address = address - self._process_id = process_id + self._process_id: str = process_id def process_id(self) -> str: return self._process_id @@ -96,11 +48,16 @@ async def stop(self) -> None: if cmd.returncode != 0: logging.info( - f"Stopping container {self._process_id} returned exit code {cmd.returncode}" + f"Stopping container {self.process_id} returned exit code {cmd.returncode}" ) def client(self) -> OcrdBrowserClient: - return HttpBrowserClient(self._address) + return HttpBrowserClient(self.address()) + + +def _container_name(owner: str, workspace: str) -> str: + workspace = path.basename(workspace) + return f"ocrd-browser-{owner}-{workspace}" class DockerOcrdBrowserFactory: @@ -109,15 +66,31 @@ def __init__(self, host: str, available_ports: set[int]) -> None: self._ports = available_ports self._containers: list[DockerOcrdBrowser] = [] - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + abs_workspace = path.abspath(workspace_path) + port = Port(self._ports) + + cmd = await _run_command( + _docker_run, + _container_name(owner, abs_workspace), + abs_workspace, + port.get(), + ) + + container_id = str(cmd.stdout).strip() + container = DockerOcrdBrowser( - self._host, Port(self._ports), owner, workspace_path + owner, + abs_workspace, + f"{self._host}:{port.get()}", + container_id, ) + self._containers.append(container) return container async def stop_all(self) -> None: - running_ids = [c.id for c in self._containers if c.id] + running_ids = [c.process_id() for c in self._containers] if running_ids: await _run_command(_docker_kill, " ".join(running_ids)) diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 4d16fd5..264b3de 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -1,34 +1,31 @@ from __future__ import annotations import asyncio -import logging import os +import signal from shutil import which -from typing import Optional -from ._browser import OcrdBrowser, OcrdBrowserClient, RunningOcrdBrowser -from ._port import Port +from ._browser import OcrdBrowser, OcrdBrowserClient from ._client import HttpBrowserClient +from ._port import Port BROADWAY_BASE_PORT = 8080 class SubProcessOcrdBrowser: - def __init__(self, localport: Port, owner: str, workspace: str) -> None: - self._localport = localport + def __init__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> None: self._owner = owner self._workspace = workspace - self._process: Optional[asyncio.subprocess.Process] = None + self._address = address + self._process_id = process_id def process_id(self) -> str: - if not self._process: - raise AttributeError - - return str(self._process.pid) + return self._process_id def address(self) -> str: - localport = self._localport.get() - return "http://localhost:" + str(localport) + return self._address def workspace(self) -> str: return self._workspace @@ -36,53 +33,51 @@ def workspace(self) -> str: def owner(self) -> str: return self._owner - async def start(self) -> RunningOcrdBrowser: + async def stop(self) -> None: + os.kill(int(self._process_id), signal.SIGKILL) + + def client(self) -> OcrdBrowserClient: + return HttpBrowserClient(self.address()) + + +class SubProcessOcrdBrowserFactory: + def __init__(self, available_ports: set[int]) -> None: + self._available_ports = available_ports + + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + port = Port(self._available_ports).get() + address = f"https://localhost:{port}" + process = await self.start_browser(workspace_path, port) + browser = SubProcessOcrdBrowser( + owner, workspace_path, address, str(process.pid) + ) + return browser + + async def start_browser( + self, workspace: str, port: int + ) -> asyncio.subprocess.Process: browse_ocrd = which("browse-ocrd") if not browse_ocrd: raise FileNotFoundError("Could not find browse-ocrd executable") - localport = self._localport.get() + # broadwayd (which uses WebSockets) only allows a single client at a time # (disconnecting concurrent connections), hence we must start a new daemon # for each new browser session # broadwayd starts counting virtual X displays from port 8080 as :0 - displayport = str(localport - BROADWAY_BASE_PORT) + displayport = str(port - BROADWAY_BASE_PORT) environment = dict(os.environ) environment["GDK_BACKEND"] = "broadway" environment["BROADWAY_DISPLAY"] = ":" + displayport - self._process = await asyncio.create_subprocess_shell( + return await asyncio.create_subprocess_shell( " ".join( [ "broadwayd", ":" + displayport + " &", browse_ocrd, - self._workspace + "/mets.xml ;", + workspace + "/mets.xml ;", "kill $!", ] ), env=environment, ) - - return self - - async def stop(self) -> None: - if self._process: - try: - self._process.terminate() - except ProcessLookupError: - logging.info( - f"Attempted to stop already terminated process {self._process.pid}" - ) - finally: - self._localport.release() - - def client(self) -> OcrdBrowserClient: - return HttpBrowserClient(self.address()) - - -class SubProcessOcrdBrowserFactory: - def __init__(self, available_ports: set[int]) -> None: - self._available_ports = available_ports - - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - return SubProcessOcrdBrowser(Port(self._available_ports), owner, workspace_path) diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index d6feeca..5a34866 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -6,7 +6,6 @@ from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect from fastapi.templating import Jinja2Templates -import ocrdbrowser import ocrdmonitor.server.proxy as proxy from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace from ocrdmonitor.browserprocess import BrowserProcessRepository @@ -47,7 +46,6 @@ async def browser(request: Request, workspace: Path) -> Response: if not existing_browsers: browser = await launch_browser(session_id, workspace) - print("Inserting", f"{workspace=}", f"owner={session_id}") redirects.add(session_id, workspace, browser) await repository.insert(browser) @@ -119,7 +117,7 @@ async def communicate_with_browser_until_closed( async def launch_browser(session_id: str, workspace: Path) -> OcrdBrowser: full_workspace_path = workspace_dir / workspace - return await ocrdbrowser.launch(str(full_workspace_path), session_id, factory) + return await factory(session_id, str(full_workspace_path)) async def stop_browser(browser: OcrdBrowser) -> None: await browser.stop() diff --git a/tests/ocrdbrowser/test_launch.py b/tests/ocrdbrowser/test_launch.py deleted file mode 100644 index 8739168..0000000 --- a/tests/ocrdbrowser/test_launch.py +++ /dev/null @@ -1,51 +0,0 @@ -from typing import cast - -import pytest - -import ocrdbrowser -from tests.testdoubles import BrowserSpy, IteratingBrowserTestDoubleFactory - - -@pytest.mark.asyncio -async def test__workspace__launch__spawns_new_ocrd_browser() -> None: - owner = "the-owner" - workspace = "path/to/workspace" - process = await ocrdbrowser.launch( - workspace, owner, IteratingBrowserTestDoubleFactory() - ) - - process = cast(BrowserSpy, process) - assert process.is_running is True - assert process.owner() == owner - assert process.workspace() == workspace - - -@pytest.mark.asyncio -async def test__workspace__launch_for_different_owners__both_processes_running() -> None: - factory = IteratingBrowserTestDoubleFactory() - - first_process = await ocrdbrowser.launch("first-path", "first-owner", factory) - second_process = await ocrdbrowser.launch( - "second-path", "second-owner", factory, {first_process} - ) - - processes = {first_process, second_process} - assert all(cast(BrowserSpy, process).is_running for process in processes) - assert {p.owner() for p in processes} == {"first-owner", "second-owner"} - assert {p.workspace() for p in processes} == {"first-path", "second-path"} - - -@pytest.mark.asyncio -async def test__workspace__launch_for_same_owner_and_workspace__does_not_start_new_process() -> ( - None -): - owner = "the-owner" - workspace = "the-workspace" - factory = IteratingBrowserTestDoubleFactory() - - first_process = await ocrdbrowser.launch(workspace, owner, factory) - second_process = await ocrdbrowser.launch( - workspace, owner, factory, {first_process} - ) - - assert first_process is second_process diff --git a/tests/testdoubles/_browserfactory.py b/tests/testdoubles/_browserfactory.py index 6a3611d..5542895 100644 --- a/tests/testdoubles/_browserfactory.py +++ b/tests/testdoubles/_browserfactory.py @@ -10,6 +10,9 @@ class BrowserTestDouble(OcrdBrowser, Protocol): def set_owner_and_workspace(self, owner: str, workspace: str) -> None: ... + async def start(self) -> None: + ... + @property def is_running(self) -> bool: ... @@ -19,8 +22,9 @@ class SingletonBrowserTestDoubleFactory: def __init__(self, browser: BrowserTestDouble | None = None) -> None: self._browser = browser or BrowserSpy() - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: self._browser.set_owner_and_workspace(owner, workspace_path) + await self._browser.start() return self._browser async def __aenter__(self) -> Self: @@ -49,9 +53,10 @@ def __init__( def add(self, process: BrowserTestDouble) -> None: self._processes.append(process) - def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: browser = next(self._proc_iter, self._default_browser()) browser.set_owner_and_workspace(owner, workspace_path) + await browser.start() self._created.append(browser) return browser diff --git a/tests/testdoubles/_browserfake.py b/tests/testdoubles/_browserfake.py index f60d37e..b3c89c5 100644 --- a/tests/testdoubles/_browserfake.py +++ b/tests/testdoubles/_browserfake.py @@ -3,7 +3,7 @@ import asyncio -from ocrdbrowser import HttpBrowserClient, OcrdBrowserClient, RunningOcrdBrowser +from ocrdbrowser import HttpBrowserClient, OcrdBrowserClient from ._backgroundprocess import BackgroundProcess from ._broadwayfake import broadway_fake, FAKE_HOST_ADDRESS @@ -36,10 +36,9 @@ def workspace(self) -> str: def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) - async def start(self) -> RunningOcrdBrowser: + async def start(self) -> None: self._running = True await asyncio.to_thread(self._browser.launch) - return self async def stop(self) -> None: self._running = False diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 1dd09af..9daab1f 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -4,7 +4,7 @@ from textwrap import dedent from typing import AsyncGenerator, Type -from ocrdbrowser import Channel, OcrdBrowserClient, RunningOcrdBrowser +from ocrdbrowser import Channel, OcrdBrowserClient Browser_Heading = "OCRD BROWSER" @@ -84,9 +84,8 @@ def owner(self) -> str: def client(self) -> OcrdBrowserClient: return self._client - async def start(self) -> RunningOcrdBrowser: + async def start(self) -> None: self.is_running = True - return self async def stop(self) -> None: self.is_running = False From 9e57fee2684385858b02ccb8c8ad1d3245efa219 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 23 May 2023 17:57:07 +0200 Subject: [PATCH 11/32] Remove RedirectMap from workspace endpoint. Restoring factory controls Browser behavior in tests --- ocrdmonitor/dbmodel.py | 11 +- ocrdmonitor/server/workspaces.py | 25 ++- tests/ocrdmonitor/server/conftest.py | 1 + tests/ocrdmonitor/server/decorators.py | 35 ++++ .../ocrdmonitor/server/fixtures/repository.py | 22 ++- .../server/test_workspace_endpoint.py | 167 +++++++++--------- tests/testdoubles/_browserfactory.py | 22 ++- .../testdoubles/_browserprocessrepository.py | 45 +++-- 8 files changed, 217 insertions(+), 111 deletions(-) create mode 100644 tests/ocrdmonitor/server/decorators.py diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 2cafce5..075dd2c 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -9,6 +9,8 @@ from ocrdbrowser import OcrdBrowser from ocrdmonitor.browserprocess import BrowserRestoringFactory +from pymongo.results import DeleteResult + class BrowserProcess(Document): address: str @@ -40,10 +42,11 @@ async def insert(self, browser: OcrdBrowser) -> None: ).insert() async def delete(self, browser: OcrdBrowser) -> None: - await BrowserProcess( # type: ignore - owner=browser.owner(), - process_id=browser.process_id(), - workspace=browser.workspace(), + await BrowserProcess.find_one( + BrowserProcess.owner == browser.owner(), + BrowserProcess.workspace == browser.workspace(), + BrowserProcess.address == browser.address(), + BrowserProcess.process_id == browser.process_id(), ).delete() async def find( diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index 5a34866..f61a4ff 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -20,8 +20,6 @@ def create_workspaces( ) -> APIRouter: router = APIRouter(prefix="/workspaces") - redirects = RedirectMap() - @router.get("/", name="workspaces.list") def list_workspaces(request: Request) -> Response: spaces = [ @@ -40,13 +38,13 @@ async def browser(request: Request, workspace: Path) -> Response: response = Response() response.set_cookie("session_id", session_id) + full_workspace = str(workspace_dir / workspace) existing_browsers = await repository.find( - owner=session_id, workspace=str(workspace) + owner=session_id, workspace=full_workspace ) if not existing_browsers: browser = await launch_browser(session_id, workspace) - redirects.add(session_id, workspace, browser) await repository.insert(browser) return response @@ -67,11 +65,11 @@ async def ping_workspace( owner=session_id, workspace=str(workspace_dir / workspace) ) ) - redirect = BrowserRedirect(workspace, browsers[0]) try: + redirect = BrowserRedirect(workspace, browsers[0]) await proxy.forward(redirect, str(workspace)) return Response(status_code=200) - except ConnectionError: + except (ConnectionError, IndexError): return Response(status_code=502) # NOTE: It is important that the route path here ends with a slash, otherwise @@ -90,6 +88,7 @@ async def workspace_reverse_proxy( try: return await proxy.forward(redirect, str(workspace)) except ConnectionError: + await stop_browser(redirect.browser) return templates.TemplateResponse( "view_workspace_failed.html.j2", {"request": request, "workspace": workspace}, @@ -99,7 +98,16 @@ async def workspace_reverse_proxy( async def workspace_socket_proxy( websocket: WebSocket, workspace: Path, session_id: str = Cookie(default=None) ) -> None: - redirect = redirects.get(session_id, workspace) + browsers = list( + await repository.find( + owner=session_id, workspace=str(workspace_dir / workspace) + ) + ) + + if not browsers: + await websocket.close(reason="No browser found") + + redirect = BrowserRedirect(workspace, browsers[0]) await websocket.accept(subprotocol="broadway") await communicate_with_browser_until_closed(websocket, redirect.browser) @@ -121,7 +129,6 @@ async def launch_browser(session_id: str, workspace: Path) -> OcrdBrowser: async def stop_browser(browser: OcrdBrowser) -> None: await browser.stop() - key = Path(browser.workspace()).relative_to(workspace_dir) - redirects.remove(browser.owner(), key) + await repository.delete(browser) return router diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index 1e5e8db..209d5f1 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -5,4 +5,5 @@ inmemory_repository, # noqa: F401 mongodb_repository, # noqa: F401 patch_repository, # noqa: F401 + singleton_restoring_factory, # noqa: F401 ) diff --git a/tests/ocrdmonitor/server/decorators.py b/tests/ocrdmonitor/server/decorators.py new file mode 100644 index 0000000..a0d54e3 --- /dev/null +++ b/tests/ocrdmonitor/server/decorators.py @@ -0,0 +1,35 @@ +from typing import Any, Awaitable, Callable, Coroutine, ParamSpec +import pytest + +from tests.ocrdmonitor.server.fixtures.repository import ( + inmemory_repository, + mongodb_repository, +) + +Decorator = Callable[[Callable[..., Any]], Callable[..., Any]] + + +def compose(*decorators: Decorator) -> Decorator: + def decorated(fn: Callable[..., Any]) -> Callable[..., Any]: + for deco in reversed(decorators): + fn = deco(fn) + + return fn + + return decorated + + +use_custom_repository = compose( + pytest.mark.asyncio, + pytest.mark.no_auto_repository, + pytest.mark.parametrize( + "repository", + ( + pytest.param(inmemory_repository), + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, pytest.mark.needs_docker), + ), + ), + ), +) diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index 8638a06..063a2d4 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -14,6 +14,13 @@ BrowserSpy, InMemoryBrowserProcessRepository, ) +from tests.testdoubles._browserfactory import SingletonRestoringBrowserFactory + + +RepositoryInitializer = Callable[ + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], +] @asynccontextmanager @@ -36,7 +43,7 @@ def spy_restoring_factory() -> BrowserRestoringFactory: def factory( owner: str, workspace: str, address: str, process_id: str ) -> OcrdBrowser: - return BrowserSpy(owner, workspace, address, process_id) + return BrowserSpy(owner, workspace, address, process_id, running=True) return factory @@ -74,10 +81,15 @@ async def auto_repository( # raises a StopIterationError if we return or yield None yield 0 else: - repository_constructor: Callable[ - [BrowserRestoringFactory], - AsyncContextManager[BrowserProcessRepository], - ] = request.param + repository_constructor: RepositoryInitializer = request.param async with repository_constructor(spy_restoring_factory()) as repository: async with patch_repository(repository): yield repository + + +@pytest_asyncio.fixture +async def singleton_restoring_factory() -> AsyncIterator[ + SingletonRestoringBrowserFactory +]: + async with SingletonRestoringBrowserFactory() as factory: + yield factory diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 948fe1f..39f2f6d 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -1,21 +1,21 @@ from __future__ import annotations -from typing import AsyncContextManager, AsyncIterator, Callable, cast +from typing import AsyncIterator, cast import pytest import pytest_asyncio from fastapi.testclient import TestClient from httpx import Response -from ocrdbrowser import ChannelClosed, OcrdBrowser -from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory +from ocrdbrowser import ChannelClosed +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.server.app import create_app from tests.ocrdmonitor.server import scraping +from tests.ocrdmonitor.server.decorators import use_custom_repository from tests.ocrdmonitor.server.fixtures.app import WORKSPACE_DIR, create_settings from tests.ocrdmonitor.server.fixtures.factory import patch_factory from tests.ocrdmonitor.server.fixtures.repository import ( - inmemory_repository, - mongodb_repository, + RepositoryInitializer, patch_repository, ) from tests.testdoubles import ( @@ -27,6 +27,7 @@ IteratingBrowserTestDoubleFactory, SingletonBrowserTestDoubleFactory, ) +from tests.testdoubles._browserfactory import SingletonRestoringBrowserFactory class DisconnectingChannel: @@ -85,15 +86,23 @@ def assert_is_browser_response(actual: Response) -> None: assert scraping.parse_texts(actual.content, "h1") == [Browser_Heading] -def view_workspace(app: TestClient, workspace: str) -> Response: - _ = app.get(f"/workspaces/browse/{workspace}") - response = app.get(f"/workspaces/view/{workspace}") +def interact_with_workspace(app: TestClient, workspace: str) -> Response: + open_workspace(app, workspace) + response = view_workspace(app, workspace) with app.websocket_connect(f"/workspaces/view/{workspace}/socket"): pass return response +def open_workspace(app: TestClient, workspace: str) -> None: + _ = app.get(f"/workspaces/browse/{workspace}") + + +def view_workspace(app: TestClient, workspace: str) -> Response: + return app.get(f"/workspaces/view/{workspace}") + + def test__workspaces__shows_the_workspace_names_starting_from_workspace_root( app: TestClient, ) -> None: @@ -126,27 +135,23 @@ def test__browse_workspace__assigns_and_tracks_session_id(app: TestClient) -> No assert first_session_id == second_session_id -def test__opened_workspace__when_socket_disconnects_on_broadway_side__shuts_down_browser( - disconnecting_browser: BrowserSpy, - app: TestClient, +@pytest.mark.usefixtures("iterating_factory") +@use_custom_repository +async def test__opened_workspace__when_socket_disconnects_on_broadway_side__shuts_down_browser( + repository: RepositoryInitializer, ) -> None: - _ = view_workspace(app, "a_workspace") - - assert disconnecting_browser.is_running is False - + factory = SingletonRestoringBrowserFactory() + disconnecting_browser = factory.browser + disconnecting_browser.configure_client(channel=DisconnectingChannel()) -def test__disconnected_workspace__when_opening_again__starts_new_browser( - disconnecting_browser: BrowserTestDouble, - browser: BrowserTestDouble, - app: TestClient, -) -> None: - workspace = "a_workspace" - _ = view_workspace(app, workspace) + async with repository(factory) as repo: + async with patch_repository(repo): + app = TestClient(await create_app(create_settings())) + await repo.insert(disconnecting_browser) - _ = view_workspace(app, workspace) + _ = interact_with_workspace(app, "a_workspace") assert disconnecting_browser.is_running is False - assert browser.is_running is True @pytest.mark.usefixtures("disconnecting_browser") @@ -154,9 +159,9 @@ def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_t app: TestClient, ) -> None: workspace = "a_workspace" - _ = view_workspace(app, workspace) + _ = interact_with_workspace(app, workspace) - actual = view_workspace(app, workspace) + actual = interact_with_workspace(app, workspace) assert_is_browser_response(actual) @@ -166,45 +171,28 @@ def test__browsed_workspace_is_ready__when_pinging__returns_ok( app: TestClient, ) -> None: workspace = "a_workspace" - _ = view_workspace(app, workspace) + _ = interact_with_workspace(app, workspace) result = app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 200 -@pytest.mark.asyncio @pytest.mark.usefixtures("iterating_factory") -@pytest.mark.parametrize( - "repository", - ( - inmemory_repository, - pytest.param( - mongodb_repository, - marks=(pytest.mark.integration, pytest.mark.needs_docker), - ), - ), -) +@use_custom_repository async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( - singleton_restoring_factory: BrowserRestoringFactory, - repository: Callable[ - [BrowserRestoringFactory], - AsyncContextManager[BrowserProcessRepository], - ], + singleton_restoring_factory: SingletonRestoringBrowserFactory, + repository: RepositoryInitializer, ) -> None: async with repository(singleton_restoring_factory) as repo: async with patch_repository(repo): app = TestClient(await create_app(create_settings())) - browser = cast( - BrowserSpy, - singleton_restoring_factory("the-owner", "a_workspace", "", ""), - ) + browser = singleton_restoring_factory.browser browser.configure_client(response=ConnectionError) - await repo.insert(browser) workspace = "a_workspace" - _ = view_workspace(app, workspace) + open_workspace(app, workspace) result = app.get(f"/workspaces/ping/{workspace}") @@ -216,7 +204,7 @@ async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( async def test__browsing_workspace__stores_browser_in_repository( auto_repository: BrowserProcessRepository, app: TestClient ) -> None: - _ = view_workspace(app, "a_workspace") + _ = interact_with_workspace(app, "a_workspace") found_browsers = list( await auto_repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) @@ -225,50 +213,67 @@ async def test__browsing_workspace__stores_browser_in_repository( assert len(found_browsers) == 1 -@pytest.fixture -def singleton_restoring_factory() -> BrowserRestoringFactory: - spy = BrowserSpy() +@pytest.mark.usefixtures("iterating_factory") +@use_custom_repository +async def test__error_connecting_to_workspace__removes_browser_from_repository( + singleton_restoring_factory: SingletonRestoringBrowserFactory, + repository: RepositoryInitializer, +) -> None: + browser = singleton_restoring_factory.browser + browser.configure_client(response=ConnectionError) + + async with repository(singleton_restoring_factory) as repo: + async with patch_repository(repo): + app = TestClient(await create_app(create_settings())) + app.cookies.set("session_id", browser.owner()) - def factory( - owner: str, workspace: str, address: str, process_id: str - ) -> OcrdBrowser: - spy.set_owner_and_workspace(owner, workspace) - return spy + open_workspace(app, "a_workspace") + _ = view_workspace(app, "a_workspace") - return factory + found_browsers = list( + await repo.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) + + assert len(found_browsers) == 0 -@pytest.mark.asyncio @pytest.mark.usefixtures("iterating_factory") -@pytest.mark.parametrize( - "repository", - ( - inmemory_repository, - pytest.param( - mongodb_repository, - marks=(pytest.mark.integration, pytest.mark.needs_docker), - ), - ), -) -@pytest.mark.no_auto_repository -async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( - singleton_restoring_factory: BrowserRestoringFactory, - repository: Callable[ - [BrowserRestoringFactory], - AsyncContextManager[BrowserProcessRepository], - ], +@use_custom_repository +async def test__when_socket_to_workspace_disconnects__removes_browser_from_repository( + singleton_restoring_factory: SingletonRestoringBrowserFactory, + repository: RepositoryInitializer, ) -> None: + browser = singleton_restoring_factory.browser + browser.configure_client(channel=DisconnectingChannel()) + async with repository(singleton_restoring_factory) as repo: async with patch_repository(repo): app = TestClient(await create_app(create_settings())) + app.cookies.set("session_id", browser.owner()) + + _ = interact_with_workspace(app, "a_workspace") - browser = cast( - BrowserSpy, - singleton_restoring_factory("the-owner", "a_workspace", "", ""), + found_browsers = list( + await repo.find(workspace=str(WORKSPACE_DIR / "a_workspace")) ) + + assert len(found_browsers) == 0 + + +@pytest.mark.usefixtures("iterating_factory") +@use_custom_repository +async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( + singleton_restoring_factory: SingletonRestoringBrowserFactory, + repository: RepositoryInitializer, +) -> None: + async with repository(singleton_restoring_factory) as repo: + async with patch_repository(repo): + app = TestClient(await create_app(create_settings())) + + browser = singleton_restoring_factory.browser browser.configure_client(response=b"RESTORED BROWSER") await repo.insert(browser) - response = view_workspace(app, "a_workspace") + response = interact_with_workspace(app, "a_workspace") assert response.content == b"RESTORED BROWSER" diff --git a/tests/testdoubles/_browserfactory.py b/tests/testdoubles/_browserfactory.py index 5542895..d3f4b7a 100644 --- a/tests/testdoubles/_browserfactory.py +++ b/tests/testdoubles/_browserfactory.py @@ -1,6 +1,6 @@ import asyncio from types import TracebackType -from typing import Callable, Protocol, Self, Type +from typing import Any, Callable, Protocol, Self, Type from ocrdbrowser import OcrdBrowser from ._browserspy import BrowserSpy @@ -77,3 +77,23 @@ async def __aexit__( BrowserTestDoubleFactory = ( SingletonBrowserTestDoubleFactory | IteratingBrowserTestDoubleFactory ) + + +class SingletonRestoringBrowserFactory: + def __init__(self) -> None: + self.browser = BrowserSpy() + + async def __aenter__(self) -> "SingletonRestoringBrowserFactory": + await self.browser.start() + return self + + async def __aexit__(self, *args: Any, **kwargs: Any) -> None: + await self.browser.stop() + + def __call__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: + self.browser.set_owner_and_workspace(owner, workspace) + self.browser._address = address + self.browser._process_id = process_id + return self.browser diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index 4bd3b0e..ade99b0 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -1,23 +1,46 @@ -from typing import Collection +from typing import Collection, NamedTuple from ocrdbrowser import OcrdBrowser from ocrdmonitor.browserprocess import BrowserRestoringFactory from tests.testdoubles import BrowserSpy +class BrowserEntry(NamedTuple): + owner: str + workspace: str + address: str + process_id: str + + class InMemoryBrowserProcessRepository: def __init__( self, restoring_factory: BrowserRestoringFactory | None = None ) -> None: - self._processes: list[OcrdBrowser] = [] + self._processes: list[BrowserEntry] = [] self.restoring_factory: BrowserRestoringFactory = ( restoring_factory or BrowserSpy ) async def insert(self, browser: OcrdBrowser) -> None: - self._processes.append(browser) + entry = BrowserEntry( + browser.owner(), + browser.workspace(), + browser.address(), + browser.process_id(), + ) + + print(f"We're adding {entry}") + self._processes.append(entry) async def delete(self, browser: OcrdBrowser) -> None: - self._processes.remove(browser) + entry = BrowserEntry( + browser.owner(), + browser.workspace(), + browser.address(), + browser.process_id(), + ) + + print(f"We're deleting {entry}") + self._processes.remove(entry) async def find( self, @@ -25,22 +48,22 @@ async def find( owner: str | None = None, workspace: str | None = None, ) -> Collection[OcrdBrowser]: - def match(browser: OcrdBrowser) -> bool: + def match(browser: BrowserEntry) -> bool: matches = True if owner is not None: - matches = matches and browser.owner() == owner + matches = matches and browser.owner == owner if workspace is not None: - matches = matches and browser.workspace() == workspace + matches = matches and browser.workspace == workspace return matches return [ self.restoring_factory( - process_id=browser.process_id(), - owner=browser.owner(), - workspace=browser.workspace(), - address=browser.address(), + process_id=browser.process_id, + owner=browser.owner, + workspace=browser.workspace, + address=browser.address, ) for browser in self._processes if match(browser) From 1406a1551e2aaaf2b0cf08b891e30ee2653854c3 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Mon, 5 Jun 2023 16:00:18 +0200 Subject: [PATCH 12/32] Use FastAPI dependency injection --- ocrdbrowser/__init__.py | 6 +- ocrdmonitor/dbmodel.py | 1 - ocrdmonitor/main.py | 3 +- ocrdmonitor/server/app.py | 11 +-- ocrdmonitor/server/settings.py | 9 ++- ocrdmonitor/server/workspaces.py | 76 +++++++++++++------ pdm.lock | 59 +++++++------- pyproject.toml | 2 +- tests/ocrdmonitor/server/fixtures/app.py | 7 +- .../server/test_workspace_endpoint.py | 10 +-- 10 files changed, 105 insertions(+), 79 deletions(-) diff --git a/ocrdbrowser/__init__.py b/ocrdbrowser/__init__.py index 6c10fc6..409d938 100644 --- a/ocrdbrowser/__init__.py +++ b/ocrdbrowser/__init__.py @@ -7,19 +7,21 @@ OcrdBrowserFactory, ) from ._client import HttpBrowserClient -from ._docker import DockerOcrdBrowserFactory +from ._docker import DockerOcrdBrowser, DockerOcrdBrowserFactory from ._port import NoPortsAvailableError -from ._subprocess import SubProcessOcrdBrowserFactory +from ._subprocess import SubProcessOcrdBrowser, SubProcessOcrdBrowserFactory __all__ = [ "Channel", "ChannelClosed", + "DockerOcrdBrowser", "DockerOcrdBrowserFactory", "HttpBrowserClient", "NoPortsAvailableError", "OcrdBrowser", "OcrdBrowserClient", "OcrdBrowserFactory", + "SubProcessOcrdBrowser", "SubProcessOcrdBrowserFactory", "workspace", ] diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 075dd2c..0004974 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -9,7 +9,6 @@ from ocrdbrowser import OcrdBrowser from ocrdmonitor.browserprocess import BrowserRestoringFactory -from pymongo.results import DeleteResult class BrowserProcess(Document): diff --git a/ocrdmonitor/main.py b/ocrdmonitor/main.py index 47a21bf..c30568b 100644 --- a/ocrdmonitor/main.py +++ b/ocrdmonitor/main.py @@ -1,6 +1,5 @@ -import asyncio from ocrdmonitor.server.settings import Settings from ocrdmonitor.server.app import create_app settings = Settings() -app = asyncio.get_event_loop().run_until_complete(create_app(settings)) +app = create_app(settings) \ No newline at end of file diff --git a/ocrdmonitor/server/app.py b/ocrdmonitor/server/app.py index 3c81a07..439187d 100644 --- a/ocrdmonitor/server/app.py +++ b/ocrdmonitor/server/app.py @@ -20,7 +20,7 @@ TEMPLATE_DIR = PKG_DIR / "templates" -async def create_app(settings: Settings) -> FastAPI: +def create_app(settings: Settings) -> FastAPI: app = FastAPI() templates = Jinja2Templates(TEMPLATE_DIR) app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") @@ -40,14 +40,7 @@ async def swallow_exceptions(request: Request, err: Exception) -> Response: ), ) ) - app.include_router( - create_workspaces( - templates, - settings.ocrd_browser.factory(), - await settings.ocrd_browser.repository(), - settings.ocrd_browser.workspace_dir, - ) - ) + app.include_router(create_workspaces(templates, settings.ocrd_browser)) app.include_router(create_logs(templates, settings.ocrd_browser.workspace_dir)) app.include_router(create_workflows(templates)) app.include_router(create_logview(templates, settings.ocrd_logview.port)) diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index 6e9c003..8476a4c 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -8,12 +8,14 @@ from pydantic import BaseModel, BaseSettings, validator from ocrdbrowser import ( + DockerOcrdBrowser, DockerOcrdBrowserFactory, OcrdBrowserFactory, SubProcessOcrdBrowserFactory, + SubProcessOcrdBrowser, ) from ocrdmonitor import dbmodel -from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.sshremote import SSHRemote @@ -42,7 +44,10 @@ class OcrdBrowserSettings(BaseModel): async def repository(self) -> BrowserProcessRepository: await dbmodel.init(self.db_connection_string) - return dbmodel.MongoBrowserProcessRepository() + restore: BrowserRestoringFactory = ( + SubProcessOcrdBrowser if self.mode == "native" else DockerOcrdBrowser + ) + return dbmodel.MongoBrowserProcessRepository(restore) def factory(self) -> OcrdBrowserFactory: port_range_set = set(range(*self.port_range)) diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index f61a4ff..6b6566e 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -3,28 +3,37 @@ import uuid from pathlib import Path -from fastapi import APIRouter, Cookie, Request, Response, WebSocket, WebSocketDisconnect +from fastapi import ( + APIRouter, + Cookie, + Depends, + Request, + Response, + WebSocket, + WebSocketDisconnect, +) from fastapi.templating import Jinja2Templates -import ocrdmonitor.server.proxy as proxy from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace from ocrdmonitor.browserprocess import BrowserProcessRepository -from ocrdmonitor.server.redirect import BrowserRedirect, RedirectMap +from ocrdmonitor.server import proxy +from ocrdmonitor.server.redirect import BrowserRedirect +from ocrdmonitor.server.settings import OcrdBrowserSettings def create_workspaces( templates: Jinja2Templates, - factory: OcrdBrowserFactory, - repository: BrowserProcessRepository, - workspace_dir: Path, + browser_settings: OcrdBrowserSettings, ) -> APIRouter: router = APIRouter(prefix="/workspaces") + WORKSPACE_DIR = browser_settings.workspace_dir + @router.get("/", name="workspaces.list") def list_workspaces(request: Request) -> Response: spaces = [ - Path(space).relative_to(workspace_dir) - for space in workspace.list_all(workspace_dir) + Path(space).relative_to(WORKSPACE_DIR) + for space in workspace.list_all(WORKSPACE_DIR) ] return templates.TemplateResponse( @@ -33,18 +42,23 @@ def list_workspaces(request: Request) -> Response: ) @router.get("/browse/{workspace:path}", name="workspaces.browse") - async def browser(request: Request, workspace: Path) -> Response: + async def browser( + request: Request, + workspace: Path, + factory: OcrdBrowserFactory = Depends(browser_settings.factory), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) response = Response() response.set_cookie("session_id", session_id) - full_workspace = str(workspace_dir / workspace) + full_workspace = str(WORKSPACE_DIR / workspace) existing_browsers = await repository.find( owner=session_id, workspace=full_workspace ) if not existing_browsers: - browser = await launch_browser(session_id, workspace) + browser = await launch_browser(factory, session_id, workspace) await repository.insert(browser) return response @@ -58,11 +72,13 @@ def open_workspace(request: Request, workspace: str) -> Response: @router.get("/ping/{workspace:path}", name="workspaces.ping") async def ping_workspace( - workspace: Path, session_id: str = Cookie(default=None) + workspace: Path, + session_id: str = Cookie(default=None), + repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: browsers = list( await repository.find( - owner=session_id, workspace=str(workspace_dir / workspace) + owner=session_id, workspace=str(WORKSPACE_DIR / workspace) ) ) try: @@ -77,18 +93,21 @@ async def ping_workspace( # which points to the last component with a trailing slash. @router.get("/view/{workspace:path}/", name="workspaces.view") async def workspace_reverse_proxy( - request: Request, workspace: Path, session_id: str = Cookie(default=None) + request: Request, + workspace: Path, + session_id: str = Cookie(default=None), + repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: browsers = list( await repository.find( - owner=session_id, workspace=str(workspace_dir / workspace) + owner=session_id, workspace=str(WORKSPACE_DIR / workspace) ) ) redirect = BrowserRedirect(workspace, browsers[0]) try: return await proxy.forward(redirect, str(workspace)) except ConnectionError: - await stop_browser(redirect.browser) + await stop_browser(repository, redirect.browser) return templates.TemplateResponse( "view_workspace_failed.html.j2", {"request": request, "workspace": workspace}, @@ -96,11 +115,14 @@ async def workspace_reverse_proxy( @router.websocket("/view/{workspace:path}/socket", name="workspaces.view.socket") async def workspace_socket_proxy( - websocket: WebSocket, workspace: Path, session_id: str = Cookie(default=None) + websocket: WebSocket, + workspace: Path, + session_id: str = Cookie(default=None), + repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> None: browsers = list( await repository.find( - owner=session_id, workspace=str(workspace_dir / workspace) + owner=session_id, workspace=str(WORKSPACE_DIR / workspace) ) ) @@ -109,25 +131,31 @@ async def workspace_socket_proxy( redirect = BrowserRedirect(workspace, browsers[0]) await websocket.accept(subprotocol="broadway") - await communicate_with_browser_until_closed(websocket, redirect.browser) + await communicate_with_browser_until_closed( + repository, websocket, redirect.browser + ) async def communicate_with_browser_until_closed( - websocket: WebSocket, browser: OcrdBrowser + repository: BrowserProcessRepository, websocket: WebSocket, browser: OcrdBrowser ) -> None: async with browser.client().open_channel() as channel: try: while True: await proxy.tunnel(channel, websocket) except ChannelClosed: - await stop_browser(browser) + await stop_browser(repository, browser) except WebSocketDisconnect: pass - async def launch_browser(session_id: str, workspace: Path) -> OcrdBrowser: - full_workspace_path = workspace_dir / workspace + async def launch_browser( + factory: OcrdBrowserFactory, session_id: str, workspace: Path + ) -> OcrdBrowser: + full_workspace_path = WORKSPACE_DIR / workspace return await factory(session_id, str(full_workspace_path)) - async def stop_browser(browser: OcrdBrowser) -> None: + async def stop_browser( + repository: BrowserProcessRepository, browser: OcrdBrowser + ) -> None: await browser.stop() await repository.delete(browser) diff --git a/pdm.lock b/pdm.lock index f6c4e50..9131652 100644 --- a/pdm.lock +++ b/pdm.lock @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "mypy" -version = "1.1.1" +version = "1.3.0" requires_python = ">=3.7" summary = "Optional static typing for Python" dependencies = [ @@ -475,8 +475,9 @@ summary = "Module for decorators, wrappers and monkey patching." [metadata] lock_version = "4.2" +cross_platform = true groups = ["default", "dev", "nox"] -content_hash = "sha256:9fccfa483f45b614a2680a3e849e3f890096aeb9ba0b41824e49540ebfd88325" +content_hash = "sha256:19d49bce6b07f9f1bacc395b0a137186149b00ad97e508f457767f79d7935f86" [metadata.files] "anyio 3.6.2" = [ @@ -727,33 +728,33 @@ content_hash = "sha256:9fccfa483f45b614a2680a3e849e3f890096aeb9ba0b41824e49540eb {url = "https://files.pythonhosted.org/packages/82/96/ae017cd62761d2fd2cc1eabfc902c3b4e3768fe994fc6a2f474694a56910/motor-3.1.2.tar.gz", hash = "sha256:80c08477c09e70db4f85c99d484f2bafa095772f1d29b3ccb253270f9041da9a"}, {url = "https://files.pythonhosted.org/packages/f9/c3/22a695d0e6c373d0a33036de7fdc084068d896e948d11b691c88b6c1672f/motor-3.1.2-py3-none-any.whl", hash = "sha256:4bfc65230853ad61af447088527c1197f91c20ee957cfaea3144226907335716"}, ] -"mypy 1.1.1" = [ - {url = "https://files.pythonhosted.org/packages/2a/28/8485aad67750b3374443d28bad3eed947737cf425a640ea4be4ac70a7827/mypy-1.1.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce61663faf7a8e5ec6f456857bfbcec2901fbdb3ad958b778403f63b9e606a1b"}, - {url = "https://files.pythonhosted.org/packages/30/da/808ceaf2bcf23a9e90156c7b11b41add8dd5a009ee48159ec820d04d97bd/mypy-1.1.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9401e33814cec6aec8c03a9548e9385e0e228fc1b8b0a37b9ea21038e64cdd8a"}, - {url = "https://files.pythonhosted.org/packages/44/9d/d23fa5d12bacbe7beea5fb6315b3325beabbe438e7e14d38c82b71609818/mypy-1.1.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39c7119335be05630611ee798cc982623b9e8f0cff04a0b48dfc26100e0b97af"}, - {url = "https://files.pythonhosted.org/packages/47/9f/34f6a2254f7d39b8c4349b8ac480c233d37c377faf2c67c6ef925b3af0ab/mypy-1.1.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:59bbd71e5c58eed2e992ce6523180e03c221dcd92b52f0e792f291d67b15a71c"}, - {url = "https://files.pythonhosted.org/packages/61/99/4a844dcacbc4990a8312236bf74a55910ee9a05db69dee7d6fb7a7ffe6c2/mypy-1.1.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dbb19c9f662e41e474e0cff502b7064a7edc6764f5262b6cd91d698163196799"}, - {url = "https://files.pythonhosted.org/packages/62/54/be80f8d01f5cf72f774a77f9f750527a6fa733f09f78b1da30e8fa3914e6/mypy-1.1.1.tar.gz", hash = "sha256:ae9ceae0f5b9059f33dbc62dea087e942c0ccab4b7a003719cb70f9b8abfa32f"}, - {url = "https://files.pythonhosted.org/packages/64/63/6a04ca7a8b7f34811cada43ed6119736a7f4a07c5e1cbd8eec0e0f4962d5/mypy-1.1.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d809f88734f44a0d44959d795b1e6f64b2bbe0ea4d9cc4776aa588bb4229fc1c"}, - {url = "https://files.pythonhosted.org/packages/65/cc/ae5032abc06949e7a8c68f9885883fdb745c96bcf137cd4fa7225d50b647/mypy-1.1.1-cp38-cp38-win_amd64.whl", hash = "sha256:2888ce4fe5aae5a673386fa232473014056967f3904f5abfcf6367b5af1f612a"}, - {url = "https://files.pythonhosted.org/packages/67/d3/1323311369eae97da4c7f47f266c55f7bdc22e74e4e2e1691be511ab8a91/mypy-1.1.1-cp39-cp39-win_amd64.whl", hash = "sha256:69b35d1dcb5707382810765ed34da9db47e7f95b3528334a3c999b0c90fe523f"}, - {url = "https://files.pythonhosted.org/packages/7e/32/1b161731d19580c55d3d7c04b8ace80dc7cf42d852adf750f348a485068f/mypy-1.1.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:b7c7b708fe9a871a96626d61912e3f4ddd365bf7f39128362bc50cbd74a634d5"}, - {url = "https://files.pythonhosted.org/packages/8a/fd/b610256224e01da4c4f315d11f62d39d815e97439a58d49d60aa4f55a60b/mypy-1.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:61bf08362e93b6b12fad3eab68c4ea903a077b87c90ac06c11e3d7a09b56b9c1"}, - {url = "https://files.pythonhosted.org/packages/8c/3d/a8d518bb06952484ada20897878a7a14741536f43514dcfecfac0676aa01/mypy-1.1.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c1c10fa12df1232c936830839e2e935d090fc9ee315744ac33b8a32216b93707"}, - {url = "https://files.pythonhosted.org/packages/91/63/55d0e62829f739f47978f1d8eb965ca8c40261841e47491ad297c84921c5/mypy-1.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:5cb14ff9919b7df3538590fc4d4c49a0f84392237cbf5f7a816b4161c061829e"}, - {url = "https://files.pythonhosted.org/packages/a4/0b/3a30f50287e42a4230320fa2eac25eb3017d38a7c31f083d407ab627607c/mypy-1.1.1-py3-none-any.whl", hash = "sha256:4e4e8b362cdf99ba00c2b218036002bdcdf1e0de085cdb296a49df03fb31dfc4"}, - {url = "https://files.pythonhosted.org/packages/b8/06/3d72d1b316ceec347874c4285fad8bf17e3fb21bb7848c1a942df239e44a/mypy-1.1.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:d64c28e03ce40d5303450f547e07418c64c241669ab20610f273c9e6290b4b0b"}, - {url = "https://files.pythonhosted.org/packages/b8/72/385f3aeaaf262325454ac7f569eb81ac623464871df23d9778c864d04c6c/mypy-1.1.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:19ba15f9627a5723e522d007fe708007bae52b93faab00f95d72f03e1afa9598"}, - {url = "https://files.pythonhosted.org/packages/b9/e5/71eef5239219ee2f4d85e2ca6368d736705a3b874023b57f7237b977839c/mypy-1.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:5b5f81b40d94c785f288948c16e1f2da37203c6006546c5d947aab6f90aefef2"}, - {url = "https://files.pythonhosted.org/packages/be/d5/5588a2ee0d77189626a57b555b6b006dda6d5b0083f16c6be0c2d761cd7b/mypy-1.1.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:4b398d8b1f4fba0e3c6463e02f8ad3346f71956b92287af22c9b12c3ec965a9f"}, - {url = "https://files.pythonhosted.org/packages/bf/2d/45a526f248719ee32ecf1261564247a2e717a9c6167de5eb67d53599c4df/mypy-1.1.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21b437be1c02712a605591e1ed1d858aba681757a1e55fe678a15c2244cd68a5"}, - {url = "https://files.pythonhosted.org/packages/c0/d6/17ba6f8749722b8f61c6ab680769658f0bc63c293556149e2bf400b1f1a2/mypy-1.1.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:315ac73cc1cce4771c27d426b7ea558fb4e2836f89cb0296cbe056894e3a1f78"}, - {url = "https://files.pythonhosted.org/packages/d3/35/a0892864f1c128dc6449ee69897f9db7a64de2c16f41c14640dd22251b1b/mypy-1.1.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0a28a76785bf57655a8ea5eb0540a15b0e781c807b5aa798bd463779988fa1d5"}, - {url = "https://files.pythonhosted.org/packages/d9/ab/d6d3884c3f432898458e2ade712988a7d1da562c1a363f2003b31677acd8/mypy-1.1.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:26cdd6a22b9b40b2fd71881a8a4f34b4d7914c679f154f43385ca878a8297389"}, - {url = "https://files.pythonhosted.org/packages/e1/a6/331cff5f7476904a2ebe6ed7cee2310b6be583ff6d45609ea0e0d67fd39d/mypy-1.1.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:64cc3afb3e9e71a79d06e3ed24bb508a6d66f782aff7e56f628bf35ba2e0ba51"}, - {url = "https://files.pythonhosted.org/packages/ed/89/85a04f32135fe4e35fd59d47100c939c7425fcb29868894c4b7a6171e065/mypy-1.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:a380c041db500e1410bb5b16b3c1c35e61e773a5c3517926b81dfdab7582be54"}, - {url = "https://files.pythonhosted.org/packages/f5/35/da01ef5831ceaf99a673e018d06ff1622ec460e4164b5e900ddaeceb52e1/mypy-1.1.1-cp37-cp37m-win_amd64.whl", hash = "sha256:ef6a01e563ec6a4940784c574d33f6ac1943864634517984471642908b30b6f7"}, - {url = "https://files.pythonhosted.org/packages/f6/57/93e676773f91141127329a56e2238eac506a78f6fb0ae0650a53fcc1355d/mypy-1.1.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2b0c373d071593deefbcdd87ec8db91ea13bd8f1328d44947e88beae21e8d5e9"}, +"mypy 1.3.0" = [ + {url = "https://files.pythonhosted.org/packages/09/7b/8eb0d648352c61b08cb364d278b5c12c3f1c5841724fdd2929d7172b7eaf/mypy-1.3.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:f9dca1e257d4cc129517779226753dbefb4f2266c4eaad610fc15c6a7e14283e"}, + {url = "https://files.pythonhosted.org/packages/11/41/d24f93eefc89c650782bf1f9acfdb02a32f327b841058a5b0ce5857b60af/mypy-1.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4c99c3ecf223cf2952638da9cd82793d8f3c0c5fa8b6ae2b2d9ed1e1ff51ba85"}, + {url = "https://files.pythonhosted.org/packages/25/c7/4735f81858a727e170279144600881fe3299aa7589ed585af6b788ea4556/mypy-1.3.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6e33bb8b2613614a33dff70565f4c803f889ebd2f859466e42b46e1df76018dd"}, + {url = "https://files.pythonhosted.org/packages/2b/27/4a26f91301804969194ee0dc9393843f10566d7fdf192ce11fc0218a989d/mypy-1.3.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c1eb485cea53f4f5284e5baf92902cd0088b24984f4209e25981cc359d64448d"}, + {url = "https://files.pythonhosted.org/packages/3c/5d/b87339c1fdfec7d13899cd7ad2ee992801695114c1cf9e1645da264cd437/mypy-1.3.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:74bc9b6e0e79808bf8678d7678b2ae3736ea72d56eede3820bd3849823e7f305"}, + {url = "https://files.pythonhosted.org/packages/47/f6/25c154bb1c479f2047093f0580c2c35ffc1ff007d52b7e50020cca60c010/mypy-1.3.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:76ec771e2342f1b558c36d49900dfe81d140361dd0d2df6cd71b3db1be155409"}, + {url = "https://files.pythonhosted.org/packages/4c/10/530d2df4d57f46f77b8211cf9bbe090baacff02e7076f21f1bf08148d541/mypy-1.3.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ddae0f39ca146972ff6bb4399f3b2943884a774b8771ea0a8f50e971f5ea5ba8"}, + {url = "https://files.pythonhosted.org/packages/55/e1/90487a3ea5a88b8f5c9d7fbf6f5fa7fcc8633d0132ce8364810a1da901c9/mypy-1.3.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:cbc07246253b9e3d7d74c9ff948cd0fd7a71afcc2b77c7f0a59c26e9395cb152"}, + {url = "https://files.pythonhosted.org/packages/5b/fb/0b1c90c635319b98dd65c6d6d6347413e42397e94057993011eeedeffbd9/mypy-1.3.0-cp37-cp37m-win_amd64.whl", hash = "sha256:8c5979d0deb27e0f4479bee18ea0f83732a893e81b78e62e2dda3e7e518c92ee"}, + {url = "https://files.pythonhosted.org/packages/6a/d0/4681d84878cecfd911752016ab30566366f6de7296fdc977b746eb68bf45/mypy-1.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:7d23370d2a6b7a71dc65d1266f9a34e4cde9e8e21511322415db4b26f46f6b8c"}, + {url = "https://files.pythonhosted.org/packages/6a/d9/48de5203f4b6287a98fadcc47072b1bc69e3faaa39cba59a3a600b05a42c/mypy-1.3.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6e42d29e324cdda61daaec2336c42512e59c7c375340bd202efa1fe0f7b8f8ca"}, + {url = "https://files.pythonhosted.org/packages/7e/75/021af7f0683ea19b9ad6a436e1b5c7cb39899c0f7b31040fa69b2395421e/mypy-1.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:a22435632710a4fcf8acf86cbd0d69f68ac389a3892cb23fbad176d1cddaf228"}, + {url = "https://files.pythonhosted.org/packages/86/56/08c5ff6b2139f301d9aa56cb8e7b2a24d4faa6fc3e94234dfe7eeecc9c44/mypy-1.3.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:faff86aa10c1aa4a10e1a301de160f3d8fc8703b88c7e98de46b531ff1276a9a"}, + {url = "https://files.pythonhosted.org/packages/88/0e/646696eb8fe7658b752009a495054a0214ae8e659e9cbcde8181f16ae999/mypy-1.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:658fe7b674769a0770d4b26cb4d6f005e88a442fe82446f020be8e5f5efb2fae"}, + {url = "https://files.pythonhosted.org/packages/8d/c8/681f4a19c62aa71bdc9ad3a4bc9a0fb8846bd0b5a8bc1b29d261c8025f80/mypy-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:550a8b3a19bb6589679a7c3c31f64312e7ff482a816c96e0cecec9ad3a7564dd"}, + {url = "https://files.pythonhosted.org/packages/90/b6/a2d2ba604982af6034e3fcad17a464a66127be47f07b4587beec76e8f80b/mypy-1.3.0-cp38-cp38-win_amd64.whl", hash = "sha256:44797d031a41516fcf5cbfa652265bb994e53e51994c1bd649ffcd0c3a7eccbf"}, + {url = "https://files.pythonhosted.org/packages/b1/ce/8d87f684bb7e2a520cfa9cd17b8dc686a83143bb12a3e1ac4ad6d8d4825c/mypy-1.3.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:c5d2cc54175bab47011b09688b418db71403aefad07cbcd62d44010543fc143f"}, + {url = "https://files.pythonhosted.org/packages/b1/e1/399e3dfeb2842e4a2634866e4ef8b69151d465b7a5ceb648d7f1296f17d0/mypy-1.3.0-cp39-cp39-win_amd64.whl", hash = "sha256:95d8d31a7713510685b05fbb18d6ac287a56c8f6554d88c19e73f724a445448a"}, + {url = "https://files.pythonhosted.org/packages/b8/36/6628916f94bb0816e1719117e1962750413ab408f83673ce7d571caf3960/mypy-1.3.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:1c4c42c60a8103ead4c1c060ac3cdd3ff01e18fddce6f1016e08939647a0e703"}, + {url = "https://files.pythonhosted.org/packages/ba/ac/1c280246fc0c5239409f31e1a321f178ba11a9c6e5eaaf6d56f9ff627cdf/mypy-1.3.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e86c2c6852f62f8f2b24cb7a613ebe8e0c7dc1402c61d36a609174f63e0ff017"}, + {url = "https://files.pythonhosted.org/packages/c9/c5/f3e4ed59e08e3a728a15da198317edfcd13b7dc2215d52b5d85fce716285/mypy-1.3.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:87df44954c31d86df96c8bd6e80dfcd773473e877ac6176a8e29898bfb3501cb"}, + {url = "https://files.pythonhosted.org/packages/cd/b9/6abe1cd8ac8e70f12f43eebe6427814f9d36142d331eae5cc5bba77585a2/mypy-1.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:d0b6c62206e04061e27009481cb0ec966f7d6172b5b936f3ead3d74f29fe3dcf"}, + {url = "https://files.pythonhosted.org/packages/d8/c6/de2e214a42b63d7ea0abef9f02a6da69cad6d532165bb7a8cc8291099a0c/mypy-1.3.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:473117e310febe632ddf10e745a355714e771ffe534f06db40702775056614c4"}, + {url = "https://files.pythonhosted.org/packages/d9/79/82d452b409d7610944ba3a1a6079987d3ed6062cb8fe5c8850f26dafb6e0/mypy-1.3.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ebc95f8386314272bbc817026f8ce8f4f0d2ef7ae44f947c4664efac9adec929"}, + {url = "https://files.pythonhosted.org/packages/e3/f7/1fed3b24abb75f244fa6bc60ea03cd9d3d8ad225a4cfda7533042fe6d831/mypy-1.3.0-py3-none-any.whl", hash = "sha256:a8763e72d5d9574d45ce5881962bc8e9046bf7b375b0abf031f3e6811732a897"}, + {url = "https://files.pythonhosted.org/packages/f9/88/3bfe07521fb9e74b449cbc4367434067ec70bfd8a24c652fa3e0f9597389/mypy-1.3.0.tar.gz", hash = "sha256:e1f4d16e296f5135624b34e8fb741eb0eadedca90862405b1f1fde2040b9bd11"}, ] "mypy-extensions 1.0.0" = [ {url = "https://files.pythonhosted.org/packages/2a/e2/5d3f6ada4297caebe1a2add3b126fe800c96f56dbe5d1988a2cbe0b267aa/mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, diff --git a/pyproject.toml b/pyproject.toml index 42f67f1..f686848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,7 +24,7 @@ license = { text = "MIT" } [project.optional-dependencies] dev = [ "beautifulsoup4>=4.11.1", - "mypy>=1.1.1", + "mypy>=1.3.0", "pytest>=7.2.2", "pytest-asyncio>=0.21.0", "testcontainers>=3.7.1", diff --git a/tests/ocrdmonitor/server/fixtures/app.py b/tests/ocrdmonitor/server/fixtures/app.py index 13a2ee5..5d805a8 100644 --- a/tests/ocrdmonitor/server/fixtures/app.py +++ b/tests/ocrdmonitor/server/fixtures/app.py @@ -2,7 +2,6 @@ from typing import Iterator import pytest -import pytest_asyncio import uvicorn from fastapi.testclient import TestClient @@ -35,9 +34,9 @@ def create_settings() -> Settings: ) -@pytest_asyncio.fixture -async def app() -> TestClient: - return TestClient(await create_app(create_settings())) +@pytest.fixture +def app() -> TestClient: + return TestClient(create_app(create_settings())) def _launch_app() -> None: diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 39f2f6d..58be863 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -146,7 +146,7 @@ async def test__opened_workspace__when_socket_disconnects_on_broadway_side__shut async with repository(factory) as repo: async with patch_repository(repo): - app = TestClient(await create_app(create_settings())) + app = TestClient(create_app(create_settings())) await repo.insert(disconnecting_browser) _ = interact_with_workspace(app, "a_workspace") @@ -186,7 +186,7 @@ async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( ) -> None: async with repository(singleton_restoring_factory) as repo: async with patch_repository(repo): - app = TestClient(await create_app(create_settings())) + app = TestClient(create_app(create_settings())) browser = singleton_restoring_factory.browser browser.configure_client(response=ConnectionError) @@ -224,7 +224,7 @@ async def test__error_connecting_to_workspace__removes_browser_from_repository( async with repository(singleton_restoring_factory) as repo: async with patch_repository(repo): - app = TestClient(await create_app(create_settings())) + app = TestClient(create_app(create_settings())) app.cookies.set("session_id", browser.owner()) open_workspace(app, "a_workspace") @@ -248,7 +248,7 @@ async def test__when_socket_to_workspace_disconnects__removes_browser_from_repos async with repository(singleton_restoring_factory) as repo: async with patch_repository(repo): - app = TestClient(await create_app(create_settings())) + app = TestClient(create_app(create_settings())) app.cookies.set("session_id", browser.owner()) _ = interact_with_workspace(app, "a_workspace") @@ -268,7 +268,7 @@ async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_res ) -> None: async with repository(singleton_restoring_factory) as repo: async with patch_repository(repo): - app = TestClient(await create_app(create_settings())) + app = TestClient(create_app(create_settings())) browser = singleton_restoring_factory.browser browser.configure_client(response=b"RESTORED BROWSER") From a0877cf2bc43272bb10a697938155f85068e9459 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Mon, 5 Jun 2023 16:00:44 +0200 Subject: [PATCH 13/32] Update docker-compose.yml and init.sh for MongoDB --- docker-compose.yml | 33 +++++++++++++++++++++++---------- init.sh | 1 + 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index afbe83a..dfe4300 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,6 +15,7 @@ services: environment: MONITOR_PORT_LOG: ${MONITOR_PORT_LOG} CONTROLLER: "${CONTROLLER_HOST}:${CONTROLLER_PORT_SSH}" + DB_CONNECTION: "${DB_ROOT_USER:-root}:${DB_ROOT_PASSWORD:-root_password}@ocrd-database:27017" ports: - ${MONITOR_PORT_WEB}:5000 @@ -25,17 +26,29 @@ services: - ${MANAGER_KEY}:/id_rsa - shared:/run/lock/ocrd.jobs - ocrd-logview: - image: amir20/dozzle:latest - volumes: - # double slash is mandatory to support windows - - //var/run/docker.sock:/var/run/docker.sock - ports: - - ${MONITOR_PORT_LOG}:8080 + # ocrd-logview: + # image: amir20/dozzle:latest + # volumes: + # # double slash is mandatory to support windows + # - //var/run/docker.sock:/var/run/docker.sock + # ports: + # - ${MONITOR_PORT_LOG}:8080 + # environment: + # - DOZZLE_FILTER=name=ocrd_kitodo + # # DOZZLE_USERNAME= + # # DOZZLE_PASSWORD= + + ocrd-database: + image: "mongo:latest" + environment: - - DOZZLE_FILTER=name=ocrd_kitodo - # DOZZLE_USERNAME= - # DOZZLE_PASSWORD= + MONGO_INITDB_ROOT_USERNAME: ${DB_ROOT_USER:-root} + MONGO_INITDB_ROOT_PASSWORD: ${DB_ROOT_PASSWORD:-root_password} + + volumes: + - db-volume:/data/db + volumes: + db-volume: shared: diff --git a/init.sh b/init.sh index f74408e..e75de6b 100755 --- a/init.sh +++ b/init.sh @@ -20,6 +20,7 @@ fi export OCRD_BROWSER__MODE=native export OCRD_BROWSER__WORKSPACE_DIR=/data export OCRD_BROWSER__PORT_RANGE="[9000,9100]" +export OCRD_BROWSER__DB_CONNECTION_STRING=$DB_CONNECTION export OCRD_LOGVIEW__PORT=$MONITOR_PORT_LOG export OCRD_CONTROLLER__JOB_DIR=/run/lock/ocrd.jobs export OCRD_CONTROLLER__HOST=$CONTROLLER_HOST From 25683e06c0de16ca69df826b3d9c961e37fae0b8 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 6 Jun 2023 18:34:07 +0200 Subject: [PATCH 14/32] Working Docker Browsers. Subprocesses are crashing --- docker-compose.yml | 13 ++- ocrdbrowser/_client.py | 13 ++- ocrdbrowser/_docker.py | 5 +- ocrdbrowser/_subprocess.py | 30 +++-- ocrdmonitor/dbmodel.py | 6 + ocrdmonitor/server/proxy.py | 15 ++- ocrdmonitor/server/redirect.py | 107 ------------------ ocrdmonitor/server/workspaces.py | 36 +++--- .../ocrdmonitor/server/fixtures/repository.py | 1 + .../server/test_workspace_endpoint.py | 23 ++++ tests/ocrdmonitor/test_redirect.py | 73 ------------ tests/testdoubles/_browserspy.py | 18 ++- 12 files changed, 116 insertions(+), 224 deletions(-) delete mode 100644 ocrdmonitor/server/redirect.py delete mode 100644 tests/ocrdmonitor/test_redirect.py diff --git a/docker-compose.yml b/docker-compose.yml index dfe4300..951ad3b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: environment: MONITOR_PORT_LOG: ${MONITOR_PORT_LOG} CONTROLLER: "${CONTROLLER_HOST}:${CONTROLLER_PORT_SSH}" - DB_CONNECTION: "${DB_ROOT_USER:-root}:${DB_ROOT_PASSWORD:-root_password}@ocrd-database:27017" + DB_CONNECTION: "mongodb://${DB_ROOT_USER:-root}:${DB_ROOT_PASSWORD:-root_password}@ocrd-database:27017" ports: - ${MONITOR_PORT_WEB}:5000 @@ -49,6 +49,17 @@ services: - db-volume:/data/db + mongo-express: + image: mongo-express:latest + depends_on: + - ocrd-database + ports: + - 8081:8081 + environment: + ME_CONFIG_MONGODB_ADMINUSERNAME: ${DB_ROOT_USER:-root} + ME_CONFIG_MONGODB_ADMINPASSWORD: ${DB_ROOT_PASSWORD:-root_password} + ME_CONFIG_MONGODB_SERVER: ocrd-database + volumes: db-volume: shared: diff --git a/ocrdbrowser/_client.py b/ocrdbrowser/_client.py index 10ab34e..3feb37f 100644 --- a/ocrdbrowser/_client.py +++ b/ocrdbrowser/_client.py @@ -1,5 +1,7 @@ from __future__ import annotations +import logging + from types import TracebackType from typing import AsyncContextManager, Type, cast @@ -66,9 +68,14 @@ def __init__(self, address: str) -> None: self.address = address async def get(self, resource: str) -> bytes: - async with httpx.AsyncClient(base_url=self.address) as client: - response = await client.get(resource) - return response.content + try: + async with httpx.AsyncClient(base_url=self.address) as client: + response = await client.get(resource) + return response.content + except Exception as ex: + logging.error(f"Tried to connect to {self.address}") + logging.error(f"Requested resource {resource}") + raise ex def open_channel(self) -> AsyncContextManager[Channel]: return WebSocketChannel(self.address + "/socket") diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index 0896fc9..f9a8182 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -77,7 +77,10 @@ async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: port.get(), ) - container_id = str(cmd.stdout).strip() + stdout = cmd.stdout + container_id = "" + if stdout: + container_id = str(await stdout.read()).strip() container = DockerOcrdBrowser( owner, diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 264b3de..40ad62b 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging import os import signal from shutil import which @@ -51,6 +52,7 @@ async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: browser = SubProcessOcrdBrowser( owner, workspace_path, address, str(process.pid) ) + print("Launched Process ID", process.pid) return browser async def start_browser( @@ -69,15 +71,19 @@ async def start_browser( environment["GDK_BACKEND"] = "broadway" environment["BROADWAY_DISPLAY"] = ":" + displayport - return await asyncio.create_subprocess_shell( - " ".join( - [ - "broadwayd", - ":" + displayport + " &", - browse_ocrd, - workspace + "/mets.xml ;", - "kill $!", - ] - ), - env=environment, - ) + try: + return await asyncio.create_subprocess_shell( + " ".join( + [ + "broadwayd", + ":" + displayport + " &", + browse_ocrd, + workspace + "/mets.xml ;", + "kill $!", + ] + ), + env=environment, + ) + except Exception as err: + logging.error(f"Tried to launch broadway at {displayport} (real port {port})") + raise err \ No newline at end of file diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 0004974..fa7de9d 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -2,6 +2,7 @@ from typing import Any, Collection, Mapping import pymongo +import urllib from beanie import Document, init_beanie from beanie.odm.queries.find import FindMany from motor.motor_asyncio import AsyncIOMotorClient @@ -90,6 +91,11 @@ async def clean(self) -> None: async def init(connection_str: str) -> None: + connection_str = connection_str.removeprefix("mongodb://") + credentials, host = connection_str.split("@") + user, password = credentials.split(":") + password = urllib.parse.quote(password) + connection_str = f"mongodb://{user}:{password}@{host}" client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) client.get_io_loop = asyncio.get_event_loop await init_beanie( diff --git a/ocrdmonitor/server/proxy.py b/ocrdmonitor/server/proxy.py index b83d710..a5020a7 100644 --- a/ocrdmonitor/server/proxy.py +++ b/ocrdmonitor/server/proxy.py @@ -3,14 +3,19 @@ import asyncio from fastapi import Response -from ocrdbrowser import Channel +from ocrdbrowser import OcrdBrowser, Channel +from difflib import SequenceMatcher -from .redirect import BrowserRedirect +def get_redirect_url(browser: OcrdBrowser, url: str) -> str: + matcher = SequenceMatcher(None, browser.workspace(), url) + match = matcher.find_longest_match() + return url[match.size :] -async def forward(redirect: BrowserRedirect, url: str) -> Response: - redirect_url = redirect.redirect_url(url) - resource = await redirect.browser.client().get(redirect_url) + +async def forward(browser: OcrdBrowser, url: str) -> Response: + url = get_redirect_url(browser, url) + resource = await browser.client().get(url) return Response(content=resource) diff --git a/ocrdmonitor/server/redirect.py b/ocrdmonitor/server/redirect.py deleted file mode 100644 index 226c6e6..0000000 --- a/ocrdmonitor/server/redirect.py +++ /dev/null @@ -1,107 +0,0 @@ -from __future__ import annotations - -from pathlib import Path -from typing import Callable - -from ocrdbrowser import OcrdBrowser - - -def removeprefix(string: str, prefix: str) -> str: - def __removeprefix(prefix: str) -> str: - if string.startswith(prefix): - len_prefix = len(prefix) - return string[len_prefix:] - - return string - - _removeprefix: Callable[[str], str] = getattr( - string, "removeprefix", __removeprefix - ) - return _removeprefix(prefix) - - -def removesuffix(string: str, suffix: str) -> str: - def __removesuffix(suffix: str) -> str: - if string.endswith(suffix): - len_suffix = len(suffix) - return string[-len_suffix:] - - return string - - _removesuffix: Callable[[str], str] = getattr( - string, "removesuffix", __removesuffix - ) - - return _removesuffix(suffix) - - -class BrowserRedirect: - def __init__(self, workspace: Path, browser: OcrdBrowser) -> None: - self._workspace = workspace - self._browser = browser - - @property - def browser(self) -> OcrdBrowser: - return self._browser - - @property - def workspace(self) -> Path: - return self._workspace - - def redirect_url(self, url: str) -> str: - url = removeprefix(url, str(self._workspace)) - url = removeprefix(url, "/") - address = removesuffix(self._browser.address(), "/") - return removesuffix(address + "/" + url, "/") - - def matches(self, path: str) -> bool: - return path.startswith(str(self.workspace)) - - -class RedirectMap: - def __init__(self) -> None: - self._redirects: dict[str, set[BrowserRedirect]] = {} - - def add( - self, session_id: str, workspace: Path, server: OcrdBrowser - ) -> BrowserRedirect: - try: - redirect = self.get(session_id, workspace) - return redirect - except KeyError: - redirect = BrowserRedirect(workspace, server) - self._redirects.setdefault(session_id, set()).add(redirect) - return redirect - - def remove(self, session_id: str, workspace: Path) -> None: - redirect = self.get(session_id, workspace) - self._redirects[session_id].remove(redirect) - - def get(self, session_id: str, workspace: Path) -> BrowserRedirect: - redirect = next( - ( - redirect - for redirect in self._redirects.get(session_id, set()) - if redirect.matches(str(workspace)) - ), - None, - ) - - return self._instance_or_raise(redirect) - - def _instance_or_raise(self, redirect: BrowserRedirect | None) -> BrowserRedirect: - if redirect is None: - raise KeyError("No redirect found") - - return redirect - - def has_redirect_to_workspace(self, session_id: str, workspace: Path) -> bool: - try: - self.get(session_id, workspace) - return True - except KeyError: - return False - - def __contains__(self, session_and_workspace: tuple[str, Path]) -> bool: - session_id, workspace = session_and_workspace - return self.has_redirect_to_workspace(session_id, workspace) diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index 6b6566e..08bdde0 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -17,7 +17,6 @@ from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.server import proxy -from ocrdmonitor.server.redirect import BrowserRedirect from ocrdmonitor.server.settings import OcrdBrowserSettings @@ -73,7 +72,7 @@ def open_workspace(request: Request, workspace: str) -> Response: @router.get("/ping/{workspace:path}", name="workspaces.ping") async def ping_workspace( workspace: Path, - session_id: str = Cookie(default=None), + session_id: str = Cookie(), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: browsers = list( @@ -81,9 +80,10 @@ async def ping_workspace( owner=session_id, workspace=str(WORKSPACE_DIR / workspace) ) ) + try: - redirect = BrowserRedirect(workspace, browsers[0]) - await proxy.forward(redirect, str(workspace)) + browser = browsers.pop() + await proxy.forward(browser, str(workspace)) return Response(status_code=200) except (ConnectionError, IndexError): return Response(status_code=502) @@ -95,19 +95,21 @@ async def ping_workspace( async def workspace_reverse_proxy( request: Request, workspace: Path, - session_id: str = Cookie(default=None), + session_id: str = Cookie(), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: - browsers = list( - await repository.find( - owner=session_id, workspace=str(WORKSPACE_DIR / workspace) - ) - ) - redirect = BrowserRedirect(workspace, browsers[0]) + requested_path = str(WORKSPACE_DIR / workspace) + browsers = [ + b + for b in await repository.find(owner=session_id) + if requested_path.startswith(b.workspace()) + ] + + browser = browsers.pop() try: - return await proxy.forward(redirect, str(workspace)) + return await proxy.forward(browser, str(workspace)) except ConnectionError: - await stop_browser(repository, redirect.browser) + await stop_browser(repository, browser) return templates.TemplateResponse( "view_workspace_failed.html.j2", {"request": request, "workspace": workspace}, @@ -117,7 +119,7 @@ async def workspace_reverse_proxy( async def workspace_socket_proxy( websocket: WebSocket, workspace: Path, - session_id: str = Cookie(default=None), + session_id: str = Cookie(), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> None: browsers = list( @@ -129,11 +131,9 @@ async def workspace_socket_proxy( if not browsers: await websocket.close(reason="No browser found") - redirect = BrowserRedirect(workspace, browsers[0]) + browser = browsers.pop() await websocket.accept(subprotocol="broadway") - await communicate_with_browser_until_closed( - repository, websocket, redirect.browser - ) + await communicate_with_browser_until_closed(repository, websocket, browser) async def communicate_with_browser_until_closed( repository: BrowserProcessRepository, websocket: WebSocket, browser: OcrdBrowser diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index 063a2d4..3544b18 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -28,6 +28,7 @@ async def mongodb_repository( restoring_factory: BrowserRestoringFactory, ) -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: with MongoDbContainer() as container: + print(container.get_connection_url()) await dbmodel.init(container.get_connection_url()) yield dbmodel.MongoBrowserProcessRepository(restoring_factory) diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 58be863..99e4f33 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -166,6 +166,29 @@ def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_t assert_is_browser_response(actual) +@pytest.mark.asyncio +@pytest.mark.usefixtures("iterating_factory") +@use_custom_repository +async def test__when_requesting_resource__returns_resource_from_workspace( + repository: RepositoryInitializer, +) -> None: + factory = SingletonRestoringBrowserFactory() + factory.browser.configure_client(response_factory=lambda path: path.encode()) + + workspace = "a_workspace" + resource = "/some_resource" + resource_in_workspace = workspace + "/some_resource" + + async with repository(factory) as repo: + async with patch_repository(repo): + app = TestClient(create_app(create_settings())) + open_workspace(app, workspace) + + actual = view_workspace(app, resource_in_workspace) + + assert actual.content == resource.encode() + + @pytest.mark.usefixtures("iterating_factory") def test__browsed_workspace_is_ready__when_pinging__returns_ok( app: TestClient, diff --git a/tests/ocrdmonitor/test_redirect.py b/tests/ocrdmonitor/test_redirect.py deleted file mode 100644 index a5abe07..0000000 --- a/tests/ocrdmonitor/test_redirect.py +++ /dev/null @@ -1,73 +0,0 @@ -from pathlib import Path - -import pytest -from ocrdmonitor.server.redirect import BrowserRedirect - -from tests.testdoubles._browserspy import BrowserSpy - - -def server_stub(address: str) -> BrowserSpy: - return BrowserSpy(address=address) - - -SERVER_ADDRESS = "http://example.com:8080" - - -def test__redirect_for_empty_url_returns_server_address() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("http://example.com:8080") - sut = BrowserRedirect(workspace, browser) - - assert sut.redirect_url("") == browser.address() - - -@pytest.mark.parametrize("address", [SERVER_ADDRESS, SERVER_ADDRESS + "/"]) -@pytest.mark.parametrize("filename", ["file.js", "/file.js"]) -def test__redirect_to_file_in_workspace__returns_server_slash_file( - address: str, - filename: str, -) -> None: - workspace = Path("path/to/workspace") - browser = server_stub(address) - sut = BrowserRedirect(workspace, browser) - - assert sut.redirect_url(filename) == url(address, filename) - - -def test__redirect_from_workspace__returns_server_address() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("http://example.com:8080") - sut = BrowserRedirect(workspace, browser) - - assert sut.redirect_url(str(workspace)) == browser.address() - - -def test__redirect_with_workspace__is_a_match() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("") - sut = BrowserRedirect(workspace, browser) - - assert sut.matches(str(workspace)) is True - - -def test__an_empty_path__does_not_match() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("") - sut = BrowserRedirect(workspace, browser) - - assert sut.matches("") is False - - -def test__a_path_starting_with_workspace__is_a_match() -> None: - workspace = Path("path/to/workspace") - browser = server_stub("") - sut = BrowserRedirect(workspace, browser) - - sub_path = workspace / "sub" / "path" / "file.txt" - assert sut.matches(str(sub_path)) is True - - -def url(server_address: str, subpath: str) -> str: - server_address = server_address.removesuffix("/") - subpath = subpath.removeprefix("/") - return server_address + "/" + subpath diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index 9daab1f..b17393f 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -2,7 +2,7 @@ from contextlib import asynccontextmanager from textwrap import dedent -from typing import AsyncGenerator, Type +from typing import AsyncGenerator, Callable, Type from ocrdbrowser import Channel, OcrdBrowserClient @@ -28,12 +28,19 @@ async def receive_bytes(self) -> bytes: class BrowserClientStub: def __init__( - self, response: bytes | Type[Exception] = b"", channel: Channel | None = None + self, + response: bytes | Type[Exception] = b"", + channel: Channel | None = None, + response_factory: Callable[[str], bytes] | None = None, ) -> None: self.channel = channel or ChannelDummy() self.response = response or html_template.encode() + self.response_factory = response_factory async def get(self, resource: str) -> bytes: + if self.response_factory is not None: + return self.response_factory(resource) + if not isinstance(self.response, bytes): raise self.response @@ -61,9 +68,12 @@ def __init__( self._client = BrowserClientStub() def configure_client( - self, response: bytes | Type[Exception] = b"", channel: Channel | None = None + self, + response: bytes | Type[Exception] = b"", + channel: Channel | None = None, + response_factory: Callable[[str], bytes] | None = None, ) -> None: - self._client = BrowserClientStub(response, channel) + self._client = BrowserClientStub(response, channel, response_factory) def set_owner_and_workspace(self, owner: str, workspace: str) -> None: self.owner_name = owner From 9124bf38d7584777223792436fede1486185f96c Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Wed, 7 Jun 2023 12:46:35 +0200 Subject: [PATCH 15/32] Working subprocess browsers --- docker-compose.yml | 23 ++++++++++--------- ocrdbrowser/_subprocess.py | 5 ++-- .../ocrdmonitor/server/fixtures/repository.py | 1 - .../testdoubles/_browserprocessrepository.py | 2 -- 4 files changed, 14 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 951ad3b..d055c3c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -26,20 +26,21 @@ services: - ${MANAGER_KEY}:/id_rsa - shared:/run/lock/ocrd.jobs - # ocrd-logview: - # image: amir20/dozzle:latest - # volumes: - # # double slash is mandatory to support windows - # - //var/run/docker.sock:/var/run/docker.sock - # ports: - # - ${MONITOR_PORT_LOG}:8080 - # environment: - # - DOZZLE_FILTER=name=ocrd_kitodo - # # DOZZLE_USERNAME= - # # DOZZLE_PASSWORD= + ocrd-logview: + image: amir20/dozzle:latest + volumes: + # double slash is mandatory to support windows + - //var/run/docker.sock:/var/run/docker.sock + ports: + - ${MONITOR_PORT_LOG}:8080 + environment: + - DOZZLE_FILTER=name=ocrd_kitodo + # DOZZLE_USERNAME= + # DOZZLE_PASSWORD= ocrd-database: image: "mongo:latest" + container_name: ocrd-database environment: MONGO_INITDB_ROOT_USERNAME: ${DB_ROOT_USER:-root} diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 40ad62b..aaa23fc 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -47,12 +47,11 @@ def __init__(self, available_ports: set[int]) -> None: async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: port = Port(self._available_ports).get() - address = f"https://localhost:{port}" + address = f"http://localhost:{port}" process = await self.start_browser(workspace_path, port) browser = SubProcessOcrdBrowser( owner, workspace_path, address, str(process.pid) ) - print("Launched Process ID", process.pid) return browser async def start_browser( @@ -85,5 +84,5 @@ async def start_browser( env=environment, ) except Exception as err: - logging.error(f"Tried to launch broadway at {displayport} (real port {port})") + logging.error(f"Failed to launch broadway at {displayport} (real port {port})") raise err \ No newline at end of file diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index 3544b18..063a2d4 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -28,7 +28,6 @@ async def mongodb_repository( restoring_factory: BrowserRestoringFactory, ) -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: with MongoDbContainer() as container: - print(container.get_connection_url()) await dbmodel.init(container.get_connection_url()) yield dbmodel.MongoBrowserProcessRepository(restoring_factory) diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index ade99b0..309d198 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -28,7 +28,6 @@ async def insert(self, browser: OcrdBrowser) -> None: browser.process_id(), ) - print(f"We're adding {entry}") self._processes.append(entry) async def delete(self, browser: OcrdBrowser) -> None: @@ -39,7 +38,6 @@ async def delete(self, browser: OcrdBrowser) -> None: browser.process_id(), ) - print(f"We're deleting {entry}") self._processes.remove(entry) async def find( From fe176f164061f12cfa9fe4f586a329fda55290db Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Mon, 12 Jun 2023 17:37:17 +0200 Subject: [PATCH 16/32] Try ports instead of taking from set --- Makefile | 5 ++ docker-browse-ocrd/Dockerfile | 17 +++++ docker-browse-ocrd/init.sh | 5 ++ ocrdbrowser/_docker.py | 47 +++++++----- ocrdbrowser/_subprocess.py | 30 +++++--- ocrdmonitor/server/settings.py | 29 +++---- tests/ocrdbrowser/test_browser_launch.py | 97 ++++++++++++++++++++++++ tests/ocrdmonitor/server/decorators.py | 2 +- tests/workspaces/a_workspace/mets.xml | 25 ++++++ 9 files changed, 214 insertions(+), 43 deletions(-) create mode 100644 docker-browse-ocrd/Dockerfile create mode 100644 docker-browse-ocrd/init.sh create mode 100644 tests/ocrdbrowser/test_browser_launch.py diff --git a/Makefile b/Makefile index f5651c7..44dd29f 100644 --- a/Makefile +++ b/Makefile @@ -10,6 +10,11 @@ build: pull: docker pull $(TAGNAME) + +build-browse-ocrd-docker: + docker build -t ocrd-browser:latest -f docker-browse-ocrd/Dockerfile docker-browse-ocrd + + define HELP cat <<"EOF" Targets: diff --git a/docker-browse-ocrd/Dockerfile b/docker-browse-ocrd/Dockerfile new file mode 100644 index 0000000..82ad71a --- /dev/null +++ b/docker-browse-ocrd/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.7 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends libcairo2-dev libgtk-3-bin libgtk-3-dev libglib2.0-dev libgtksourceview-3.0-dev libgirepository1.0-dev gir1.2-webkit2-4.0 pkg-config cmake \ + && pip3 install -U setuptools \ + && pip3 install browse-ocrd + +ENV GDK_BACKEND broadway +ENV BROADWAY_DISPLAY :5 + +EXPOSE 8085 + +COPY init.sh /init.sh + +RUN chmod +x /init.sh + +CMD ["/init.sh"] \ No newline at end of file diff --git a/docker-browse-ocrd/init.sh b/docker-browse-ocrd/init.sh new file mode 100644 index 0000000..2c2e572 --- /dev/null +++ b/docker-browse-ocrd/init.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -x +nohup broadwayd :5 & +browse-ocrd /data/mets.xml \ No newline at end of file diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index f9a8182..c3ab5e7 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -6,7 +6,7 @@ from typing import Any from ._browser import OcrdBrowser, OcrdBrowserClient -from ._port import Port +from ._port import NoPortsAvailableError from ._client import HttpBrowserClient _docker_run = "docker run --rm -d --name {} -v {}:/data -p {}:8085 ocrd-browser:latest" @@ -19,6 +19,7 @@ async def _run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process: return await asyncio.create_subprocess_shell( command, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) @@ -68,33 +69,43 @@ def __init__(self, host: str, available_ports: set[int]) -> None: async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: abs_workspace = path.abspath(workspace_path) - port = Port(self._ports) - cmd = await _run_command( - _docker_run, - _container_name(owner, abs_workspace), - abs_workspace, - port.get(), - ) + container: DockerOcrdBrowser | None = None + for port in self._ports: + cmd = await _run_command( + _docker_run, + _container_name(owner, abs_workspace), + abs_workspace, + port, + ) + + return_code = await cmd.wait() + if return_code != 0: + continue + + container = DockerOcrdBrowser( + owner, + abs_workspace, + f"{self._host}:{port}", + await self._read_container_id(cmd), + ) + self._containers.append(container) + return container + + raise NoPortsAvailableError() + async def _read_container_id(self, cmd: asyncio.subprocess.Process) -> str: stdout = cmd.stdout container_id = "" if stdout: container_id = str(await stdout.read()).strip() - container = DockerOcrdBrowser( - owner, - abs_workspace, - f"{self._host}:{port.get()}", - container_id, - ) - - self._containers.append(container) - return container + return container_id async def stop_all(self) -> None: running_ids = [c.process_id() for c in self._containers] if running_ids: - await _run_command(_docker_kill, " ".join(running_ids)) + cmd = await _run_command(_docker_kill, " ".join(running_ids)) + await cmd.wait() self._containers = [] diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index aaa23fc..cadd49a 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -8,7 +8,7 @@ from ._browser import OcrdBrowser, OcrdBrowserClient from ._client import HttpBrowserClient -from ._port import Port +from ._port import NoPortsAvailableError BROADWAY_BASE_PORT = 8080 @@ -46,13 +46,19 @@ def __init__(self, available_ports: set[int]) -> None: self._available_ports = available_ports async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - port = Port(self._available_ports).get() - address = f"http://localhost:{port}" - process = await self.start_browser(workspace_path, port) - browser = SubProcessOcrdBrowser( - owner, workspace_path, address, str(process.pid) - ) - return browser + for port in self._available_ports: + address = f"http://localhost:{port}" + process = await self.start_browser(workspace_path, port) + + await asyncio.sleep(1) + if process.returncode is None: + return SubProcessOcrdBrowser( + owner, workspace_path, address, str(process.pid) + ) + else: + continue + + raise NoPortsAvailableError() async def start_browser( self, workspace: str, port: int @@ -82,7 +88,11 @@ async def start_browser( ] ), env=environment, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) except Exception as err: - logging.error(f"Failed to launch broadway at {displayport} (real port {port})") - raise err \ No newline at end of file + logging.error( + f"Failed to launch broadway at {displayport} (real port {port})" + ) + raise err diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index 8476a4c..facac35 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -2,8 +2,9 @@ import asyncio import atexit +import functools from pathlib import Path -from typing import Literal +from typing import Callable, Literal, Type from pydantic import BaseModel, BaseSettings, validator @@ -20,6 +21,17 @@ from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.sshremote import SSHRemote +BrowserType = Type[SubProcessOcrdBrowser] | Type[DockerOcrdBrowser] +CreatingFactories: dict[str, Callable[[set[int]], OcrdBrowserFactory]] = { + "native": SubProcessOcrdBrowserFactory, + "docker": functools.partial(DockerOcrdBrowserFactory, "http://localhost"), +} + +RestoringFactories: dict[str, BrowserType] = { + "native": SubProcessOcrdBrowser, + "docker": DockerOcrdBrowser, +} + class OcrdControllerSettings(BaseModel): job_dir: Path @@ -44,23 +56,12 @@ class OcrdBrowserSettings(BaseModel): async def repository(self) -> BrowserProcessRepository: await dbmodel.init(self.db_connection_string) - restore: BrowserRestoringFactory = ( - SubProcessOcrdBrowser if self.mode == "native" else DockerOcrdBrowser - ) + restore = RestoringFactories[self.mode] return dbmodel.MongoBrowserProcessRepository(restore) def factory(self) -> OcrdBrowserFactory: port_range_set = set(range(*self.port_range)) - if self.mode == "native": - return SubProcessOcrdBrowserFactory(port_range_set) - else: - factory = DockerOcrdBrowserFactory("http://localhost", port_range_set) - - @atexit.register - def stop_containers() -> None: - asyncio.get_event_loop().run_until_complete(factory.stop_all()) - - return factory + return CreatingFactories[self.mode](port_range_set) @validator("port_range", pre=True) def validator(cls, value: str | tuple[int, int]) -> tuple[int, int]: diff --git a/tests/ocrdbrowser/test_browser_launch.py b/tests/ocrdbrowser/test_browser_launch.py new file mode 100644 index 0000000..26517e2 --- /dev/null +++ b/tests/ocrdbrowser/test_browser_launch.py @@ -0,0 +1,97 @@ +import asyncio +import functools +from typing import AsyncIterator, Callable + +import pytest +import pytest_asyncio + +from ocrdbrowser import ( + DockerOcrdBrowserFactory, + NoPortsAvailableError, + OcrdBrowserFactory, + SubProcessOcrdBrowserFactory, +) +from tests.ocrdmonitor.server.decorators import compose + +# NOTE: We are using different ports in each test case, because I think that tests are executed +# faster than docker is able to free the ports again + + +def browse_ocrd_not_available() -> bool: + import shutil + + browse_ocrd = shutil.which("browse-ocrd") + broadway = shutil.which("broadwayd") + return not all((browse_ocrd, broadway)) + + +def docker_not_available() -> bool: + import shutil + + return not bool(shutil.which("docker")) + + +create_docker_browser_factory = functools.partial( + DockerOcrdBrowserFactory, "http://localhost" +) + +browser_factory_test = compose( + pytest.mark.asyncio, + pytest.mark.parametrize( + "create_browser_factory", + ( + pytest.param( + create_docker_browser_factory, + marks=( + pytest.mark.needs_docker, + pytest.mark.skipif( + docker_not_available(), + reason="Skipping because Docker is not available", + ), + ), + ), + pytest.param( + SubProcessOcrdBrowserFactory, + marks=pytest.mark.skipif( + browse_ocrd_not_available(), + reason="Skipping because browse-ocrd or broadwayd are not available", + ), + ), + ), + ), +) + + +@pytest_asyncio.fixture(autouse=True) +async def stop_browsers() -> AsyncIterator[None]: + yield + + cmd = await asyncio.create_subprocess_shell( + "docker stop $(docker ps | grep ocrd-browser | awk '{ print $1 }')" + ) + await cmd.wait() + + +@browser_factory_test +async def test__launching_on_an_allocated_port__raises_unavailable_port_error( + create_browser_factory: Callable[[set[int]], OcrdBrowserFactory] +) -> None: + _factory = create_browser_factory({9000}) + first = await _factory("first-owner", "tests/workspaces/a_workspace") + + sut = create_browser_factory({9000}) + with pytest.raises(NoPortsAvailableError): + second = await sut("second-owner", "tests/workspaces/a_workspace") + + +@browser_factory_test +async def test__one_port_allocated__launches_on_next_available( + create_browser_factory: Callable[[set[int]], OcrdBrowserFactory] +) -> None: + _factory = create_browser_factory({9000}) + await _factory("other-owner", "tests/workspaces/a_workspace") + + sut = create_browser_factory({9000, 9001}) + browser = await sut("second-other-owner", "tests/workspaces/a_workspace") + + assert browser.address() == "http://localhost:9001" diff --git a/tests/ocrdmonitor/server/decorators.py b/tests/ocrdmonitor/server/decorators.py index a0d54e3..0efc7e4 100644 --- a/tests/ocrdmonitor/server/decorators.py +++ b/tests/ocrdmonitor/server/decorators.py @@ -1,4 +1,4 @@ -from typing import Any, Awaitable, Callable, Coroutine, ParamSpec +from typing import Any, Callable import pytest from tests.ocrdmonitor.server.fixtures.repository import ( diff --git a/tests/workspaces/a_workspace/mets.xml b/tests/workspaces/a_workspace/mets.xml index e69de29..82bb527 100644 --- a/tests/workspaces/a_workspace/mets.xml +++ b/tests/workspaces/a_workspace/mets.xml @@ -0,0 +1,25 @@ + + + + + Your Organization Name + + UniqueIdentifier123 + + + + + + + + \ No newline at end of file From 59ae366db8870885383da95b4d1f5aaedf0dcaf2 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Mon, 12 Jun 2023 17:38:02 +0200 Subject: [PATCH 17/32] Mark launch tests as integration --- tests/ocrdbrowser/test_browser_launch.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/ocrdbrowser/test_browser_launch.py b/tests/ocrdbrowser/test_browser_launch.py index 26517e2..359d4a1 100644 --- a/tests/ocrdbrowser/test_browser_launch.py +++ b/tests/ocrdbrowser/test_browser_launch.py @@ -37,6 +37,7 @@ def docker_not_available() -> bool: browser_factory_test = compose( pytest.mark.asyncio, + pytest.mark.integration, pytest.mark.parametrize( "create_browser_factory", ( From f7cdf42ff00a0f3544cb93d05627fb2a7fe75c63 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 13 Jun 2023 10:43:46 +0200 Subject: [PATCH 18/32] Add simple test to ping browser. Remove unused Port class --- ocrdbrowser/_docker.py | 15 +++++++++- ocrdbrowser/_port.py | 27 ----------------- tests/decorators.py | 14 +++++++++ tests/ocrdbrowser/test_browser_launch.py | 37 +++++++++++++++--------- tests/ocrdmonitor/server/decorators.py | 14 +-------- 5 files changed, 53 insertions(+), 54 deletions(-) create mode 100644 tests/decorators.py diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index c3ab5e7..eda4631 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -61,6 +61,13 @@ def _container_name(owner: str, workspace: str) -> str: return f"ocrd-browser-{owner}-{workspace}" +async def log_from_stream(stream: asyncio.StreamReader | None) -> None: + if not stream: + return + + logging.info(await stream.read()) + + class DockerOcrdBrowserFactory: def __init__(self, host: str, available_ports: set[int]) -> None: self._host = host @@ -79,7 +86,7 @@ async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: port, ) - return_code = await cmd.wait() + return_code = await self.wait_for(cmd) if return_code != 0: continue @@ -94,6 +101,12 @@ async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: raise NoPortsAvailableError() + async def wait_for(self, cmd: asyncio.subprocess.Process) -> int: + return_code = await cmd.wait() + await log_from_stream(cmd.stderr) + + return return_code + async def _read_container_id(self, cmd: asyncio.subprocess.Process) -> str: stdout = cmd.stdout container_id = "" diff --git a/ocrdbrowser/_port.py b/ocrdbrowser/_port.py index 02938ac..01781a6 100644 --- a/ocrdbrowser/_port.py +++ b/ocrdbrowser/_port.py @@ -1,32 +1,5 @@ from __future__ import annotations -from typing import Optional, Set - class NoPortsAvailableError(RuntimeError): pass - - -class Port: - def __init__(self, available_ports: Set[int]) -> None: - self._available_ports = available_ports - self._port: Optional[int] = self._try_pop() - - def get(self) -> int: - return self._port or self._try_pop() - - def release(self) -> None: - if not self._port: - return - self._available_ports.add(self._port) - self._port = None - - def _try_pop(self) -> int: - # FIXME: check if port is still free - try: - return self._available_ports.pop() - except KeyError as err: - raise NoPortsAvailableError() from err - - def __str__(self) -> str: - return str(self._port) diff --git a/tests/decorators.py b/tests/decorators.py new file mode 100644 index 0000000..ecdc42d --- /dev/null +++ b/tests/decorators.py @@ -0,0 +1,14 @@ +from typing import Any, Callable + + +Decorator = Callable[[Callable[..., Any]], Callable[..., Any]] + + +def compose(*decorators: Decorator) -> Decorator: + def decorated(fn: Callable[..., Any]) -> Callable[..., Any]: + for deco in reversed(decorators): + fn = deco(fn) + + return fn + + return decorated diff --git a/tests/ocrdbrowser/test_browser_launch.py b/tests/ocrdbrowser/test_browser_launch.py index 359d4a1..f50a9e3 100644 --- a/tests/ocrdbrowser/test_browser_launch.py +++ b/tests/ocrdbrowser/test_browser_launch.py @@ -1,5 +1,6 @@ import asyncio import functools +import shutil from typing import AsyncIterator, Callable import pytest @@ -11,23 +12,16 @@ OcrdBrowserFactory, SubProcessOcrdBrowserFactory, ) -from tests.ocrdmonitor.server.decorators import compose - -# NOTE: We are using different ports in each test case, because I think that tests are executed -# faster than docker is able to free the ports again +from tests.decorators import compose def browse_ocrd_not_available() -> bool: - import shutil - browse_ocrd = shutil.which("browse-ocrd") broadway = shutil.which("broadwayd") return not all((browse_ocrd, broadway)) def docker_not_available() -> bool: - import shutil - return not bool(shutil.which("docker")) @@ -68,26 +62,43 @@ async def stop_browsers() -> AsyncIterator[None]: yield cmd = await asyncio.create_subprocess_shell( - "docker stop $(docker ps | grep ocrd-browser | awk '{ print $1 }')" + "docker stop $(docker ps | grep ocrd-browser | awk '{ print $1 }')", + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, ) await cmd.wait() +CreateBrowserFactory = Callable[[set[int]], OcrdBrowserFactory] + + +@browser_factory_test +async def test__factory__launches_new_browser_instance( + create_browser_factory: CreateBrowserFactory, +) -> None: + sut = create_browser_factory({9000}) + browser = await sut("the-owner", "tests/workspaces/a_workspace") + + client = browser.client() + response = await client.get("/") + assert response is not None + + @browser_factory_test async def test__launching_on_an_allocated_port__raises_unavailable_port_error( - create_browser_factory: Callable[[set[int]], OcrdBrowserFactory] + create_browser_factory: CreateBrowserFactory, ) -> None: _factory = create_browser_factory({9000}) - first = await _factory("first-owner", "tests/workspaces/a_workspace") + await _factory("first-owner", "tests/workspaces/a_workspace") sut = create_browser_factory({9000}) with pytest.raises(NoPortsAvailableError): - second = await sut("second-owner", "tests/workspaces/a_workspace") + await sut("second-owner", "tests/workspaces/a_workspace") @browser_factory_test async def test__one_port_allocated__launches_on_next_available( - create_browser_factory: Callable[[set[int]], OcrdBrowserFactory] + create_browser_factory: CreateBrowserFactory, ) -> None: _factory = create_browser_factory({9000}) await _factory("other-owner", "tests/workspaces/a_workspace") diff --git a/tests/ocrdmonitor/server/decorators.py b/tests/ocrdmonitor/server/decorators.py index 0efc7e4..4ed289b 100644 --- a/tests/ocrdmonitor/server/decorators.py +++ b/tests/ocrdmonitor/server/decorators.py @@ -1,23 +1,11 @@ -from typing import Any, Callable import pytest +from tests.decorators import compose from tests.ocrdmonitor.server.fixtures.repository import ( inmemory_repository, mongodb_repository, ) -Decorator = Callable[[Callable[..., Any]], Callable[..., Any]] - - -def compose(*decorators: Decorator) -> Decorator: - def decorated(fn: Callable[..., Any]) -> Callable[..., Any]: - for deco in reversed(decorators): - fn = deco(fn) - - return fn - - return decorated - use_custom_repository = compose( pytest.mark.asyncio, From f16cff55b7221ee9d527d3d2eba179ad3ff3969e Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 13 Jun 2023 11:07:11 +0200 Subject: [PATCH 19/32] Build browse-ocrd docker image in CI --- .github/workflows/test-ci.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test-ci.yml b/.github/workflows/test-ci.yml index 2e36786..c5a760e 100644 --- a/.github/workflows/test-ci.yml +++ b/.github/workflows/test-ci.yml @@ -50,6 +50,9 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' + - name: Build browse-ocrd Docker image + run: make build-browse-ocrd-docker + - name: Install dependencies using pip run: pip install -e ".[dev]" From d1a0167452d6b7876489e7ed1b500e88964b1fc7 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 13 Jun 2023 12:17:00 +0200 Subject: [PATCH 20/32] Introduce first method for repository --- ocrdmonitor/browserprocess.py | 3 +++ ocrdmonitor/dbmodel.py | 14 ++++++++++- ocrdmonitor/server/workspaces.py | 25 ++++++++++--------- .../testdoubles/_browserprocessrepository.py | 4 +++ 4 files changed, 33 insertions(+), 13 deletions(-) diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py index 0a3e895..3796397 100644 --- a/ocrdmonitor/browserprocess.py +++ b/ocrdmonitor/browserprocess.py @@ -23,3 +23,6 @@ async def find( workspace: str | None = None, ) -> Collection[OcrdBrowser]: ... + + async def first(self, *, owner: str, workspace: str) -> OcrdBrowser | None: + ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index fa7de9d..c23c65d 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -11,7 +11,6 @@ from ocrdmonitor.browserprocess import BrowserRestoringFactory - class BrowserProcess(Document): address: str owner: str @@ -86,6 +85,19 @@ def find( else [] ) + async def first(self, owner: str, workspace: str) -> OcrdBrowser | None: + result = await BrowserProcess.find_one( + BrowserProcess.owner == owner, + BrowserProcess.workspace == workspace, + ) + + return self._restoring_factory( + result.owner, + result.workspace, + result.address, + result.process_id, + ) + async def clean(self) -> None: await BrowserProcess.delete_all() diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index 08bdde0..e35f4f9 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -75,14 +75,14 @@ async def ping_workspace( session_id: str = Cookie(), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: - browsers = list( - await repository.find( - owner=session_id, workspace=str(WORKSPACE_DIR / workspace) - ) + browser = await repository.first( + owner=session_id, workspace=str(WORKSPACE_DIR / workspace) ) + if not browser: + return Response(status_code=404) + try: - browser = browsers.pop() await proxy.forward(browser, str(workspace)) return Response(status_code=200) except (ConnectionError, IndexError): @@ -105,8 +105,8 @@ async def workspace_reverse_proxy( if requested_path.startswith(b.workspace()) ] - browser = browsers.pop() try: + browser = browsers.pop() return await proxy.forward(browser, str(workspace)) except ConnectionError: await stop_browser(repository, browser) @@ -114,6 +114,10 @@ async def workspace_reverse_proxy( "view_workspace_failed.html.j2", {"request": request, "workspace": workspace}, ) + except IndexError: + return Response( + content=f"No browser found for {workspace}", status_code=404 + ) @router.websocket("/view/{workspace:path}/socket", name="workspaces.view.socket") async def workspace_socket_proxy( @@ -122,16 +126,13 @@ async def workspace_socket_proxy( session_id: str = Cookie(), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> None: - browsers = list( - await repository.find( - owner=session_id, workspace=str(WORKSPACE_DIR / workspace) - ) + browser = await repository.first( + owner=session_id, workspace=str(WORKSPACE_DIR / workspace) ) - if not browsers: + if browser is None: await websocket.close(reason="No browser found") - browser = browsers.pop() await websocket.accept(subprotocol="broadway") await communicate_with_browser_until_closed(repository, websocket, browser) diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index 309d198..6ba6000 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -66,3 +66,7 @@ def match(browser: BrowserEntry) -> bool: for browser in self._processes if match(browser) ] + + async def first(self, *, owner: str, workspace: str) -> OcrdBrowser | None: + results = await self.find(owner=owner, workspace=workspace) + return next(iter(results), None) From fd4767b4cadd5bde760777ea3b3f39e8592d48f4 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 13 Jun 2023 12:24:21 +0200 Subject: [PATCH 21/32] Fix mypy failure --- ocrdmonitor/dbmodel.py | 3 +++ ocrdmonitor/server/workspaces.py | 1 + 2 files changed, 4 insertions(+) diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index c23c65d..84869e7 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -91,6 +91,9 @@ async def first(self, owner: str, workspace: str) -> OcrdBrowser | None: BrowserProcess.workspace == workspace, ) + if result is None: + return None + return self._restoring_factory( result.owner, result.workspace, diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py index e35f4f9..6583cde 100644 --- a/ocrdmonitor/server/workspaces.py +++ b/ocrdmonitor/server/workspaces.py @@ -132,6 +132,7 @@ async def workspace_socket_proxy( if browser is None: await websocket.close(reason="No browser found") + return await websocket.accept(subprotocol="broadway") await communicate_with_browser_until_closed(repository, websocket, browser) From 94a5966ef4e9d1769e044a50e49368b0ce26af66 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Wed, 14 Jun 2023 16:07:16 +0200 Subject: [PATCH 22/32] Clean unreachable browsers on startup. Refactor workspace endpoint --- ocrdbrowser/_client.py | 2 +- ocrdmonitor/browserprocess.py | 3 + ocrdmonitor/dbmodel.py | 86 ++++++--- ocrdmonitor/server/app.py | 3 +- ocrdmonitor/server/lifespan.py | 36 ++++ ocrdmonitor/server/proxy.py | 42 ----- ocrdmonitor/server/settings.py | 6 +- ocrdmonitor/server/workspaces.py | 164 ------------------ ocrdmonitor/server/workspaces/__init__.py | 29 ++++ .../workspaces/_browsercommunication.py | 60 +++++++ .../server/workspaces/_launchroutes.py | 48 +++++ ocrdmonitor/server/workspaces/_listroutes.py | 23 +++ ocrdmonitor/server/workspaces/_proxyroutes.py | 112 ++++++++++++ .../ocrdmonitor/server/fixtures/repository.py | 2 +- tests/ocrdmonitor/server/test_startup.py | 38 ++++ tests/testdoubles/_broadwayfake.py | 2 +- .../testdoubles/_browserprocessrepository.py | 3 + 17 files changed, 421 insertions(+), 238 deletions(-) create mode 100644 ocrdmonitor/server/lifespan.py delete mode 100644 ocrdmonitor/server/proxy.py delete mode 100644 ocrdmonitor/server/workspaces.py create mode 100644 ocrdmonitor/server/workspaces/__init__.py create mode 100644 ocrdmonitor/server/workspaces/_browsercommunication.py create mode 100644 ocrdmonitor/server/workspaces/_launchroutes.py create mode 100644 ocrdmonitor/server/workspaces/_listroutes.py create mode 100644 ocrdmonitor/server/workspaces/_proxyroutes.py create mode 100644 tests/ocrdmonitor/server/test_startup.py diff --git a/ocrdbrowser/_client.py b/ocrdbrowser/_client.py index 3feb37f..3e3a36f 100644 --- a/ocrdbrowser/_client.py +++ b/ocrdbrowser/_client.py @@ -75,7 +75,7 @@ async def get(self, resource: str) -> bytes: except Exception as ex: logging.error(f"Tried to connect to {self.address}") logging.error(f"Requested resource {resource}") - raise ex + raise ConnectionError from ex def open_channel(self) -> AsyncContextManager[Channel]: return WebSocketChannel(self.address + "/socket") diff --git a/ocrdmonitor/browserprocess.py b/ocrdmonitor/browserprocess.py index 3796397..15df446 100644 --- a/ocrdmonitor/browserprocess.py +++ b/ocrdmonitor/browserprocess.py @@ -26,3 +26,6 @@ async def find( async def first(self, *, owner: str, workspace: str) -> OcrdBrowser | None: ... + + async def count(self) -> int: + ... diff --git a/ocrdmonitor/dbmodel.py b/ocrdmonitor/dbmodel.py index 84869e7..8d2910c 100644 --- a/ocrdmonitor/dbmodel.py +++ b/ocrdmonitor/dbmodel.py @@ -1,8 +1,8 @@ import asyncio -from typing import Any, Collection, Mapping +import urllib +from typing import Any, Collection, Mapping, Protocol import pymongo -import urllib from beanie import Document, init_beanie from beanie.odm.queries.find import FindMany from motor.motor_asyncio import AsyncIOMotorClient @@ -41,12 +41,17 @@ async def insert(self, browser: OcrdBrowser) -> None: ).insert() async def delete(self, browser: OcrdBrowser) -> None: - await BrowserProcess.find_one( + result = await BrowserProcess.find_one( BrowserProcess.owner == browser.owner(), BrowserProcess.workspace == browser.workspace(), BrowserProcess.address == browser.address(), BrowserProcess.process_id == browser.process_id(), - ).delete() + ) + + if not result: + return + + await result.delete() async def find( self, @@ -71,19 +76,18 @@ def find( if workspace is not None: results = find(results, BrowserProcess.workspace == workspace) - return ( - [ - self._restoring_factory( - browser.owner, - browser.workspace, - browser.address, - browser.process_id, - ) - for browser in await results.to_list() - ] - if results - else [] - ) + if results is None: + results = BrowserProcess.find_all() + + return [ + self._restoring_factory( + browser.owner, + browser.workspace, + browser.address, + browser.process_id, + ) + for browser in await results.to_list() + ] async def first(self, owner: str, workspace: str) -> OcrdBrowser | None: result = await BrowserProcess.find_one( @@ -101,19 +105,51 @@ async def first(self, owner: str, workspace: str) -> OcrdBrowser | None: result.process_id, ) + async def count(self) -> int: + return await BrowserProcess.count() + async def clean(self) -> None: await BrowserProcess.delete_all() -async def init(connection_str: str) -> None: +def rebuild_connection_string(connection_str: str) -> str: connection_str = connection_str.removeprefix("mongodb://") credentials, host = connection_str.split("@") user, password = credentials.split(":") password = urllib.parse.quote(password) - connection_str = f"mongodb://{user}:{password}@{host}" - client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) - client.get_io_loop = asyncio.get_event_loop - await init_beanie( - database=client.browsers, - document_models=[BrowserProcess], # type: ignore - ) + return f"mongodb://{user}:{password}@{host}" + + +class InitDatabase(Protocol): + async def __call__( + self, connection_str: str, force_initialize: bool = False + ) -> None: + ... + + +def __beanie_initializer() -> InitDatabase: + """ + We use this as a workaround to prevent beanie from being initialized + multiple times when requesting the repository from OcrdBrowserSettings + unless stated explicitly (e.g. for testing purposes) + """ + __initialized = False + + async def init(connection_str: str, force_initialize: bool = False) -> None: + nonlocal __initialized + if __initialized and not force_initialize: + return + + __initialized = True + connection_str = rebuild_connection_string(connection_str) + client: AsyncIOMotorClient = AsyncIOMotorClient(connection_str) + client.get_io_loop = asyncio.get_event_loop + await init_beanie( + database=client.browsers, + document_models=[BrowserProcess], # type: ignore + ) + + return init + + +init = __beanie_initializer() diff --git a/ocrdmonitor/server/app.py b/ocrdmonitor/server/app.py index 439187d..ff8f6be 100644 --- a/ocrdmonitor/server/app.py +++ b/ocrdmonitor/server/app.py @@ -9,6 +9,7 @@ from ocrdmonitor.ocrdcontroller import OcrdController from ocrdmonitor.server.index import create_index from ocrdmonitor.server.jobs import create_jobs +from ocrdmonitor.server.lifespan import lifespan from ocrdmonitor.server.logs import create_logs from ocrdmonitor.server.logview import create_logview from ocrdmonitor.server.settings import Settings @@ -21,7 +22,7 @@ def create_app(settings: Settings) -> FastAPI: - app = FastAPI() + app = FastAPI(lifespan=lifespan(settings.ocrd_browser)) templates = Jinja2Templates(TEMPLATE_DIR) app.mount("/static", StaticFiles(directory=STATIC_DIR), name="static") diff --git a/ocrdmonitor/server/lifespan.py b/ocrdmonitor/server/lifespan.py new file mode 100644 index 0000000..9a3cfb6 --- /dev/null +++ b/ocrdmonitor/server/lifespan.py @@ -0,0 +1,36 @@ +import asyncio +from contextlib import asynccontextmanager +from typing import AsyncContextManager, AsyncIterator, Callable + +from fastapi import FastAPI + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.settings import OcrdBrowserSettings + + +Lifespan = Callable[[FastAPI], AsyncContextManager[None]] + + +def lifespan(browser_settings: OcrdBrowserSettings) -> Lifespan: + @asynccontextmanager + async def _lifespan(_: FastAPI) -> AsyncIterator[None]: + await clean_unreachable_browsers(browser_settings) + yield + + return _lifespan + + +async def clean_unreachable_browsers(browser_settings: OcrdBrowserSettings) -> None: + repo = await browser_settings.repository() + all_browsers = await repo.find() + async with asyncio.TaskGroup() as group: + for browser in all_browsers: + group.create_task(ping_or_delete(repo, browser)) + + +async def ping_or_delete(repo: BrowserProcessRepository, browser: OcrdBrowser) -> None: + try: + await browser.client().get("/") + except ConnectionError: + await repo.delete(browser) diff --git a/ocrdmonitor/server/proxy.py b/ocrdmonitor/server/proxy.py deleted file mode 100644 index a5020a7..0000000 --- a/ocrdmonitor/server/proxy.py +++ /dev/null @@ -1,42 +0,0 @@ -from __future__ import annotations - -import asyncio - -from fastapi import Response -from ocrdbrowser import OcrdBrowser, Channel -from difflib import SequenceMatcher - - -def get_redirect_url(browser: OcrdBrowser, url: str) -> str: - matcher = SequenceMatcher(None, browser.workspace(), url) - match = matcher.find_longest_match() - return url[match.size :] - - -async def forward(browser: OcrdBrowser, url: str) -> Response: - url = get_redirect_url(browser, url) - resource = await browser.client().get(url) - return Response(content=resource) - - -async def tunnel( - source: Channel, - target: Channel, - timeout: float = 0.001, -) -> None: - await _tunnel_one_way(source, target, timeout) - await _tunnel_one_way(target, source, timeout) - - -async def _tunnel_one_way( - source: Channel, - target: Channel, - timeout: float, -) -> None: - try: - source_data = await asyncio.wait_for(source.receive_bytes(), timeout) - await target.send_bytes(source_data) - except asyncio.exceptions.TimeoutError: - # a timeout is rather common if no data is being sent, - # so we are simply ignoring this exception - pass diff --git a/ocrdmonitor/server/settings.py b/ocrdmonitor/server/settings.py index facac35..ab617aa 100644 --- a/ocrdmonitor/server/settings.py +++ b/ocrdmonitor/server/settings.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio -import atexit import functools from pathlib import Path from typing import Callable, Literal, Type @@ -16,7 +14,7 @@ SubProcessOcrdBrowser, ) from ocrdmonitor import dbmodel -from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory +from ocrdmonitor.browserprocess import BrowserProcessRepository from ocrdmonitor.ocrdcontroller import RemoteServer from ocrdmonitor.sshremote import SSHRemote @@ -55,7 +53,9 @@ class OcrdBrowserSettings(BaseModel): db_connection_string: str async def repository(self) -> BrowserProcessRepository: + # if not self._repository_initialized: await dbmodel.init(self.db_connection_string) + restore = RestoringFactories[self.mode] return dbmodel.MongoBrowserProcessRepository(restore) diff --git a/ocrdmonitor/server/workspaces.py b/ocrdmonitor/server/workspaces.py deleted file mode 100644 index 6583cde..0000000 --- a/ocrdmonitor/server/workspaces.py +++ /dev/null @@ -1,164 +0,0 @@ -from __future__ import annotations - -import uuid -from pathlib import Path - -from fastapi import ( - APIRouter, - Cookie, - Depends, - Request, - Response, - WebSocket, - WebSocketDisconnect, -) -from fastapi.templating import Jinja2Templates - -from ocrdbrowser import ChannelClosed, OcrdBrowser, OcrdBrowserFactory, workspace -from ocrdmonitor.browserprocess import BrowserProcessRepository -from ocrdmonitor.server import proxy -from ocrdmonitor.server.settings import OcrdBrowserSettings - - -def create_workspaces( - templates: Jinja2Templates, - browser_settings: OcrdBrowserSettings, -) -> APIRouter: - router = APIRouter(prefix="/workspaces") - - WORKSPACE_DIR = browser_settings.workspace_dir - - @router.get("/", name="workspaces.list") - def list_workspaces(request: Request) -> Response: - spaces = [ - Path(space).relative_to(WORKSPACE_DIR) - for space in workspace.list_all(WORKSPACE_DIR) - ] - - return templates.TemplateResponse( - "list_workspaces.html.j2", - {"request": request, "workspaces": spaces}, - ) - - @router.get("/browse/{workspace:path}", name="workspaces.browse") - async def browser( - request: Request, - workspace: Path, - factory: OcrdBrowserFactory = Depends(browser_settings.factory), - repository: BrowserProcessRepository = Depends(browser_settings.repository), - ) -> Response: - session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) - response = Response() - response.set_cookie("session_id", session_id) - - full_workspace = str(WORKSPACE_DIR / workspace) - existing_browsers = await repository.find( - owner=session_id, workspace=full_workspace - ) - - if not existing_browsers: - browser = await launch_browser(factory, session_id, workspace) - await repository.insert(browser) - - return response - - @router.get("/open/{workspace:path}", name="workspaces.open") - def open_workspace(request: Request, workspace: str) -> Response: - return templates.TemplateResponse( - "workspace.html.j2", - {"request": request, "workspace": workspace}, - ) - - @router.get("/ping/{workspace:path}", name="workspaces.ping") - async def ping_workspace( - workspace: Path, - session_id: str = Cookie(), - repository: BrowserProcessRepository = Depends(browser_settings.repository), - ) -> Response: - browser = await repository.first( - owner=session_id, workspace=str(WORKSPACE_DIR / workspace) - ) - - if not browser: - return Response(status_code=404) - - try: - await proxy.forward(browser, str(workspace)) - return Response(status_code=200) - except (ConnectionError, IndexError): - return Response(status_code=502) - - # NOTE: It is important that the route path here ends with a slash, otherwise - # the reverse routing will not work as broadway.js uses window.location - # which points to the last component with a trailing slash. - @router.get("/view/{workspace:path}/", name="workspaces.view") - async def workspace_reverse_proxy( - request: Request, - workspace: Path, - session_id: str = Cookie(), - repository: BrowserProcessRepository = Depends(browser_settings.repository), - ) -> Response: - requested_path = str(WORKSPACE_DIR / workspace) - browsers = [ - b - for b in await repository.find(owner=session_id) - if requested_path.startswith(b.workspace()) - ] - - try: - browser = browsers.pop() - return await proxy.forward(browser, str(workspace)) - except ConnectionError: - await stop_browser(repository, browser) - return templates.TemplateResponse( - "view_workspace_failed.html.j2", - {"request": request, "workspace": workspace}, - ) - except IndexError: - return Response( - content=f"No browser found for {workspace}", status_code=404 - ) - - @router.websocket("/view/{workspace:path}/socket", name="workspaces.view.socket") - async def workspace_socket_proxy( - websocket: WebSocket, - workspace: Path, - session_id: str = Cookie(), - repository: BrowserProcessRepository = Depends(browser_settings.repository), - ) -> None: - browser = await repository.first( - owner=session_id, workspace=str(WORKSPACE_DIR / workspace) - ) - - if browser is None: - await websocket.close(reason="No browser found") - return - - await websocket.accept(subprotocol="broadway") - await communicate_with_browser_until_closed(repository, websocket, browser) - - async def communicate_with_browser_until_closed( - repository: BrowserProcessRepository, websocket: WebSocket, browser: OcrdBrowser - ) -> None: - async with browser.client().open_channel() as channel: - try: - while True: - await proxy.tunnel(channel, websocket) - except ChannelClosed: - await stop_browser(repository, browser) - except WebSocketDisconnect: - pass - - async def launch_browser( - factory: OcrdBrowserFactory, session_id: str, workspace: Path - ) -> OcrdBrowser: - full_workspace_path = WORKSPACE_DIR / workspace - return await factory(session_id, str(full_workspace_path)) - - async def stop_browser( - repository: BrowserProcessRepository, browser: OcrdBrowser - ) -> None: - await browser.stop() - await repository.delete(browser) - - return router diff --git a/ocrdmonitor/server/workspaces/__init__.py b/ocrdmonitor/server/workspaces/__init__.py new file mode 100644 index 0000000..9535bcc --- /dev/null +++ b/ocrdmonitor/server/workspaces/__init__.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +from pathlib import Path + +from fastapi import APIRouter +from fastapi.templating import Jinja2Templates + +from ocrdmonitor.server.settings import OcrdBrowserSettings + +from ._launchroutes import register_launchroutes +from ._listroutes import register_listroutes +from ._proxyroutes import register_proxyroutes + + +def create_workspaces( + templates: Jinja2Templates, browser_settings: OcrdBrowserSettings +) -> APIRouter: + router = APIRouter(prefix="/workspaces") + + WORKSPACE_DIR = browser_settings.workspace_dir + + def full_workspace(workspace: Path | str) -> str: + return str(WORKSPACE_DIR / workspace) + + register_listroutes(router, templates, browser_settings) + register_launchroutes(router, templates, browser_settings, full_workspace) + register_proxyroutes(router, templates, browser_settings, full_workspace) + + return router diff --git a/ocrdmonitor/server/workspaces/_browsercommunication.py b/ocrdmonitor/server/workspaces/_browsercommunication.py new file mode 100644 index 0000000..2b471e4 --- /dev/null +++ b/ocrdmonitor/server/workspaces/_browsercommunication.py @@ -0,0 +1,60 @@ +from __future__ import annotations + +import asyncio +from difflib import SequenceMatcher +from typing import Awaitable, Callable + +from fastapi import Response + +from ocrdbrowser import Channel, ChannelClosed, OcrdBrowser + + +async def forward(browser: OcrdBrowser, partial_workspace: str) -> Response: + url = _get_redirect_url(browser, partial_workspace) + resource = await browser.client().get(url) + return Response(content=resource) + + +def _get_redirect_url(browser: OcrdBrowser, partial_workspace: str) -> str: + matcher = SequenceMatcher(None, browser.workspace(), partial_workspace) + match = matcher.find_longest_match() + return partial_workspace[match.size :] + + +CloseCallback = Callable[[OcrdBrowser], Awaitable[None]] + + +async def communicate_until_closed( + websocket: Channel, browser: OcrdBrowser, close_callback: CloseCallback +) -> None: + async with browser.client().open_channel() as channel: + try: + while True: + await _tunnel(channel, websocket) + except ChannelClosed: + await close_callback(browser) + except Exception: + pass + + +async def _tunnel( + source: Channel, + target: Channel, + timeout: float = 0.001, +) -> None: + await _tunnel_one_way(source, target, timeout) + await _tunnel_one_way(target, source, timeout) + + +async def _tunnel_one_way( + source: Channel, + target: Channel, + timeout: float, +) -> None: + try: + source_data = await asyncio.wait_for(source.receive_bytes(), timeout) + await target.send_bytes(source_data) + except asyncio.exceptions.TimeoutError: + # a timeout is rather common if no data is being sent, + # so we are simply ignoring this exception + pass diff --git a/ocrdmonitor/server/workspaces/_launchroutes.py b/ocrdmonitor/server/workspaces/_launchroutes.py new file mode 100644 index 0000000..fcf13df --- /dev/null +++ b/ocrdmonitor/server/workspaces/_launchroutes.py @@ -0,0 +1,48 @@ +from typing import Callable +import uuid +from pathlib import Path + +from fastapi import APIRouter, Depends, Request, Response +from fastapi.templating import Jinja2Templates + +from ocrdbrowser import OcrdBrowserFactory +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.settings import OcrdBrowserSettings + + +def session_response(session_id: str) -> Response: + response = Response() + response.set_cookie("session_id", session_id) + return response + + +def register_launchroutes( + router: APIRouter, + templates: Jinja2Templates, + browser_settings: OcrdBrowserSettings, + full_workspace: Callable[[str | Path], str], +) -> None: + @router.get("/open/{workspace:path}", name="workspaces.open") + def open_workspace(request: Request, workspace: str) -> Response: + return templates.TemplateResponse( + "workspace.html.j2", + {"request": request, "workspace": workspace}, + ) + + @router.get("/browse/{workspace:path}", name="workspaces.browse") + async def browser( + request: Request, + workspace: Path, + factory: OcrdBrowserFactory = Depends(browser_settings.factory), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: + session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) + + full_path = full_workspace(workspace) + existing_browsers = await repository.find(owner=session_id, workspace=full_path) + + if not existing_browsers: + browser = await factory(session_id, full_path) + await repository.insert(browser) + + return session_response(session_id) diff --git a/ocrdmonitor/server/workspaces/_listroutes.py b/ocrdmonitor/server/workspaces/_listroutes.py new file mode 100644 index 0000000..b2dbd5e --- /dev/null +++ b/ocrdmonitor/server/workspaces/_listroutes.py @@ -0,0 +1,23 @@ +from pathlib import Path + +from fastapi import APIRouter, Request, Response +from fastapi.templating import Jinja2Templates + +from ocrdbrowser import workspace +from ocrdmonitor.server.settings import OcrdBrowserSettings + + +def register_listroutes( + router: APIRouter, templates: Jinja2Templates, browser_settings: OcrdBrowserSettings +) -> None: + @router.get("/", name="workspaces.list") + def list_workspaces(request: Request) -> Response: + spaces = [ + Path(space).relative_to(browser_settings.workspace_dir) + for space in workspace.list_all(browser_settings.workspace_dir) + ] + + return templates.TemplateResponse( + "list_workspaces.html.j2", + {"request": request, "workspaces": spaces}, + ) diff --git a/ocrdmonitor/server/workspaces/_proxyroutes.py b/ocrdmonitor/server/workspaces/_proxyroutes.py new file mode 100644 index 0000000..f036b7a --- /dev/null +++ b/ocrdmonitor/server/workspaces/_proxyroutes.py @@ -0,0 +1,112 @@ +from __future__ import annotations + +import functools +from pathlib import Path +from typing import Callable + +from fastapi import APIRouter, Cookie, Depends, Request, Response, WebSocket +from fastapi.templating import Jinja2Templates + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.settings import OcrdBrowserSettings + +from ._browsercommunication import CloseCallback, communicate_until_closed, forward + + +async def stop_and_remove_browser( + repository: BrowserProcessRepository, browser: OcrdBrowser +) -> None: + await browser.stop() + await repository.delete(browser) + + +async def first_owned_browser_in_workspace( + session_id: str, workspace: str, repository: BrowserProcessRepository +) -> OcrdBrowser | None: + def in_workspace(browser: OcrdBrowser) -> bool: + return workspace.startswith(browser.workspace()) + + browsers_in_workspace = filter( + in_workspace, await repository.find(owner=session_id) + ) + return next(browsers_in_workspace, None) + + +def browser_closed_callback(reposository: BrowserProcessRepository) -> CloseCallback: + return functools.partial(stop_and_remove_browser, reposository) + + +def register_proxyroutes( + router: APIRouter, + templates: Jinja2Templates, + browser_settings: OcrdBrowserSettings, + full_workspace: Callable[[str | Path], str], +) -> None: + @router.get("/ping/{workspace:path}", name="workspaces.ping") + async def ping_workspace( + workspace: Path, + session_id: str = Cookie(), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: + browser = await repository.first( + owner=session_id, workspace=full_workspace(workspace) + ) + + if not browser: + return Response(status_code=404) + + try: + await forward(browser, str(workspace)) + return Response(status_code=200) + except ConnectionError: + return Response(status_code=502) + + # NOTE: It is important that the route path here ends with a slash, otherwise + # the reverse routing will not work as broadway.js uses window.location + # which points to the last component with a trailing slash. + @router.get("/view/{workspace:path}/", name="workspaces.view") + async def workspace_reverse_proxy( + request: Request, + workspace: Path, + session_id: str = Cookie(), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> Response: + browser = await first_owned_browser_in_workspace( + session_id, full_workspace(workspace), repository + ) + + if not browser: + return Response( + content=f"No browser found for {workspace}", status_code=404 + ) + try: + return await forward(browser, str(workspace)) + except ConnectionError: + await stop_and_remove_browser(repository, browser) + return templates.TemplateResponse( + "view_workspace_failed.html.j2", + {"request": request, "workspace": workspace}, + ) + + @router.websocket("/view/{workspace:path}/socket", name="workspaces.view.socket") + async def workspace_socket_proxy( + websocket: WebSocket, + workspace: Path, + session_id: str = Cookie(), + repository: BrowserProcessRepository = Depends(browser_settings.repository), + ) -> None: + browser = await repository.first( + owner=session_id, workspace=full_workspace(workspace) + ) + + if browser is None: + await websocket.close(reason="No browser found") + return + + await websocket.accept(subprotocol="broadway") + await communicate_until_closed( + websocket, + browser, + close_callback=browser_closed_callback(repository), + ) diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index 063a2d4..d27fe50 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -28,7 +28,7 @@ async def mongodb_repository( restoring_factory: BrowserRestoringFactory, ) -> AsyncIterator[dbmodel.MongoBrowserProcessRepository]: with MongoDbContainer() as container: - await dbmodel.init(container.get_connection_url()) + await dbmodel.init(container.get_connection_url(), force_initialize=True) yield dbmodel.MongoBrowserProcessRepository(restoring_factory) diff --git a/tests/ocrdmonitor/server/test_startup.py b/tests/ocrdmonitor/server/test_startup.py new file mode 100644 index 0000000..746d410 --- /dev/null +++ b/tests/ocrdmonitor/server/test_startup.py @@ -0,0 +1,38 @@ +from fastapi.testclient import TestClient + +from ocrdbrowser import OcrdBrowser +from ocrdmonitor.server.app import create_app +from tests.ocrdmonitor.server.decorators import use_custom_repository +from tests.ocrdmonitor.server.fixtures.app import create_settings +from tests.ocrdmonitor.server.fixtures.repository import ( + RepositoryInitializer, + patch_repository, +) +from tests.testdoubles import BrowserSpy + + +@use_custom_repository +async def test__browsers_in_db__on_startup__cleans_unreachables_from_db( + repository: RepositoryInitializer, +) -> None: + reachable = BrowserSpy(address="http://reachable.com") + unreachable = BrowserSpy(address="http://unreachable.com") + unreachable.configure_client(response=ConnectionError) + + def factory( + owner: str, workspace: str, address: str, process_id: str + ) -> OcrdBrowser: + if "unreachable" in address: + return unreachable + else: + return reachable + + async with repository(factory) as repo: + async with patch_repository(repo): + await repo.insert(unreachable) + await repo.insert(reachable) + + with TestClient(create_app(create_settings())): + pass + + assert await repo.count() == 1 diff --git a/tests/testdoubles/_broadwayfake.py b/tests/testdoubles/_broadwayfake.py index 5d7b07a..9824abc 100644 --- a/tests/testdoubles/_broadwayfake.py +++ b/tests/testdoubles/_broadwayfake.py @@ -8,7 +8,7 @@ FAKE_HOST_IP = "127.0.0.1" -FAKE_HOST_PORT = 7000 +FAKE_HOST_PORT = 8000 FAKE_HOST_ADDRESS = f"{FAKE_HOST_IP}:{FAKE_HOST_PORT}" diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index 6ba6000..2f035ee 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -70,3 +70,6 @@ def match(browser: BrowserEntry) -> bool: async def first(self, *, owner: str, workspace: str) -> OcrdBrowser | None: results = await self.find(owner=owner, workspace=workspace) return next(iter(results), None) + + async def count(self) -> int: + return len(self._processes) From 546aecd773eb95576da0f11ec37f7004fca4a6c2 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Mon, 19 Jun 2023 15:49:32 +0200 Subject: [PATCH 23/32] Catch exception when killing browser fails --- ocrdbrowser/_subprocess.py | 5 ++++- .../server/workspaces/_browsercommunication.py | 10 ++++++++-- ocrdmonitor/server/workspaces/_proxyroutes.py | 15 ++++++++++----- 3 files changed, 22 insertions(+), 8 deletions(-) diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index cadd49a..fe2abba 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -35,7 +35,10 @@ def owner(self) -> str: return self._owner async def stop(self) -> None: - os.kill(int(self._process_id), signal.SIGKILL) + try: + os.kill(int(self._process_id), signal.SIGKILL) + except ProcessLookupError: + logging.warning(f"Could not find process with ID {self._process_id}") def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) diff --git a/ocrdmonitor/server/workspaces/_browsercommunication.py b/ocrdmonitor/server/workspaces/_browsercommunication.py index 2b471e4..13d66e8 100644 --- a/ocrdmonitor/server/workspaces/_browsercommunication.py +++ b/ocrdmonitor/server/workspaces/_browsercommunication.py @@ -1,6 +1,7 @@ from __future__ import annotations import asyncio +import logging from difflib import SequenceMatcher from typing import Awaitable, Callable @@ -33,8 +34,13 @@ async def communicate_until_closed( await _tunnel(channel, websocket) except ChannelClosed: await close_callback(browser) - except Exception: - pass + except Exception as err: + logging.error( + f""" + An exception occurred during communication with the browser {repr(browser)}. + The exception was {repr(err)} + """ + ) async def _tunnel( diff --git a/ocrdmonitor/server/workspaces/_proxyroutes.py b/ocrdmonitor/server/workspaces/_proxyroutes.py index f036b7a..bde7570 100644 --- a/ocrdmonitor/server/workspaces/_proxyroutes.py +++ b/ocrdmonitor/server/workspaces/_proxyroutes.py @@ -1,6 +1,7 @@ from __future__ import annotations +import asyncio +import logging -import functools from pathlib import Path from typing import Callable @@ -17,8 +18,10 @@ async def stop_and_remove_browser( repository: BrowserProcessRepository, browser: OcrdBrowser ) -> None: - await browser.stop() - await repository.delete(browser) + async with asyncio.TaskGroup() as group: + group.create_task(browser.stop()) + group.create_task(repository.delete(browser)) + logging.info(f"Stopping browser {browser.workspace()}") async def first_owned_browser_in_workspace( @@ -33,8 +36,10 @@ def in_workspace(browser: OcrdBrowser) -> bool: return next(browsers_in_workspace, None) -def browser_closed_callback(reposository: BrowserProcessRepository) -> CloseCallback: - return functools.partial(stop_and_remove_browser, reposository) +def browser_closed_callback(repository: BrowserProcessRepository) -> CloseCallback: + async def _callback(browser: OcrdBrowser) -> None: + await stop_and_remove_browser(repository, browser) + return _callback def register_proxyroutes( From 98d03ce2f068fbc6909e7dfb33da5697b147114b Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Wed, 21 Jun 2023 11:19:51 +0000 Subject: [PATCH 24/32] Pass SubProcessBrowser Tests. Refactor towards common port binding function. --- ocrdbrowser/_docker.py | 103 ++++++++++----------- ocrdbrowser/_port.py | 30 ++++++ ocrdbrowser/_subprocess.py | 111 +++++++++++++---------- tests/ocrdbrowser/test_browser_launch.py | 30 +++++- 4 files changed, 169 insertions(+), 105 deletions(-) diff --git a/ocrdbrowser/_docker.py b/ocrdbrowser/_docker.py index eda4631..96b2922 100644 --- a/ocrdbrowser/_docker.py +++ b/ocrdbrowser/_docker.py @@ -1,20 +1,21 @@ from __future__ import annotations import asyncio +import functools import logging import os.path as path from typing import Any from ._browser import OcrdBrowser, OcrdBrowserClient -from ._port import NoPortsAvailableError from ._client import HttpBrowserClient +from ._port import PortBindingError, PortBindingResult, try_bind _docker_run = "docker run --rm -d --name {} -v {}:/data -p {}:8085 ocrd-browser:latest" _docker_stop = "docker stop {}" _docker_kill = "docker kill {}" -async def _run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process: +async def run_command(cmd: str, *args: Any) -> asyncio.subprocess.Process: command = cmd.format(*args) return await asyncio.create_subprocess_shell( command, @@ -45,7 +46,7 @@ def owner(self) -> str: return self._owner async def stop(self) -> None: - cmd = await _run_command(_docker_stop, self._process_id) + cmd = await run_command(_docker_stop, self._process_id) if cmd.returncode != 0: logging.info( @@ -56,18 +57,6 @@ def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) -def _container_name(owner: str, workspace: str) -> str: - workspace = path.basename(workspace) - return f"ocrd-browser-{owner}-{workspace}" - - -async def log_from_stream(stream: asyncio.StreamReader | None) -> None: - if not stream: - return - - logging.info(await stream.read()) - - class DockerOcrdBrowserFactory: def __init__(self, host: str, available_ports: set[int]) -> None: self._host = host @@ -76,49 +65,61 @@ def __init__(self, host: str, available_ports: set[int]) -> None: async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: abs_workspace = path.abspath(workspace_path) + port_binding = functools.partial(start_browser, owner, abs_workspace) + container, _ = await try_bind(port_binding, self._host, self._ports) + self._containers.append(container) + return container - container: DockerOcrdBrowser | None = None - for port in self._ports: - cmd = await _run_command( - _docker_run, - _container_name(owner, abs_workspace), - abs_workspace, - port, - ) + async def stop_all(self) -> None: + running_ids = [c.process_id() for c in self._containers] + if running_ids: + cmd = await run_command(_docker_kill, " ".join(running_ids)) + await cmd.wait() - return_code = await self.wait_for(cmd) - if return_code != 0: - continue + self._containers = [] - container = DockerOcrdBrowser( - owner, - abs_workspace, - f"{self._host}:{port}", - await self._read_container_id(cmd), - ) - self._containers.append(container) - return container - raise NoPortsAvailableError() +async def start_browser( + owner: str, workspace: str, host: str, port: int +) -> PortBindingResult[DockerOcrdBrowser]: + cmd = await run_command( + _docker_run, container_name(owner, workspace), workspace, port + ) - async def wait_for(self, cmd: asyncio.subprocess.Process) -> int: - return_code = await cmd.wait() - await log_from_stream(cmd.stderr) + return_code = await wait_for(cmd) + if return_code != 0: + return PortBindingError() - return return_code + container = DockerOcrdBrowser( + owner, workspace, f"{host}:{port}", await read_container_id(cmd) + ) - async def _read_container_id(self, cmd: asyncio.subprocess.Process) -> str: - stdout = cmd.stdout - container_id = "" - if stdout: - container_id = str(await stdout.read()).strip() + return container - return container_id - async def stop_all(self) -> None: - running_ids = [c.process_id() for c in self._containers] - if running_ids: - cmd = await _run_command(_docker_kill, " ".join(running_ids)) - await cmd.wait() +def container_name(owner: str, workspace: str) -> str: + workspace = path.basename(workspace) + return f"ocrd-browser-{owner}-{workspace}" - self._containers = [] + +async def wait_for(cmd: asyncio.subprocess.Process) -> int: + return_code = await cmd.wait() + await log_from_stream(cmd.stderr) + + return return_code + + +async def read_container_id(cmd: asyncio.subprocess.Process) -> str: + stdout = cmd.stdout + container_id = "" + if stdout: + container_id = str(await stdout.read()).strip() + + return container_id + + +async def log_from_stream(stream: asyncio.StreamReader | None) -> None: + if not stream: + return + + logging.info(await stream.read()) diff --git a/ocrdbrowser/_port.py b/ocrdbrowser/_port.py index 01781a6..bd0464f 100644 --- a/ocrdbrowser/_port.py +++ b/ocrdbrowser/_port.py @@ -1,5 +1,35 @@ from __future__ import annotations +from typing import Awaitable, Callable, Generic, Iterable, NamedTuple, TypeVar, Union class NoPortsAvailableError(RuntimeError): pass + + +T = TypeVar("T") + + +class PortBindingError(RuntimeError): + pass + + +PortBindingResult = Union[T, PortBindingError] +PortBinding = Callable[[str, int], Awaitable[PortBindingResult[T]]] + + +class BoundPort(NamedTuple, Generic[T]): + bound_app: T + port: int + + +async def try_bind( + binding: PortBinding[T], host: str, ports: Iterable[int] +) -> BoundPort[T]: + for port in ports: + result = await binding(host, port) + if isinstance(result, PortBindingError): + continue + + return BoundPort(result, port) + + raise NoPortsAvailableError() diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index fe2abba..2681c26 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -1,14 +1,16 @@ from __future__ import annotations import asyncio +import functools import logging import os import signal from shutil import which +from typing import cast from ._browser import OcrdBrowser, OcrdBrowserClient from ._client import HttpBrowserClient -from ._port import NoPortsAvailableError +from ._port import PortBindingError, PortBindingResult, try_bind BROADWAY_BASE_PORT = 8080 @@ -44,58 +46,69 @@ def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) +class ProcessLaunchFailedError(RuntimeError): + pass + + class SubProcessOcrdBrowserFactory: def __init__(self, available_ports: set[int]) -> None: self._available_ports = available_ports async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - for port in self._available_ports: - address = f"http://localhost:{port}" - process = await self.start_browser(workspace_path, port) - - await asyncio.sleep(1) - if process.returncode is None: - return SubProcessOcrdBrowser( - owner, workspace_path, address, str(process.pid) - ) - else: - continue - - raise NoPortsAvailableError() - - async def start_browser( - self, workspace: str, port: int - ) -> asyncio.subprocess.Process: - browse_ocrd = which("browse-ocrd") - if not browse_ocrd: - raise FileNotFoundError("Could not find browse-ocrd executable") - - # broadwayd (which uses WebSockets) only allows a single client at a time - # (disconnecting concurrent connections), hence we must start a new daemon - # for each new browser session - # broadwayd starts counting virtual X displays from port 8080 as :0 - displayport = str(port - BROADWAY_BASE_PORT) - environment = dict(os.environ) - environment["GDK_BACKEND"] = "broadway" - environment["BROADWAY_DISPLAY"] = ":" + displayport + port_binding = functools.partial(start_browser, workspace_path) + process, port = await try_bind( + port_binding, "http://localhost", self._available_ports + ) + + address = f"http://localhost:{port}" + return SubProcessOcrdBrowser(owner, workspace_path, address, str(process.pid)) + + # stderr = cast(asyncio.StreamReader, process.stderr) + # logging.error((await stderr.read()).splitlines()) + # raise ProcessLaunchFailedError() + + +async def start_browser( + workspace: str, host: str, port: int +) -> PortBindingResult[asyncio.subprocess.Process]: + browse_ocrd = which("browse-ocrd") + if not browse_ocrd: + raise FileNotFoundError("Could not find browse-ocrd executable") + + # broadwayd (which uses WebSockets) only allows a single client at a time + # (disconnecting concurrent connections), hence we must start a new daemon + # for each new browser session + # broadwayd starts counting virtual X displays from port 8080 as :0 + displayport = str(port - BROADWAY_BASE_PORT) + environment = dict(os.environ) + environment["GDK_BACKEND"] = "broadway" + environment["BROADWAY_DISPLAY"] = ":" + displayport + + try: + process = await asyncio.create_subprocess_shell( + " ".join( + [ + "broadwayd", + ":" + displayport, + browse_ocrd, + workspace + "/mets.xml", + ] + ), + env=environment, + stderr=asyncio.subprocess.PIPE, + ) try: - return await asyncio.create_subprocess_shell( - " ".join( - [ - "broadwayd", - ":" + displayport + " &", - browse_ocrd, - workspace + "/mets.xml ;", - "kill $!", - ] - ), - env=environment, - stdout=asyncio.subprocess.PIPE, - stderr=asyncio.subprocess.PIPE, - ) - except Exception as err: - logging.error( - f"Failed to launch broadway at {displayport} (real port {port})" - ) - raise err + stderr = cast(asyncio.StreamReader, process.stderr) + line = await asyncio.wait_for(stderr.readline(), 1) + if b"Address already in use" in line: + return PortBindingError() + except TimeoutError: + # If we don't get a timeout, the process didn't crash to the best of our knowledge + pass + + return process + except Exception as err: + logging.error(f"Failed to launch broadway at {displayport} (real port {port})") + logging.error(repr(err)) + return PortBindingError() diff --git a/tests/ocrdbrowser/test_browser_launch.py b/tests/ocrdbrowser/test_browser_launch.py index f50a9e3..f0f2cc1 100644 --- a/tests/ocrdbrowser/test_browser_launch.py +++ b/tests/ocrdbrowser/test_browser_launch.py @@ -1,7 +1,7 @@ import asyncio import functools import shutil -from typing import AsyncIterator, Callable +from typing import AsyncIterator, Callable, Literal, NamedTuple import pytest import pytest_asyncio @@ -57,18 +57,38 @@ def docker_not_available() -> bool: ) -@pytest_asyncio.fixture(autouse=True) -async def stop_browsers() -> AsyncIterator[None]: - yield +class DockerProcessKiller(NamedTuple): + kill: str = "docker stop" + ps: str = "docker ps" + +class NativeProcessKiller(NamedTuple): + kill: str = "kill" + ps: str = "ps" + + +async def kill_processes( + killer: NativeProcessKiller | DockerProcessKiller, name_filter: str +) -> None: + kill_cmd, ps_cmd = killer cmd = await asyncio.create_subprocess_shell( - "docker stop $(docker ps | grep ocrd-browser | awk '{ print $1 }')", + f"{kill_cmd} $({ps_cmd} | grep {name_filter} | awk '{{ print $1 }}')", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, ) await cmd.wait() +@pytest_asyncio.fixture(autouse=True) +async def stop_browsers() -> AsyncIterator[None]: + yield + + async with asyncio.TaskGroup() as group: + group.create_task(kill_processes(DockerProcessKiller(), "ocrd-browser")) + group.create_task(kill_processes(NativeProcessKiller(), "broadwayd")) + group.create_task(kill_processes(NativeProcessKiller(), "browse-ocrd")) + + CreateBrowserFactory = Callable[[set[int]], OcrdBrowserFactory] From a8bf2b8eda04a0930246d816153283da23967e13 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Fri, 23 Jun 2023 11:15:19 +0000 Subject: [PATCH 25/32] Better, but still insufficient, handling of native browser launches --- docker-compose.yml | 3 +++ ocrdbrowser/_port.py | 2 ++ ocrdbrowser/_subprocess.py | 15 +++++++++------ ocrdmonitor/server/app.py | 19 +++++++++++++++++-- 4 files changed, 31 insertions(+), 8 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index d055c3c..cd42cf9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,9 @@ version: "3.9" services: ocrd-monitor: + depends_on: + - ocrd-database + build: context: . # args: diff --git a/ocrdbrowser/_port.py b/ocrdbrowser/_port.py index bd0464f..2e5abff 100644 --- a/ocrdbrowser/_port.py +++ b/ocrdbrowser/_port.py @@ -1,4 +1,5 @@ from __future__ import annotations +import logging from typing import Awaitable, Callable, Generic, Iterable, NamedTuple, TypeVar, Union @@ -28,6 +29,7 @@ async def try_bind( for port in ports: result = await binding(host, port) if isinstance(result, PortBindingError): + logging.info(f"Port {port} already in use, continuing to next port") continue return BoundPort(result, port) diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index 2681c26..a9984be 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -89,9 +89,10 @@ async def start_browser( " ".join( [ "broadwayd", - ":" + displayport, + ":" + displayport + " &", browse_ocrd, - workspace + "/mets.xml", + workspace + "/mets.xml" + " ;", + "kill" + " $!", ] ), env=environment, @@ -100,11 +101,13 @@ async def start_browser( try: stderr = cast(asyncio.StreamReader, process.stderr) - line = await asyncio.wait_for(stderr.readline(), 1) - if b"Address already in use" in line: + err_output = await asyncio.wait_for(stderr.readline(), 5) + if b"Address already in use" in err_output: return PortBindingError() - except TimeoutError: - # If we don't get a timeout, the process didn't crash to the best of our knowledge + except asyncio.TimeoutError: + logging.info( + f"The process didn't exit within the given timeout. Assuming browser on port {port} launched successfully" + ) pass return process diff --git a/ocrdmonitor/server/app.py b/ocrdmonitor/server/app.py index ff8f6be..98358f5 100644 --- a/ocrdmonitor/server/app.py +++ b/ocrdmonitor/server/app.py @@ -1,8 +1,10 @@ import logging from pathlib import Path -from fastapi import FastAPI, Request, Response -from fastapi.responses import RedirectResponse +from fastapi import FastAPI, Request, Response, status +from fastapi.encoders import jsonable_encoder +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse, RedirectResponse from fastapi.staticfiles import StaticFiles from fastapi.templating import Jinja2Templates @@ -31,6 +33,19 @@ async def swallow_exceptions(request: Request, err: Exception) -> Response: logging.error(err) return RedirectResponse("/") + @app.exception_handler(RequestValidationError) + async def validation_exception( + request: Request, exc: RequestValidationError + ) -> Response: + logging.error(f"Unprocessable entity on route {request.url}") + logging.error("Error details:") + logging.error(exc.errors()) + logging.error(exc.body) + return JSONResponse( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + content=jsonable_encoder({"detail": exc.errors(), "body": exc.body}), + ) + app.include_router(create_index(templates)) app.include_router( create_jobs( From 4efba42a631d7f38fa2d5da52920485efe134bbc Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Fri, 14 Jul 2023 15:14:46 +0200 Subject: [PATCH 26/32] Fix duplicate window bug --- ocrdbrowser/_subprocess.py | 123 ++++++++++++------ .../server/workspaces/_launchroutes.py | 14 +- ocrdmonitor/server/workspaces/_proxyroutes.py | 10 +- 3 files changed, 96 insertions(+), 51 deletions(-) diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index a9984be..bd1892e 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -6,7 +6,7 @@ import os import signal from shutil import which -from typing import cast +from typing import NamedTuple, Self, Type, cast from ._browser import OcrdBrowser, OcrdBrowserClient from ._client import HttpBrowserClient @@ -15,6 +15,19 @@ BROADWAY_BASE_PORT = 8080 +class BroadwayBrowserId(NamedTuple): + broadway_pid: int + browser_pid: int + + @classmethod + def from_str(cls: Type[Self], id_str: str) -> Self: + ids = map(int, id_str.split("-")) + return BroadwayBrowserId(*ids) + + def __str__(self) -> str: + return f"{self.broadway_pid}-{self.browser_pid}" + + class SubProcessOcrdBrowser: def __init__( self, owner: str, workspace: str, address: str, process_id: str @@ -22,10 +35,10 @@ def __init__( self._owner = owner self._workspace = workspace self._address = address - self._process_id = process_id + self._process_id = BroadwayBrowserId.from_str(process_id) def process_id(self) -> str: - return self._process_id + return str(self._process_id) def address(self) -> str: return self._address @@ -37,10 +50,15 @@ def owner(self) -> str: return self._owner async def stop(self) -> None: + self._try_kill(self._process_id.broadway_pid) + self._try_kill(self._process_id.browser_pid) + + @staticmethod + def _try_kill(pid: int) -> None: try: - os.kill(int(self._process_id), signal.SIGKILL) + os.kill(pid, signal.SIGKILL) except ProcessLookupError: - logging.warning(f"Could not find process with ID {self._process_id}") + logging.warning(f"Could not find process with ID {pid}") def client(self) -> OcrdBrowserClient: return HttpBrowserClient(self.address()) @@ -56,62 +74,81 @@ def __init__(self, available_ports: set[int]) -> None: async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: port_binding = functools.partial(start_browser, workspace_path) - process, port = await try_bind( + pid, port = await try_bind( port_binding, "http://localhost", self._available_ports ) address = f"http://localhost:{port}" - return SubProcessOcrdBrowser(owner, workspace_path, address, str(process.pid)) - - # stderr = cast(asyncio.StreamReader, process.stderr) - # logging.error((await stderr.read()).splitlines()) - # raise ProcessLaunchFailedError() + return SubProcessOcrdBrowser(owner, workspace_path, address, str(pid)) async def start_browser( workspace: str, host: str, port: int -) -> PortBindingResult[asyncio.subprocess.Process]: - browse_ocrd = which("browse-ocrd") - if not browse_ocrd: - raise FileNotFoundError("Could not find browse-ocrd executable") +) -> PortBindingResult[BroadwayBrowserId]: + find_executables_or_raise() # broadwayd (which uses WebSockets) only allows a single client at a time # (disconnecting concurrent connections), hence we must start a new daemon # for each new browser session # broadwayd starts counting virtual X displays from port 8080 as :0 displayport = str(port - BROADWAY_BASE_PORT) - environment = dict(os.environ) - environment["GDK_BACKEND"] = "broadway" - environment["BROADWAY_DISPLAY"] = ":" + displayport try: - process = await asyncio.create_subprocess_shell( - " ".join( - [ - "broadwayd", - ":" + displayport + " &", - browse_ocrd, - workspace + "/mets.xml" + " ;", - "kill" + " $!", - ] - ), - env=environment, - stderr=asyncio.subprocess.PIPE, + broadway_process = await launch_broadway(displayport) + + if broadway_process is None: + return PortBindingError() + + environment = prepare_env(displayport) + full_cmd = browser_command(workspace, broadway_process.pid) + browser_process = await asyncio.create_subprocess_shell( + full_cmd, env=environment ) - try: - stderr = cast(asyncio.StreamReader, process.stderr) - err_output = await asyncio.wait_for(stderr.readline(), 5) - if b"Address already in use" in err_output: - return PortBindingError() - except asyncio.TimeoutError: - logging.info( - f"The process didn't exit within the given timeout. Assuming browser on port {port} launched successfully" - ) - pass - - return process + return BroadwayBrowserId(broadway_process.pid, browser_process.pid) except Exception as err: - logging.error(f"Failed to launch broadway at {displayport} (real port {port})") + logging.error(f"Failed to launch broadway at (real port {port})") logging.error(repr(err)) return PortBindingError() + + +def find_executables_or_raise(): + if not which("broadwayd"): + raise FileNotFoundError("Could not find broadwayd executable") + + if not which("browse-ocrd"): + raise FileNotFoundError("Could not find browse-ocrd executable") + + +async def launch_broadway( + displayport: int, +) -> asyncio.subprocess.Process | None: + broadway_process = await asyncio.create_subprocess_exec( + which("broadwayd"), f":{displayport}", stderr=asyncio.subprocess.PIPE + ) + + try: + stderr = cast(asyncio.StreamReader, broadway_process.stderr) + err_output = await asyncio.wait_for(stderr.readline(), 5) + if b"Address already in use" in err_output: + return None + except asyncio.TimeoutError: + logging.info( + "The process didn't exit within the given timeout." + + f"Assuming broadway on port {displayport} launched successfully" + ) + + return broadway_process + + +def prepare_env(displayport: int) -> dict[str, str]: + environment = dict(os.environ) + environment["GDK_BACKEND"] = "broadway" + environment["BROADWAY_DISPLAY"] = ":" + displayport + return environment + + +def browser_command(workspace: str, broadway_pid: int) -> str: + mets_path = workspace + "/mets.xml" + kill_broadway = f"; kill {broadway_pid}" + return " ".join([which("browse-ocrd"), mets_path, kill_broadway]) diff --git a/ocrdmonitor/server/workspaces/_launchroutes.py b/ocrdmonitor/server/workspaces/_launchroutes.py index fcf13df..ede7661 100644 --- a/ocrdmonitor/server/workspaces/_launchroutes.py +++ b/ocrdmonitor/server/workspaces/_launchroutes.py @@ -1,8 +1,9 @@ -from typing import Callable import uuid from pathlib import Path +from typing import Callable from fastapi import APIRouter, Depends, Request, Response +from fastapi.params import Cookie from fastapi.templating import Jinja2Templates from ocrdbrowser import OcrdBrowserFactory @@ -24,20 +25,21 @@ def register_launchroutes( ) -> None: @router.get("/open/{workspace:path}", name="workspaces.open") def open_workspace(request: Request, workspace: str) -> Response: - return templates.TemplateResponse( + session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) + response = templates.TemplateResponse( "workspace.html.j2", - {"request": request, "workspace": workspace}, + {"request": request, "session_id": session_id, "workspace": workspace}, ) + response.set_cookie("session_id", session_id) + return response @router.get("/browse/{workspace:path}", name="workspaces.browse") async def browser( - request: Request, workspace: Path, + session_id: str = Cookie(), factory: OcrdBrowserFactory = Depends(browser_settings.factory), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: - session_id = request.cookies.setdefault("session_id", str(uuid.uuid4())) - full_path = full_workspace(workspace) existing_browsers = await repository.find(owner=session_id, workspace=full_path) diff --git a/ocrdmonitor/server/workspaces/_proxyroutes.py b/ocrdmonitor/server/workspaces/_proxyroutes.py index bde7570..a312248 100644 --- a/ocrdmonitor/server/workspaces/_proxyroutes.py +++ b/ocrdmonitor/server/workspaces/_proxyroutes.py @@ -39,6 +39,7 @@ def in_workspace(browser: OcrdBrowser) -> bool: def browser_closed_callback(repository: BrowserProcessRepository) -> CloseCallback: async def _callback(browser: OcrdBrowser) -> None: await stop_and_remove_browser(repository, browser) + return _callback @@ -74,16 +75,21 @@ async def ping_workspace( async def workspace_reverse_proxy( request: Request, workspace: Path, - session_id: str = Cookie(), + session_id: str = Cookie(default=None), repository: BrowserProcessRepository = Depends(browser_settings.repository), ) -> Response: + # The session_id cookie is not always properly injected for some reason + # Therefore we try to get it from the request if it is None + session_id = session_id or request.cookies.get("session_id") + browser = await first_owned_browser_in_workspace( session_id, full_workspace(workspace), repository ) if not browser: return Response( - content=f"No browser found for {workspace}", status_code=404 + content=f"No browser found for {workspace} and session ID {session_id}", + status_code=404, ) try: return await forward(browser, str(workspace)) From 037633c83889811a6bfe5619b12977448c54c4a1 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Fri, 14 Jul 2023 15:41:46 +0200 Subject: [PATCH 27/32] Fix the tests --- tests/ocrdmonitor/server/test_workspace_endpoint.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 99e4f33..26c838b 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -4,6 +4,7 @@ import pytest import pytest_asyncio +from fastapi import WebSocketDisconnect from fastapi.testclient import TestClient from httpx import Response @@ -91,12 +92,12 @@ def interact_with_workspace(app: TestClient, workspace: str) -> Response: response = view_workspace(app, workspace) with app.websocket_connect(f"/workspaces/view/{workspace}/socket"): pass - return response def open_workspace(app: TestClient, workspace: str) -> None: - _ = app.get(f"/workspaces/browse/{workspace}") + _ = app.get(f"/workspaces/open/{workspace}") + return app.get(f"/workspaces/browse/{workspace}") def view_workspace(app: TestClient, workspace: str) -> Response: @@ -116,7 +117,7 @@ def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( browser: BrowserTestDouble, app: TestClient, ) -> None: - response = app.get("/workspaces/browse/a_workspace") + response = open_workspace(app, "a_workspace") assert browser.is_running is True assert browser.workspace() == str(WORKSPACE_DIR / "a_workspace") @@ -125,7 +126,7 @@ def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( @pytest.mark.usefixtures("iterating_factory") def test__browse_workspace__assigns_and_tracks_session_id(app: TestClient) -> None: - response = app.get("/workspaces/browse/a_workspace") + response = open_workspace(app, "a_workspace") first_session_id = response.cookies.get("session_id") response = app.get("/workspaces/browse/a_workspace") From de3c815fd808af1808a466d564f6a3bc09cd02b2 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Fri, 14 Jul 2023 16:00:25 +0200 Subject: [PATCH 28/32] Passes mypy --- ocrdbrowser/_subprocess.py | 14 ++++++++------ ocrdmonitor/server/workspaces/_launchroutes.py | 3 +-- ocrdmonitor/server/workspaces/_proxyroutes.py | 6 +++++- .../ocrdmonitor/server/test_workspace_endpoint.py | 2 +- 4 files changed, 15 insertions(+), 10 deletions(-) diff --git a/ocrdbrowser/_subprocess.py b/ocrdbrowser/_subprocess.py index bd1892e..7989dd6 100644 --- a/ocrdbrowser/_subprocess.py +++ b/ocrdbrowser/_subprocess.py @@ -20,7 +20,7 @@ class BroadwayBrowserId(NamedTuple): browser_pid: int @classmethod - def from_str(cls: Type[Self], id_str: str) -> Self: + def from_str(cls: Type["BroadwayBrowserId"], id_str: str) -> "BroadwayBrowserId": ids = map(int, id_str.split("-")) return BroadwayBrowserId(*ids) @@ -112,7 +112,7 @@ async def start_browser( return PortBindingError() -def find_executables_or_raise(): +def find_executables_or_raise() -> None: if not which("broadwayd"): raise FileNotFoundError("Could not find broadwayd executable") @@ -121,10 +121,11 @@ def find_executables_or_raise(): async def launch_broadway( - displayport: int, + displayport: str, ) -> asyncio.subprocess.Process | None: + broadway = cast(str, which("broadwayd")) broadway_process = await asyncio.create_subprocess_exec( - which("broadwayd"), f":{displayport}", stderr=asyncio.subprocess.PIPE + broadway, f":{displayport}", stderr=asyncio.subprocess.PIPE ) try: @@ -141,7 +142,7 @@ async def launch_broadway( return broadway_process -def prepare_env(displayport: int) -> dict[str, str]: +def prepare_env(displayport: str) -> dict[str, str]: environment = dict(os.environ) environment["GDK_BACKEND"] = "broadway" environment["BROADWAY_DISPLAY"] = ":" + displayport @@ -151,4 +152,5 @@ def prepare_env(displayport: int) -> dict[str, str]: def browser_command(workspace: str, broadway_pid: int) -> str: mets_path = workspace + "/mets.xml" kill_broadway = f"; kill {broadway_pid}" - return " ".join([which("browse-ocrd"), mets_path, kill_broadway]) + browse_ocrd = cast(str, which("browse-ocrd")) + return " ".join([browse_ocrd, mets_path, kill_broadway]) diff --git a/ocrdmonitor/server/workspaces/_launchroutes.py b/ocrdmonitor/server/workspaces/_launchroutes.py index ede7661..e43f867 100644 --- a/ocrdmonitor/server/workspaces/_launchroutes.py +++ b/ocrdmonitor/server/workspaces/_launchroutes.py @@ -2,8 +2,7 @@ from pathlib import Path from typing import Callable -from fastapi import APIRouter, Depends, Request, Response -from fastapi.params import Cookie +from fastapi import APIRouter, Cookie, Depends, Request, Response from fastapi.templating import Jinja2Templates from ocrdbrowser import OcrdBrowserFactory diff --git a/ocrdmonitor/server/workspaces/_proxyroutes.py b/ocrdmonitor/server/workspaces/_proxyroutes.py index a312248..1ff040d 100644 --- a/ocrdmonitor/server/workspaces/_proxyroutes.py +++ b/ocrdmonitor/server/workspaces/_proxyroutes.py @@ -43,6 +43,10 @@ async def _callback(browser: OcrdBrowser) -> None: return _callback +def get_session_id(request: Request, session_id: str | None) -> str: + return session_id or request.cookies["session_id"] + + def register_proxyroutes( router: APIRouter, templates: Jinja2Templates, @@ -80,7 +84,7 @@ async def workspace_reverse_proxy( ) -> Response: # The session_id cookie is not always properly injected for some reason # Therefore we try to get it from the request if it is None - session_id = session_id or request.cookies.get("session_id") + session_id = get_session_id(request, session_id) browser = await first_owned_browser_in_workspace( session_id, full_workspace(workspace), repository diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 26c838b..62b27f3 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -95,7 +95,7 @@ def interact_with_workspace(app: TestClient, workspace: str) -> Response: return response -def open_workspace(app: TestClient, workspace: str) -> None: +def open_workspace(app: TestClient, workspace: str) -> Response: _ = app.get(f"/workspaces/open/{workspace}") return app.get(f"/workspaces/browse/{workspace}") From 18519c0c0ed36764122a98fe669e109326d7c7a5 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 18 Jul 2023 13:00:44 +0200 Subject: [PATCH 29/32] Introduce Fixture facade --- .../server/fixtures/environment.py | 128 +++++++++++ .../ocrdmonitor/server/fixtures/repository.py | 1 - tests/ocrdmonitor/server/test_startup.py | 51 ++--- .../server/test_workspace_endpoint.py | 215 +++++++++--------- tests/testdoubles/__init__.py | 22 +- tests/testdoubles/_browserfactory.py | 33 +-- .../testdoubles/_browserprocessrepository.py | 4 +- tests/testdoubles/_browserspy.py | 32 ++- tests/testdoubles/_registrybrowserfactory.py | 55 +++++ 9 files changed, 368 insertions(+), 173 deletions(-) create mode 100644 tests/ocrdmonitor/server/fixtures/environment.py create mode 100644 tests/testdoubles/_registrybrowserfactory.py diff --git a/tests/ocrdmonitor/server/fixtures/environment.py b/tests/ocrdmonitor/server/fixtures/environment.py new file mode 100644 index 0000000..9bdf97a --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/environment.py @@ -0,0 +1,128 @@ +from dataclasses import dataclass +from types import TracebackType +from typing import ( + Any, + AsyncContextManager, + Callable, + ContextManager, + Self, + Type, +) + +from fastapi.testclient import TestClient + +from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.server.app import create_app +from tests.ocrdmonitor.server.fixtures.app import create_settings +from tests.ocrdmonitor.server.fixtures.factory import patch_factory +from tests.ocrdmonitor.server.fixtures.repository import ( + RepositoryInitializer, + inmemory_repository, + patch_repository, +) +from tests.testdoubles import ( + BrowserRegistry, + BrowserSpy, + BrowserTestDouble, + IteratingBrowserTestDoubleFactory, + RegistryBrowserFactory, + RestoringRegistryBrowserFactory, +) + + +@dataclass +class Environment: + repository: BrowserProcessRepository + app: TestClient + + +BrowserConstructor = Callable[[], BrowserTestDouble] + + +class Fixture: + def __init__(self) -> None: + self.browser_constructor: BrowserConstructor = BrowserSpy + self.repo_constructor: RepositoryInitializer = inmemory_repository + self.existing_browsers: list[BrowserTestDouble] = [] + self.session_id = "" + + self._open_contexts: list[ContextManager[Any] | AsyncContextManager[Any]] = [] + + def with_browser_type(self, browser_constructor: BrowserConstructor) -> Self: + self.browser_constructor = browser_constructor + return self + + def with_repository_type(self, repo_constructor: RepositoryInitializer) -> Self: + self.repo_constructor = repo_constructor + return self + + def with_running_browsers(self, *browsers: BrowserTestDouble) -> Self: + self.existing_browsers = list(browsers) + return self + + def with_session_id(self, session_id: str) -> Self: + self.session_id = session_id + return self + + async def __aenter__(self) -> Environment: + registry = BrowserRegistry({}) + repository = await self._patch_repository(registry) + await self._patch_factory(registry) + await self._insert_running_browsers(registry, repository) + app = self._build_app() + + return Environment(repository=repository, app=app) + + async def _patch_repository( + self, registry: BrowserRegistry + ) -> BrowserProcessRepository: + repository = await self._init_repo(registry) + patcher = patch_repository(repository) + self._open_contexts.append(patcher) + await patcher.__aenter__() + return repository + + async def _init_repo(self, registry: BrowserRegistry) -> BrowserProcessRepository: + restoring_factory = RestoringRegistryBrowserFactory(registry) + repo_ctx = self.repo_constructor(restoring_factory) + self._open_contexts.append(repo_ctx) + repository = await repo_ctx.__aenter__() + return repository + + async def _patch_factory(self, registry: BrowserRegistry) -> None: + creating_factory = RegistryBrowserFactory( + IteratingBrowserTestDoubleFactory(default_browser=self.browser_constructor), + registry, + ) + patcher = patch_factory(creating_factory) + await patcher.__aenter__() + self._open_contexts.append(patcher) + + def _build_app(self) -> TestClient: + app = TestClient(create_app(create_settings())) + app.__enter__() + if self.session_id: + app.cookies["session_id"] = self.session_id + + self._open_contexts.append(app) + return app + + async def _insert_running_browsers( + self, registry: BrowserRegistry, repository: BrowserProcessRepository + ) -> None: + for browser in self.existing_browsers: + registry[browser.address()] = browser + await repository.insert(browser) + await browser.start() + + async def __aexit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + for ctx in self._open_contexts: + if isinstance(ctx, AsyncContextManager): + await ctx.__aexit__(exc_type, exc_value, traceback) + else: + ctx.__exit__(exc_type, exc_value, traceback) diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index d27fe50..0d6aa3a 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -16,7 +16,6 @@ ) from tests.testdoubles._browserfactory import SingletonRestoringBrowserFactory - RepositoryInitializer = Callable[ [BrowserRestoringFactory], AsyncContextManager[BrowserProcessRepository], diff --git a/tests/ocrdmonitor/server/test_startup.py b/tests/ocrdmonitor/server/test_startup.py index 746d410..2c3e064 100644 --- a/tests/ocrdmonitor/server/test_startup.py +++ b/tests/ocrdmonitor/server/test_startup.py @@ -1,38 +1,21 @@ -from fastapi.testclient import TestClient +import pytest -from ocrdbrowser import OcrdBrowser -from ocrdmonitor.server.app import create_app -from tests.ocrdmonitor.server.decorators import use_custom_repository -from tests.ocrdmonitor.server.fixtures.app import create_settings -from tests.ocrdmonitor.server.fixtures.repository import ( - RepositoryInitializer, - patch_repository, -) -from tests.testdoubles import BrowserSpy +from tests.ocrdmonitor.server.fixtures.environment import Fixture +from tests.testdoubles import BrowserSpy, unreachable_browser -@use_custom_repository -async def test__browsers_in_db__on_startup__cleans_unreachables_from_db( - repository: RepositoryInitializer, -) -> None: - reachable = BrowserSpy(address="http://reachable.com") - unreachable = BrowserSpy(address="http://unreachable.com") - unreachable.configure_client(response=ConnectionError) +@pytest.mark.asyncio +@pytest.mark.no_auto_repository +async def test__browsers_in_db__on_startup__cleans_unreachables_from_db() -> None: + session_id = "the-owner" + reachable = BrowserSpy(owner=session_id, address="http://reachable.com") + unreachable = unreachable_browser( + owner=session_id, address="http://unreachable.com" + ) - def factory( - owner: str, workspace: str, address: str, process_id: str - ) -> OcrdBrowser: - if "unreachable" in address: - return unreachable - else: - return reachable - - async with repository(factory) as repo: - async with patch_repository(repo): - await repo.insert(unreachable) - await repo.insert(reachable) - - with TestClient(create_app(create_settings())): - pass - - assert await repo.count() == 1 + async with ( + Fixture() + .with_running_browsers(reachable, unreachable) + .with_session_id(session_id) + ) as env: + assert await env.repository.count() == 1 diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 62b27f3..4d1d034 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -4,20 +4,16 @@ import pytest import pytest_asyncio -from fastapi import WebSocketDisconnect from fastapi.testclient import TestClient from httpx import Response -from ocrdbrowser import ChannelClosed -from ocrdmonitor.browserprocess import BrowserProcessRepository -from ocrdmonitor.server.app import create_app from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.decorators import use_custom_repository -from tests.ocrdmonitor.server.fixtures.app import WORKSPACE_DIR, create_settings +from tests.ocrdmonitor.server.fixtures.app import WORKSPACE_DIR +from tests.ocrdmonitor.server.fixtures.environment import Environment, Fixture from tests.ocrdmonitor.server.fixtures.factory import patch_factory from tests.ocrdmonitor.server.fixtures.repository import ( RepositoryInitializer, - patch_repository, ) from tests.testdoubles import ( Browser_Heading, @@ -26,17 +22,9 @@ BrowserTestDouble, BrowserTestDoubleFactory, IteratingBrowserTestDoubleFactory, - SingletonBrowserTestDoubleFactory, + browser_with_disconnecting_channel, + unreachable_browser, ) -from tests.testdoubles._browserfactory import SingletonRestoringBrowserFactory - - -class DisconnectingChannel: - async def send_bytes(self, data: bytes) -> None: - raise ChannelClosed() - - async def receive_bytes(self) -> bytes: - raise ChannelClosed() @pytest_asyncio.fixture( @@ -51,13 +39,6 @@ async def iterating_factory( yield factory -@pytest_asyncio.fixture -async def singleton_browser_spy() -> AsyncIterator[BrowserSpy]: - browser_spy = BrowserSpy() - async with patch_factory(SingletonBrowserTestDoubleFactory(browser_spy)): - yield browser_spy - - @pytest.fixture( params=(BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)) ) @@ -76,8 +57,7 @@ def browser( def disconnecting_browser( iterating_factory: IteratingBrowserTestDoubleFactory, ) -> BrowserSpy: - disconnecting_browser = BrowserSpy() - disconnecting_browser.configure_client(channel=DisconnectingChannel()) + disconnecting_browser = browser_with_disconnecting_channel() iterating_factory.add(disconnecting_browser) return disconnecting_browser @@ -104,6 +84,12 @@ def view_workspace(app: TestClient, workspace: str) -> Response: return app.get(f"/workspaces/view/{workspace}") +@pytest_asyncio.fixture +async def defaultenv() -> AsyncIterator[Environment]: + async with Fixture() as env: + yield env + + def test__workspaces__shows_the_workspace_names_starting_from_workspace_root( app: TestClient, ) -> None: @@ -124,8 +110,10 @@ def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( assert response.status_code == 200 -@pytest.mark.usefixtures("iterating_factory") -def test__browse_workspace__assigns_and_tracks_session_id(app: TestClient) -> None: +def test__browse_workspace__assigns_and_tracks_session_id( + defaultenv: Environment, +) -> None: + app = defaultenv.app response = open_workspace(app, "a_workspace") first_session_id = response.cookies.get("session_id") @@ -136,21 +124,24 @@ def test__browse_workspace__assigns_and_tracks_session_id(app: TestClient) -> No assert first_session_id == second_session_id -@pytest.mark.usefixtures("iterating_factory") +@pytest.mark.asyncio @use_custom_repository -async def test__opened_workspace__when_socket_disconnects_on_broadway_side__shuts_down_browser( +async def test__opened_workspace__when_socket_disconnects__shuts_down_browser( repository: RepositoryInitializer, ) -> None: - factory = SingletonRestoringBrowserFactory() - disconnecting_browser = factory.browser - disconnecting_browser.configure_client(channel=DisconnectingChannel()) - - async with repository(factory) as repo: - async with patch_repository(repo): - app = TestClient(create_app(create_settings())) - await repo.insert(disconnecting_browser) + session_id = "the-owner" + disconnecting_browser = browser_with_disconnecting_channel( + session_id, str(WORKSPACE_DIR / "a_workspace") + ) + fixture = ( + Fixture() + .with_repository_type(repository) + .with_running_browsers(disconnecting_browser) + .with_session_id(session_id) + ) - _ = interact_with_workspace(app, "a_workspace") + async with fixture as env: + _ = interact_with_workspace(env.app, "a_workspace") assert disconnecting_browser.is_running is False @@ -168,32 +159,42 @@ def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_t @pytest.mark.asyncio -@pytest.mark.usefixtures("iterating_factory") @use_custom_repository async def test__when_requesting_resource__returns_resource_from_workspace( repository: RepositoryInitializer, ) -> None: - factory = SingletonRestoringBrowserFactory() - factory.browser.configure_client(response_factory=lambda path: path.encode()) + session_id = "the-owner" workspace = "a_workspace" resource = "/some_resource" resource_in_workspace = workspace + "/some_resource" + full_workspace = str(WORKSPACE_DIR / workspace) - async with repository(factory) as repo: - async with patch_repository(repo): - app = TestClient(create_app(create_settings())) - open_workspace(app, workspace) + def echo_bytes(path: str) -> bytes: + return path.encode() - actual = view_workspace(app, resource_in_workspace) + browser = BrowserSpy(session_id, full_workspace) + browser.configure_client(response_factory=echo_bytes) - assert actual.content == resource.encode() + fixture = ( + Fixture() + .with_repository_type(repository) + .with_running_browsers(browser) + .with_session_id(session_id) + ) + + async with fixture as env: + open_workspace(env.app, workspace) + + actual = view_workspace(env.app, resource_in_workspace) + + assert actual.content == resource.encode() -@pytest.mark.usefixtures("iterating_factory") def test__browsed_workspace_is_ready__when_pinging__returns_ok( - app: TestClient, + defaultenv: Environment, ) -> None: + app = defaultenv.app workspace = "a_workspace" _ = interact_with_workspace(app, workspace) @@ -202,102 +203,104 @@ def test__browsed_workspace_is_ready__when_pinging__returns_ok( assert result.status_code == 200 -@pytest.mark.usefixtures("iterating_factory") @use_custom_repository async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( - singleton_restoring_factory: SingletonRestoringBrowserFactory, repository: RepositoryInitializer, ) -> None: - async with repository(singleton_restoring_factory) as repo: - async with patch_repository(repo): - app = TestClient(create_app(create_settings())) - - browser = singleton_restoring_factory.browser - browser.configure_client(response=ConnectionError) + workspace = "a_workspace" + fixture = ( + Fixture() + .with_browser_type(unreachable_browser) + .with_repository_type(repository) + ) - workspace = "a_workspace" - open_workspace(app, workspace) + async with fixture as env: + open_workspace(env.app, workspace) - result = app.get(f"/workspaces/ping/{workspace}") + result = env.app.get(f"/workspaces/ping/{workspace}") assert result.status_code == 502 -@pytest.mark.asyncio -@pytest.mark.usefixtures("iterating_factory") +@use_custom_repository async def test__browsing_workspace__stores_browser_in_repository( - auto_repository: BrowserProcessRepository, app: TestClient + repository: RepositoryInitializer, ) -> None: - _ = interact_with_workspace(app, "a_workspace") + fixture = Fixture().with_repository_type(repository) - found_browsers = list( - await auto_repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) - ) + async with fixture as env: + _ = interact_with_workspace(env.app, "a_workspace") + + found_browsers = list( + await env.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) assert len(found_browsers) == 1 -@pytest.mark.usefixtures("iterating_factory") @use_custom_repository async def test__error_connecting_to_workspace__removes_browser_from_repository( - singleton_restoring_factory: SingletonRestoringBrowserFactory, repository: RepositoryInitializer, ) -> None: - browser = singleton_restoring_factory.browser - browser.configure_client(response=ConnectionError) - - async with repository(singleton_restoring_factory) as repo: - async with patch_repository(repo): - app = TestClient(create_app(create_settings())) - app.cookies.set("session_id", browser.owner()) + fixture = ( + Fixture() + .with_browser_type(unreachable_browser) + .with_repository_type(repository) + ) - open_workspace(app, "a_workspace") - _ = view_workspace(app, "a_workspace") + async with fixture as env: + open_workspace(env.app, "a_workspace") + _ = view_workspace(env.app, "a_workspace") - found_browsers = list( - await repo.find(workspace=str(WORKSPACE_DIR / "a_workspace")) - ) + found_browsers = list( + await env.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) - assert len(found_browsers) == 0 + assert len(found_browsers) == 0 -@pytest.mark.usefixtures("iterating_factory") @use_custom_repository async def test__when_socket_to_workspace_disconnects__removes_browser_from_repository( - singleton_restoring_factory: SingletonRestoringBrowserFactory, repository: RepositoryInitializer, ) -> None: - browser = singleton_restoring_factory.browser - browser.configure_client(channel=DisconnectingChannel()) - - async with repository(singleton_restoring_factory) as repo: - async with patch_repository(repo): - app = TestClient(create_app(create_settings())) - app.cookies.set("session_id", browser.owner()) + # NOTE: it seems something is weird with the event loop in this test. + # Searching for browsers inside the with block happens BEFORE the browser is deleted. + # Therefore we run the lookup after the contextmanager has been closed + + fixture = ( + Fixture() + .with_repository_type(repository) + .with_browser_type(browser_with_disconnecting_channel) + ) - _ = interact_with_workspace(app, "a_workspace") + async with fixture as env: + _ = interact_with_workspace(env.app, "a_workspace") - found_browsers = list( - await repo.find(workspace=str(WORKSPACE_DIR / "a_workspace")) - ) + found_browsers = list( + await env.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) - assert len(found_browsers) == 0 + assert len(found_browsers) == 0 -@pytest.mark.usefixtures("iterating_factory") @use_custom_repository async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( - singleton_restoring_factory: SingletonRestoringBrowserFactory, repository: RepositoryInitializer, ) -> None: - async with repository(singleton_restoring_factory) as repo: - async with patch_repository(repo): - app = TestClient(create_app(create_settings())) - - browser = singleton_restoring_factory.browser - browser.configure_client(response=b"RESTORED BROWSER") - await repo.insert(browser) + session_id = "the-owner" + workspace = "a_workspace" + full_workspace = str(WORKSPACE_DIR / workspace) + browser = BrowserSpy(session_id, full_workspace) + browser.configure_client(response=b"RESTORED BROWSER") + + fixture = ( + Fixture() + .with_repository_type(repository) + .with_running_browsers(browser) + .with_session_id(session_id) + ) - response = interact_with_workspace(app, "a_workspace") + async with fixture as env: + response = interact_with_workspace(env.app, "a_workspace") assert response.content == b"RESTORED BROWSER" diff --git a/tests/testdoubles/__init__.py b/tests/testdoubles/__init__.py index 6467af9..6b5bfa3 100644 --- a/tests/testdoubles/__init__.py +++ b/tests/testdoubles/__init__.py @@ -1,14 +1,23 @@ from ._backgroundprocess import BackgroundProcess -from ._broadwayfake import broadway_fake, FAKE_HOST_ADDRESS +from ._broadwayfake import FAKE_HOST_ADDRESS, broadway_fake from ._browserfactory import ( BrowserTestDouble, BrowserTestDoubleFactory, IteratingBrowserTestDoubleFactory, - SingletonBrowserTestDoubleFactory, ) from ._browserfake import BrowserFake -from ._browserspy import BrowserSpy, Browser_Heading from ._browserprocessrepository import InMemoryBrowserProcessRepository +from ._browserspy import ( + Browser_Heading, + BrowserSpy, + browser_with_disconnecting_channel, + unreachable_browser, +) +from ._registrybrowserfactory import ( + BrowserRegistry, + RegistryBrowserFactory, + RestoringRegistryBrowserFactory, +) __all__ = [ "BackgroundProcess", @@ -19,7 +28,12 @@ "BrowserTestDouble", "BrowserTestDoubleFactory", "FAKE_HOST_ADDRESS", - "SingletonBrowserTestDoubleFactory", "IteratingBrowserTestDoubleFactory", "InMemoryBrowserProcessRepository", + "BrowserRegistry", + "ProxyBrowser", + "RegistryBrowserFactory", + "RestoringRegistryBrowserFactory", + "browser_with_disconnecting_channel", + "unreachable_browser", ] diff --git a/tests/testdoubles/_browserfactory.py b/tests/testdoubles/_browserfactory.py index d3f4b7a..0bbba57 100644 --- a/tests/testdoubles/_browserfactory.py +++ b/tests/testdoubles/_browserfactory.py @@ -1,8 +1,9 @@ import asyncio from types import TracebackType -from typing import Any, Callable, Protocol, Self, Type +from typing import Any, AsyncContextManager, Callable, Protocol, Self, Type + +from ocrdbrowser import OcrdBrowser, OcrdBrowserFactory -from ocrdbrowser import OcrdBrowser from ._browserspy import BrowserSpy @@ -18,27 +19,6 @@ def is_running(self) -> bool: ... -class SingletonBrowserTestDoubleFactory: - def __init__(self, browser: BrowserTestDouble | None = None) -> None: - self._browser = browser or BrowserSpy() - - async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: - self._browser.set_owner_and_workspace(owner, workspace_path) - await self._browser.start() - return self._browser - - async def __aenter__(self) -> Self: - return self - - async def __aexit__( - self, - exc_type: Type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self._browser.stop() - - class IteratingBrowserTestDoubleFactory: def __init__( self, @@ -74,9 +54,10 @@ async def __aexit__( group.create_task(browser.stop()) -BrowserTestDoubleFactory = ( - SingletonBrowserTestDoubleFactory | IteratingBrowserTestDoubleFactory -) +class BrowserTestDoubleFactory( + OcrdBrowserFactory, AsyncContextManager[OcrdBrowserFactory], Protocol +): + pass class SingletonRestoringBrowserFactory: diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index 2f035ee..f6f8ec4 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -1,7 +1,9 @@ from typing import Collection, NamedTuple + from ocrdbrowser import OcrdBrowser from ocrdmonitor.browserprocess import BrowserRestoringFactory -from tests.testdoubles import BrowserSpy + +from ._browserspy import BrowserSpy class BrowserEntry(NamedTuple): diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index b17393f..a3fcb5d 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -4,7 +4,7 @@ from textwrap import dedent from typing import AsyncGenerator, Callable, Type -from ocrdbrowser import Channel, OcrdBrowserClient +from ocrdbrowser import Channel, ChannelClosed, OcrdBrowserClient Browser_Heading = "OCRD BROWSER" @@ -26,6 +26,14 @@ async def receive_bytes(self) -> bytes: return bytes() +class DisconnectingChannel: + async def send_bytes(self, data: bytes) -> None: + raise ChannelClosed() + + async def receive_bytes(self) -> bytes: + raise ChannelClosed() + + class BrowserClientStub: def __init__( self, @@ -109,3 +117,25 @@ def __repr__(self) -> str: running: {self.is_running} """ ) + + +def browser_with_disconnecting_channel( + owner: str = "", + workspace: str = "", + address: str = "http://unreachable.example.com", + process_id: str = "1234", +) -> BrowserSpy: + spy = BrowserSpy(owner, workspace, address, process_id) + spy.configure_client(channel=DisconnectingChannel()) + return spy + + +def unreachable_browser( + owner: str = "", + workspace: str = "", + address: str = "http://unreachable.example.com", + process_id: str = "1234", +) -> BrowserSpy: + spy = BrowserSpy(owner, workspace, address, process_id) + spy.configure_client(response=ConnectionError) + return spy diff --git a/tests/testdoubles/_registrybrowserfactory.py b/tests/testdoubles/_registrybrowserfactory.py new file mode 100644 index 0000000..f3dcd00 --- /dev/null +++ b/tests/testdoubles/_registrybrowserfactory.py @@ -0,0 +1,55 @@ +from types import TracebackType +from typing import NewType, Self, Type, cast + +from ocrdbrowser import OcrdBrowser + +from ._browserfactory import ( + BrowserTestDouble, + BrowserTestDoubleFactory, + IteratingBrowserTestDoubleFactory, +) + +BrowserRegistry = NewType("BrowserRegistry", dict[str, BrowserTestDouble]) + + +class RegistryBrowserFactory: + + @classmethod + def iteratingfactory(cls: Type[Self], browser_registry: BrowserRegistry) -> Self: + return cls(IteratingBrowserTestDoubleFactory(), browser_registry) + + def __init__( + self, + internal_factory: BrowserTestDoubleFactory, + browser_registry: BrowserRegistry, + ) -> None: + self._factory = internal_factory + self._registry = browser_registry + + async def __aenter__(self) -> Self: + await self._factory.__aenter__() + return self + + async def __aexit__( + self, + exc_type: Type[BaseException] | None, + exc_value: BaseException | None, + traceback: TracebackType | None, + ) -> None: + await self._factory.__aexit__(exc_type, exc_value, traceback) + + async def __call__(self, owner: str, workspace_path: str) -> OcrdBrowser: + browser = await self._factory(owner, workspace_path) + self._registry[browser.address()] = cast(BrowserTestDouble, browser) + return browser + + +class RestoringRegistryBrowserFactory: + def __init__(self, browser_registry: BrowserRegistry) -> None: + self._registry = browser_registry + + def __call__( + self, owner: str, workspace: str, address: str, process_id: str + ) -> BrowserTestDouble: + browser = self._registry[address] + return browser From 5554c85abbc3c2c330b5c2cf19051e015478e076 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 18 Jul 2023 17:12:20 +0200 Subject: [PATCH 30/32] Refactor existing tests for Fixture API --- tests/ocrdmonitor/server/conftest.py | 6 +- tests/ocrdmonitor/server/fixtures/app.py | 52 +---- .../server/fixtures/environment.py | 9 +- .../ocrdmonitor/server/fixtures/repository.py | 63 +---- tests/ocrdmonitor/server/fixtures/settings.py | 27 +++ tests/ocrdmonitor/server/test_job_endpoint.py | 2 +- .../server/test_workspace_endpoint.py | 217 ++++++++---------- tests/testdoubles/_browserfactory.py | 20 -- .../testdoubles/_browserprocessrepository.py | 1 + tests/testdoubles/_browserspy.py | 3 +- tests/testdoubles/_registrybrowserfactory.py | 1 - 11 files changed, 146 insertions(+), 255 deletions(-) create mode 100644 tests/ocrdmonitor/server/fixtures/settings.py diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index 209d5f1..d7c2541 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -1,9 +1,7 @@ -from .fixtures.app import app, create_settings # noqa: F401 -from .fixtures.factory import patch_factory # noqa: F401 +from .fixtures.app import app # noqa: F401 from .fixtures.repository import ( - auto_repository, # noqa: F401 + # auto_repository, # noqa: F401 inmemory_repository, # noqa: F401 mongodb_repository, # noqa: F401 patch_repository, # noqa: F401 - singleton_restoring_factory, # noqa: F401 ) diff --git a/tests/ocrdmonitor/server/fixtures/app.py b/tests/ocrdmonitor/server/fixtures/app.py index 5d805a8..0cb9e6e 100644 --- a/tests/ocrdmonitor/server/fixtures/app.py +++ b/tests/ocrdmonitor/server/fixtures/app.py @@ -1,50 +1,12 @@ -from pathlib import Path -from typing import Iterator +from typing import AsyncIterator -import pytest -import uvicorn +import pytest_asyncio from fastapi.testclient import TestClient -from ocrdmonitor.server.app import create_app -from ocrdmonitor.server.settings import ( - OcrdBrowserSettings, - OcrdControllerSettings, - OcrdLogViewSettings, - Settings, -) -from tests.testdoubles import BackgroundProcess +from tests.ocrdmonitor.server.fixtures.environment import Fixture -JOB_DIR = Path(__file__).parent / "ocrd.jobs" -WORKSPACE_DIR = Path("tests") / "workspaces" - -def create_settings() -> Settings: - return Settings( - ocrd_browser=OcrdBrowserSettings( - workspace_dir=WORKSPACE_DIR, - port_range=(9000, 9100), - db_connection_string="", - ), - ocrd_controller=OcrdControllerSettings( - job_dir=JOB_DIR, - host="", - user="", - ), - ocrd_logview=OcrdLogViewSettings(port=8022), - ) - - -@pytest.fixture -def app() -> TestClient: - return TestClient(create_app(create_settings())) - - -def _launch_app() -> None: - app = create_app(create_settings()) - uvicorn.run(app, port=3000) - - -@pytest.fixture -def launch_monitor() -> Iterator[None]: - with BackgroundProcess(_launch_app): - yield +@pytest_asyncio.fixture +async def app() -> AsyncIterator[TestClient]: + async with Fixture() as env: + yield env.app diff --git a/tests/ocrdmonitor/server/fixtures/environment.py b/tests/ocrdmonitor/server/fixtures/environment.py index 9bdf97a..134eba9 100644 --- a/tests/ocrdmonitor/server/fixtures/environment.py +++ b/tests/ocrdmonitor/server/fixtures/environment.py @@ -11,15 +11,14 @@ from fastapi.testclient import TestClient -from ocrdmonitor.browserprocess import BrowserProcessRepository +from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory from ocrdmonitor.server.app import create_app -from tests.ocrdmonitor.server.fixtures.app import create_settings from tests.ocrdmonitor.server.fixtures.factory import patch_factory from tests.ocrdmonitor.server.fixtures.repository import ( - RepositoryInitializer, inmemory_repository, patch_repository, ) +from tests.ocrdmonitor.server.fixtures.settings import create_settings from tests.testdoubles import ( BrowserRegistry, BrowserSpy, @@ -37,6 +36,10 @@ class Environment: BrowserConstructor = Callable[[], BrowserTestDouble] +RepositoryInitializer = Callable[ + [BrowserRestoringFactory], + AsyncContextManager[BrowserProcessRepository], +] class Fixture: diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index 0d6aa3a..1daf695 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -1,25 +1,13 @@ from contextlib import asynccontextmanager -from typing import AsyncContextManager, AsyncIterator, Callable +from typing import Any, AsyncIterator, Iterator from unittest.mock import patch -import pytest -import pytest_asyncio from testcontainers.mongodb import MongoDbContainer -from ocrdbrowser import OcrdBrowser from ocrdmonitor import dbmodel from ocrdmonitor.browserprocess import BrowserProcessRepository, BrowserRestoringFactory from ocrdmonitor.server.settings import OcrdBrowserSettings -from tests.testdoubles import ( - BrowserSpy, - InMemoryBrowserProcessRepository, -) -from tests.testdoubles._browserfactory import SingletonRestoringBrowserFactory - -RepositoryInitializer = Callable[ - [BrowserRestoringFactory], - AsyncContextManager[BrowserProcessRepository], -] +from tests.testdoubles import InMemoryBrowserProcessRepository @asynccontextmanager @@ -38,15 +26,6 @@ async def inmemory_repository( yield InMemoryBrowserProcessRepository(restoring_factory) -def spy_restoring_factory() -> BrowserRestoringFactory: - def factory( - owner: str, workspace: str, address: str, process_id: str - ) -> OcrdBrowser: - return BrowserSpy(owner, workspace, address, process_id, running=True) - - return factory - - @asynccontextmanager async def patch_repository(repository: BrowserProcessRepository) -> AsyncIterator[None]: async def _repository(_: OcrdBrowserSettings) -> BrowserProcessRepository: @@ -54,41 +33,3 @@ async def _repository(_: OcrdBrowserSettings) -> BrowserProcessRepository: with patch.object(OcrdBrowserSettings, "repository", _repository): yield - - -@pytest_asyncio.fixture( - autouse=True, - params=[ - inmemory_repository, - pytest.param( - mongodb_repository, - marks=(pytest.mark.integration, pytest.mark.needs_docker), - ), - ], -) -async def auto_repository( - request: pytest.FixtureRequest, -) -> AsyncIterator[BrowserProcessRepository] | None: - """ - This fixture will be used automatically for all tests, - as a repository for browser processes is pretty much always needed. - - It can be turned off by marking a test with 'pytest.mark.no_auto_repository' - """ - if "no_auto_repository" in request.keywords: - # NOTE: we're yielding 0 here, because pytest_asyncio - # raises a StopIterationError if we return or yield None - yield 0 - else: - repository_constructor: RepositoryInitializer = request.param - async with repository_constructor(spy_restoring_factory()) as repository: - async with patch_repository(repository): - yield repository - - -@pytest_asyncio.fixture -async def singleton_restoring_factory() -> AsyncIterator[ - SingletonRestoringBrowserFactory -]: - async with SingletonRestoringBrowserFactory() as factory: - yield factory diff --git a/tests/ocrdmonitor/server/fixtures/settings.py b/tests/ocrdmonitor/server/fixtures/settings.py new file mode 100644 index 0000000..aab973b --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/settings.py @@ -0,0 +1,27 @@ +from pathlib import Path + +from ocrdmonitor.server.settings import ( + OcrdBrowserSettings, + OcrdControllerSettings, + OcrdLogViewSettings, + Settings, +) + +JOB_DIR = Path(__file__).parent / "ocrd.jobs" +WORKSPACE_DIR = Path("tests") / "workspaces" + + +def create_settings() -> Settings: + return Settings( + ocrd_browser=OcrdBrowserSettings( + workspace_dir=WORKSPACE_DIR, + port_range=(9000, 9100), + db_connection_string="", + ), + ocrd_controller=OcrdControllerSettings( + job_dir=JOB_DIR, + host="", + user="", + ), + ocrd_logview=OcrdLogViewSettings(port=8022), + ) diff --git a/tests/ocrdmonitor/server/test_job_endpoint.py b/tests/ocrdmonitor/server/test_job_endpoint.py index 6a34cfe..930adc8 100644 --- a/tests/ocrdmonitor/server/test_job_endpoint.py +++ b/tests/ocrdmonitor/server/test_job_endpoint.py @@ -14,7 +14,7 @@ from ocrdmonitor.processstatus import ProcessState, ProcessStatus from ocrdmonitor.server.settings import OcrdControllerSettings from tests.ocrdmonitor.server import scraping -from tests.ocrdmonitor.server.fixtures.app import JOB_DIR +from tests.ocrdmonitor.server.fixtures.settings import JOB_DIR from tests.ocrdmonitor.test_jobs import JOB_TEMPLATE, jobfile_content_for diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index 4d1d034..e79139d 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -1,6 +1,7 @@ from __future__ import annotations -from typing import AsyncIterator, cast +import asyncio +from typing import AsyncIterator import pytest import pytest_asyncio @@ -9,58 +10,52 @@ from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.decorators import use_custom_repository -from tests.ocrdmonitor.server.fixtures.app import WORKSPACE_DIR -from tests.ocrdmonitor.server.fixtures.environment import Environment, Fixture -from tests.ocrdmonitor.server.fixtures.factory import patch_factory -from tests.ocrdmonitor.server.fixtures.repository import ( +from tests.ocrdmonitor.server.fixtures.environment import ( + Environment, + Fixture, RepositoryInitializer, ) +from tests.ocrdmonitor.server.fixtures.repository import ( + inmemory_repository, + mongodb_repository, +) +from tests.ocrdmonitor.server.fixtures.settings import WORKSPACE_DIR from tests.testdoubles import ( Browser_Heading, BrowserFake, BrowserSpy, - BrowserTestDouble, - BrowserTestDoubleFactory, - IteratingBrowserTestDoubleFactory, browser_with_disconnecting_channel, unreachable_browser, ) -@pytest_asyncio.fixture( - params=(BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)) +@pytest.fixture( + params=[ + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, pytest.mark.needs_docker), + ), + ] ) -async def iterating_factory( - request: pytest.FixtureRequest, -) -> AsyncIterator[BrowserTestDoubleFactory]: - async with patch_factory( - IteratingBrowserTestDoubleFactory(default_browser=request.param) - ) as factory: - yield factory +def repository_fixture(request: pytest.FixtureRequest) -> Fixture: + repository: RepositoryInitializer = request.param + return Fixture().with_repository_type(repository) @pytest.fixture( - params=(BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)) + params=[BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)] ) -def browser( - iterating_factory: IteratingBrowserTestDoubleFactory, - request: pytest.FixtureRequest, -) -> BrowserTestDouble: - browser_type = request.param - browser = cast(BrowserTestDouble, browser_type()) - iterating_factory.add(browser) - - return browser +def browser_fixture( + repository_fixture: Fixture, request: pytest.FixtureRequest +) -> Fixture: + return repository_fixture.with_browser_type(request.param) -@pytest.fixture -def disconnecting_browser( - iterating_factory: IteratingBrowserTestDoubleFactory, -) -> BrowserSpy: - disconnecting_browser = browser_with_disconnecting_channel() - iterating_factory.add(disconnecting_browser) - - return disconnecting_browser +@pytest_asyncio.fixture +async def app(browser_fixture: Fixture) -> AsyncIterator[TestClient]: + async with browser_fixture as env: + yield env.app def assert_is_browser_response(actual: Response) -> None: @@ -85,8 +80,8 @@ def view_workspace(app: TestClient, workspace: str) -> Response: @pytest_asyncio.fixture -async def defaultenv() -> AsyncIterator[Environment]: - async with Fixture() as env: +async def defaultenv(browser_fixture: Fixture) -> AsyncIterator[Environment]: + async with browser_fixture as env: yield env @@ -99,21 +94,28 @@ def test__workspaces__shows_the_workspace_names_starting_from_workspace_root( assert set(texts) == {"a_workspace", "another workspace", "nested/workspace"} -def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( - browser: BrowserTestDouble, - app: TestClient, +@use_custom_repository +async def test__browse_workspace__passes_full_workspace_path_to_ocrdbrowser( + repository: RepositoryInitializer, ) -> None: - response = open_workspace(app, "a_workspace") + workspace = "a_workspace" + full_workspace = str(WORKSPACE_DIR / workspace) + browser = BrowserSpy() + fixture = ( + Fixture().with_repository_type(repository).with_browser_type(lambda: browser) + ) + + async with fixture as env: + response = open_workspace(env.app, workspace) - assert browser.is_running is True - assert browser.workspace() == str(WORKSPACE_DIR / "a_workspace") - assert response.status_code == 200 + assert browser.is_running is True + assert browser.workspace() == full_workspace + assert response.status_code == 200 def test__browse_workspace__assigns_and_tracks_session_id( - defaultenv: Environment, + app: TestClient, ) -> None: - app = defaultenv.app response = open_workspace(app, "a_workspace") first_session_id = response.cookies.get("session_id") @@ -125,20 +127,17 @@ def test__browse_workspace__assigns_and_tracks_session_id( @pytest.mark.asyncio -@use_custom_repository async def test__opened_workspace__when_socket_disconnects__shuts_down_browser( - repository: RepositoryInitializer, + browser_fixture: Fixture, ) -> None: session_id = "the-owner" disconnecting_browser = browser_with_disconnecting_channel( session_id, str(WORKSPACE_DIR / "a_workspace") ) - fixture = ( - Fixture() - .with_repository_type(repository) - .with_running_browsers(disconnecting_browser) - .with_session_id(session_id) - ) + + fixture = browser_fixture.with_running_browsers( + disconnecting_browser + ).with_session_id(session_id) async with fixture as env: _ = interact_with_workspace(env.app, "a_workspace") @@ -146,25 +145,30 @@ async def test__opened_workspace__when_socket_disconnects__shuts_down_browser( assert disconnecting_browser.is_running is False -@pytest.mark.usefixtures("disconnecting_browser") -def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_to_browser( - app: TestClient, +@pytest.mark.asyncio +async def test__disconnected_workspace__when_opening_again__viewing_proxies_requests_to_browser( + browser_fixture: Fixture, ) -> None: + session_id = "the-owner" workspace = "a_workspace" - _ = interact_with_workspace(app, workspace) + full_workspace = str(WORKSPACE_DIR / workspace) + fixture = browser_fixture.with_running_browsers( + browser_with_disconnecting_channel(session_id, full_workspace) + ).with_session_id(session_id) - actual = interact_with_workspace(app, workspace) + async with fixture as env: + _ = interact_with_workspace(env.app, workspace) + + actual = interact_with_workspace(env.app, workspace) assert_is_browser_response(actual) @pytest.mark.asyncio -@use_custom_repository async def test__when_requesting_resource__returns_resource_from_workspace( - repository: RepositoryInitializer, + browser_fixture: Fixture, ) -> None: session_id = "the-owner" - workspace = "a_workspace" resource = "/some_resource" resource_in_workspace = workspace + "/some_resource" @@ -176,12 +180,7 @@ def echo_bytes(path: str) -> bytes: browser = BrowserSpy(session_id, full_workspace) browser.configure_client(response_factory=echo_bytes) - fixture = ( - Fixture() - .with_repository_type(repository) - .with_running_browsers(browser) - .with_session_id(session_id) - ) + fixture = browser_fixture.with_running_browsers(browser).with_session_id(session_id) async with fixture as env: open_workspace(env.app, workspace) @@ -192,9 +191,8 @@ def echo_bytes(path: str) -> bytes: def test__browsed_workspace_is_ready__when_pinging__returns_ok( - defaultenv: Environment, + app: TestClient, ) -> None: - app = defaultenv.app workspace = "a_workspace" _ = interact_with_workspace(app, workspace) @@ -203,16 +201,12 @@ def test__browsed_workspace_is_ready__when_pinging__returns_ok( assert result.status_code == 200 -@use_custom_repository +@pytest.mark.asyncio async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( - repository: RepositoryInitializer, + repository_fixture: Fixture, ) -> None: workspace = "a_workspace" - fixture = ( - Fixture() - .with_browser_type(unreachable_browser) - .with_repository_type(repository) - ) + fixture = repository_fixture.with_browser_type(unreachable_browser) async with fixture as env: open_workspace(env.app, workspace) @@ -222,70 +216,60 @@ async def test__browsed_workspace_not_ready__when_pinging__returns_bad_gateway( assert result.status_code == 502 -@use_custom_repository +@pytest.mark.asyncio async def test__browsing_workspace__stores_browser_in_repository( - repository: RepositoryInitializer, + defaultenv: Environment, ) -> None: - fixture = Fixture().with_repository_type(repository) - - async with fixture as env: - _ = interact_with_workspace(env.app, "a_workspace") + _ = interact_with_workspace(defaultenv.app, "a_workspace") - found_browsers = list( - await env.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) - ) + found_browsers = list( + await defaultenv.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + ) assert len(found_browsers) == 1 -@use_custom_repository +@pytest.mark.asyncio async def test__error_connecting_to_workspace__removes_browser_from_repository( - repository: RepositoryInitializer, + repository_fixture: Fixture, ) -> None: - fixture = ( - Fixture() - .with_browser_type(unreachable_browser) - .with_repository_type(repository) - ) - + fixture = repository_fixture.with_browser_type(unreachable_browser) async with fixture as env: open_workspace(env.app, "a_workspace") _ = view_workspace(env.app, "a_workspace") - found_browsers = list( - await env.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) + browsers = await env.repository.find( + workspace=str(WORKSPACE_DIR / "a_workspace") ) - assert len(found_browsers) == 0 + assert len(list(browsers)) == 0 -@use_custom_repository +@pytest.mark.asyncio async def test__when_socket_to_workspace_disconnects__removes_browser_from_repository( - repository: RepositoryInitializer, + repository_fixture: Fixture, ) -> None: - # NOTE: it seems something is weird with the event loop in this test. - # Searching for browsers inside the with block happens BEFORE the browser is deleted. - # Therefore we run the lookup after the contextmanager has been closed + # NOTE: it seems something is weird with the event loop in this test + # Searching for browsers inside the with block happens BEFORE the browser is deleted + # I'm not sure if this is a bug in the FastAPI TestClient or if we're doing something wrong here + # We apply a little hack and sleep for .1 seconds, handing control back to the event loop - fixture = ( - Fixture() - .with_repository_type(repository) - .with_browser_type(browser_with_disconnecting_channel) - ) + fixture = repository_fixture.with_browser_type(browser_with_disconnecting_channel) async with fixture as env: _ = interact_with_workspace(env.app, "a_workspace") + await asyncio.sleep(0.1) - found_browsers = list( - await env.repository.find(workspace=str(WORKSPACE_DIR / "a_workspace")) - ) + browsers = await env.repository.find( + workspace=str(WORKSPACE_DIR / "a_workspace") + ) - assert len(found_browsers) == 0 + assert len(list(browsers)) == 0 -@use_custom_repository +@pytest.mark.asyncio async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_restored_browser( - repository: RepositoryInitializer, + browser_fixture: Fixture, ) -> None: session_id = "the-owner" workspace = "a_workspace" @@ -293,12 +277,7 @@ async def test__browser_stored_in_repo__when_browsing_workspace_redirects_to_res browser = BrowserSpy(session_id, full_workspace) browser.configure_client(response=b"RESTORED BROWSER") - fixture = ( - Fixture() - .with_repository_type(repository) - .with_running_browsers(browser) - .with_session_id(session_id) - ) + fixture = browser_fixture.with_running_browsers(browser).with_session_id(session_id) async with fixture as env: response = interact_with_workspace(env.app, "a_workspace") diff --git a/tests/testdoubles/_browserfactory.py b/tests/testdoubles/_browserfactory.py index 0bbba57..df7e550 100644 --- a/tests/testdoubles/_browserfactory.py +++ b/tests/testdoubles/_browserfactory.py @@ -58,23 +58,3 @@ class BrowserTestDoubleFactory( OcrdBrowserFactory, AsyncContextManager[OcrdBrowserFactory], Protocol ): pass - - -class SingletonRestoringBrowserFactory: - def __init__(self) -> None: - self.browser = BrowserSpy() - - async def __aenter__(self) -> "SingletonRestoringBrowserFactory": - await self.browser.start() - return self - - async def __aexit__(self, *args: Any, **kwargs: Any) -> None: - await self.browser.stop() - - def __call__( - self, owner: str, workspace: str, address: str, process_id: str - ) -> OcrdBrowser: - self.browser.set_owner_and_workspace(owner, workspace) - self.browser._address = address - self.browser._process_id = process_id - return self.browser diff --git a/tests/testdoubles/_browserprocessrepository.py b/tests/testdoubles/_browserprocessrepository.py index f6f8ec4..eb8a09e 100644 --- a/tests/testdoubles/_browserprocessrepository.py +++ b/tests/testdoubles/_browserprocessrepository.py @@ -58,6 +58,7 @@ def match(browser: BrowserEntry) -> bool: return matches + print(self._processes) return [ self.restoring_factory( process_id=browser.process_id, diff --git a/tests/testdoubles/_browserspy.py b/tests/testdoubles/_browserspy.py index a3fcb5d..58a27de 100644 --- a/tests/testdoubles/_browserspy.py +++ b/tests/testdoubles/_browserspy.py @@ -115,6 +115,7 @@ def __repr__(self) -> str: workspace: {self.workspace()} owner: {self.owner()} running: {self.is_running} + process id: {self._process_id} """ ) @@ -126,7 +127,7 @@ def browser_with_disconnecting_channel( process_id: str = "1234", ) -> BrowserSpy: spy = BrowserSpy(owner, workspace, address, process_id) - spy.configure_client(channel=DisconnectingChannel()) + spy.configure_client(response=b"Disconnected", channel=DisconnectingChannel()) return spy diff --git a/tests/testdoubles/_registrybrowserfactory.py b/tests/testdoubles/_registrybrowserfactory.py index f3dcd00..82b9f06 100644 --- a/tests/testdoubles/_registrybrowserfactory.py +++ b/tests/testdoubles/_registrybrowserfactory.py @@ -13,7 +13,6 @@ class RegistryBrowserFactory: - @classmethod def iteratingfactory(cls: Type[Self], browser_registry: BrowserRegistry) -> Self: return cls(IteratingBrowserTestDoubleFactory(), browser_registry) From 9246c618b49f8d64ce29320e0a24fac359354c50 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Tue, 18 Jul 2023 17:12:20 +0200 Subject: [PATCH 31/32] Refactor existing tests for Fixture API --- tests/ocrdmonitor/server/fixtures/repository.py | 2 +- tests/testdoubles/_browserfactory.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ocrdmonitor/server/fixtures/repository.py b/tests/ocrdmonitor/server/fixtures/repository.py index 1daf695..4a1d6bb 100644 --- a/tests/ocrdmonitor/server/fixtures/repository.py +++ b/tests/ocrdmonitor/server/fixtures/repository.py @@ -1,5 +1,5 @@ from contextlib import asynccontextmanager -from typing import Any, AsyncIterator, Iterator +from typing import AsyncIterator from unittest.mock import patch from testcontainers.mongodb import MongoDbContainer diff --git a/tests/testdoubles/_browserfactory.py b/tests/testdoubles/_browserfactory.py index df7e550..6807a02 100644 --- a/tests/testdoubles/_browserfactory.py +++ b/tests/testdoubles/_browserfactory.py @@ -1,6 +1,6 @@ import asyncio from types import TracebackType -from typing import Any, AsyncContextManager, Callable, Protocol, Self, Type +from typing import AsyncContextManager, Callable, Protocol, Self, Type from ocrdbrowser import OcrdBrowser, OcrdBrowserFactory From f6babea9a767f40cee04cd3720e75e6eebd7ec03 Mon Sep 17 00:00:00 2001 From: Sven Marcus Date: Wed, 19 Jul 2023 14:13:00 +0000 Subject: [PATCH 32/32] Remove unnecessary markers --- Makefile | 2 +- pyproject.toml | 4 +- tests/markers.py | 24 +++++++++++ tests/ocrdbrowser/test_browser_launch.py | 28 ++----------- tests/ocrdmonitor/server/conftest.py | 10 ++--- tests/ocrdmonitor/server/decorators.py | 4 +- tests/ocrdmonitor/server/fixtures/app.py | 12 ------ .../server/fixtures/fixtureconfig.py | 40 +++++++++++++++++++ tests/ocrdmonitor/server/test_startup.py | 15 +++---- .../server/test_workspace_endpoint.py | 3 +- tests/ocrdmonitor/test_sshremote.py | 3 +- 11 files changed, 88 insertions(+), 57 deletions(-) create mode 100644 tests/markers.py delete mode 100644 tests/ocrdmonitor/server/fixtures/app.py create mode 100644 tests/ocrdmonitor/server/fixtures/fixtureconfig.py diff --git a/Makefile b/Makefile index 44dd29f..c883c78 100644 --- a/Makefile +++ b/Makefile @@ -73,7 +73,7 @@ test: { echo set -e; \ echo cd /usr/local/ocrd-monitor/; \ echo pip install nox; \ - echo "nox -- -m 'not needs_docker'"; } | \ + echo "nox"; } | \ docker run --rm -i \ $(TAGNAME) bash diff --git a/pyproject.toml b/pyproject.toml index f686848..9a3dbff 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,9 +42,7 @@ line-length = 100 [tool.pytest.ini_options] markers = [ - "integration: mark test as integration test", - "needs_docker: marks tests that need access to Docker in order to run", - "no_auto_repository: marks test to not automatically use the auto_repository fixture" + "integration: mark test as integration test" ] [tool.pdm.scripts] diff --git a/tests/markers.py b/tests/markers.py new file mode 100644 index 0000000..39c7cfa --- /dev/null +++ b/tests/markers.py @@ -0,0 +1,24 @@ +import shutil + +import pytest + + +def browse_ocrd_not_available() -> bool: + browse_ocrd = shutil.which("browse-ocrd") + broadway = shutil.which("broadwayd") + return not all((browse_ocrd, broadway)) + + +def docker_not_available() -> bool: + return not bool(shutil.which("docker")) + + +skip_if_no_docker = pytest.mark.skipif( + docker_not_available(), + reason="Skipping because Docker is not available", +) + +skip_if_no_browse_ocrd = pytest.mark.skipif( + browse_ocrd_not_available(), + reason="Skipping because browse-ocrd or broadwayd are not available", +) diff --git a/tests/ocrdbrowser/test_browser_launch.py b/tests/ocrdbrowser/test_browser_launch.py index f0f2cc1..95cfa1e 100644 --- a/tests/ocrdbrowser/test_browser_launch.py +++ b/tests/ocrdbrowser/test_browser_launch.py @@ -1,7 +1,6 @@ import asyncio import functools -import shutil -from typing import AsyncIterator, Callable, Literal, NamedTuple +from typing import AsyncIterator, Callable, NamedTuple import pytest import pytest_asyncio @@ -12,19 +11,10 @@ OcrdBrowserFactory, SubProcessOcrdBrowserFactory, ) +from tests import markers from tests.decorators import compose -def browse_ocrd_not_available() -> bool: - browse_ocrd = shutil.which("browse-ocrd") - broadway = shutil.which("broadwayd") - return not all((browse_ocrd, broadway)) - - -def docker_not_available() -> bool: - return not bool(shutil.which("docker")) - - create_docker_browser_factory = functools.partial( DockerOcrdBrowserFactory, "http://localhost" ) @@ -37,20 +27,10 @@ def docker_not_available() -> bool: ( pytest.param( create_docker_browser_factory, - marks=( - pytest.mark.needs_docker, - pytest.mark.skipif( - docker_not_available(), - reason="Skipping because Docker is not available", - ), - ), + marks=markers.skip_if_no_docker, ), pytest.param( - SubProcessOcrdBrowserFactory, - marks=pytest.mark.skipif( - browse_ocrd_not_available(), - reason="Skipping because browse-ocrd or broadwayd are not available", - ), + SubProcessOcrdBrowserFactory, marks=markers.skip_if_no_browse_ocrd ), ), ), diff --git a/tests/ocrdmonitor/server/conftest.py b/tests/ocrdmonitor/server/conftest.py index d7c2541..8197e28 100644 --- a/tests/ocrdmonitor/server/conftest.py +++ b/tests/ocrdmonitor/server/conftest.py @@ -1,7 +1,5 @@ -from .fixtures.app import app # noqa: F401 -from .fixtures.repository import ( - # auto_repository, # noqa: F401 - inmemory_repository, # noqa: F401 - mongodb_repository, # noqa: F401 - patch_repository, # noqa: F401 +from .fixtures.fixtureconfig import ( + app, # noqa: F401 + browser_fixture, # noqa: F401 + repository_fixture, # noqa: F401 ) diff --git a/tests/ocrdmonitor/server/decorators.py b/tests/ocrdmonitor/server/decorators.py index 4ed289b..53be862 100644 --- a/tests/ocrdmonitor/server/decorators.py +++ b/tests/ocrdmonitor/server/decorators.py @@ -1,4 +1,5 @@ import pytest +from tests import markers from tests.decorators import compose from tests.ocrdmonitor.server.fixtures.repository import ( @@ -9,14 +10,13 @@ use_custom_repository = compose( pytest.mark.asyncio, - pytest.mark.no_auto_repository, pytest.mark.parametrize( "repository", ( pytest.param(inmemory_repository), pytest.param( mongodb_repository, - marks=(pytest.mark.integration, pytest.mark.needs_docker), + marks=(pytest.mark.integration, markers.skip_if_no_docker), ), ), ), diff --git a/tests/ocrdmonitor/server/fixtures/app.py b/tests/ocrdmonitor/server/fixtures/app.py deleted file mode 100644 index 0cb9e6e..0000000 --- a/tests/ocrdmonitor/server/fixtures/app.py +++ /dev/null @@ -1,12 +0,0 @@ -from typing import AsyncIterator - -import pytest_asyncio -from fastapi.testclient import TestClient - -from tests.ocrdmonitor.server.fixtures.environment import Fixture - - -@pytest_asyncio.fixture -async def app() -> AsyncIterator[TestClient]: - async with Fixture() as env: - yield env.app diff --git a/tests/ocrdmonitor/server/fixtures/fixtureconfig.py b/tests/ocrdmonitor/server/fixtures/fixtureconfig.py new file mode 100644 index 0000000..44f3fe9 --- /dev/null +++ b/tests/ocrdmonitor/server/fixtures/fixtureconfig.py @@ -0,0 +1,40 @@ +from typing import AsyncIterator + +import pytest +import pytest_asyncio +from fastapi.testclient import TestClient + +from tests import markers +from tests.testdoubles import BrowserFake, BrowserSpy + +from .environment import Fixture, RepositoryInitializer +from .repository import inmemory_repository, mongodb_repository + + +@pytest.fixture( + params=[ + inmemory_repository, + pytest.param( + mongodb_repository, + marks=(pytest.mark.integration, markers.skip_if_no_docker), + ), + ] +) +def repository_fixture(request: pytest.FixtureRequest) -> Fixture: + repository: RepositoryInitializer = request.param + return Fixture().with_repository_type(repository) + + +@pytest.fixture( + params=[BrowserSpy, pytest.param(BrowserFake, marks=pytest.mark.integration)] +) +def browser_fixture( + repository_fixture: Fixture, request: pytest.FixtureRequest +) -> Fixture: + return repository_fixture.with_browser_type(request.param) + + +@pytest_asyncio.fixture +async def app() -> AsyncIterator[TestClient]: + async with Fixture() as env: + yield env.app diff --git a/tests/ocrdmonitor/server/test_startup.py b/tests/ocrdmonitor/server/test_startup.py index 2c3e064..0db4568 100644 --- a/tests/ocrdmonitor/server/test_startup.py +++ b/tests/ocrdmonitor/server/test_startup.py @@ -5,17 +5,18 @@ @pytest.mark.asyncio -@pytest.mark.no_auto_repository -async def test__browsers_in_db__on_startup__cleans_unreachables_from_db() -> None: +async def test__browsers_in_db__on_startup__cleans_unreachables_from_db( + repository_fixture: Fixture, +) -> None: session_id = "the-owner" reachable = BrowserSpy(owner=session_id, address="http://reachable.com") unreachable = unreachable_browser( owner=session_id, address="http://unreachable.com" ) - async with ( - Fixture() - .with_running_browsers(reachable, unreachable) - .with_session_id(session_id) - ) as env: + fixture = repository_fixture.with_running_browsers( + reachable, unreachable + ).with_session_id(session_id) + + async with fixture as env: assert await env.repository.count() == 1 diff --git a/tests/ocrdmonitor/server/test_workspace_endpoint.py b/tests/ocrdmonitor/server/test_workspace_endpoint.py index e79139d..8287c41 100644 --- a/tests/ocrdmonitor/server/test_workspace_endpoint.py +++ b/tests/ocrdmonitor/server/test_workspace_endpoint.py @@ -7,6 +7,7 @@ import pytest_asyncio from fastapi.testclient import TestClient from httpx import Response +from tests import markers from tests.ocrdmonitor.server import scraping from tests.ocrdmonitor.server.decorators import use_custom_repository @@ -34,7 +35,7 @@ inmemory_repository, pytest.param( mongodb_repository, - marks=(pytest.mark.integration, pytest.mark.needs_docker), + marks=(pytest.mark.integration, markers.skip_if_no_docker), ), ] ) diff --git a/tests/ocrdmonitor/test_sshremote.py b/tests/ocrdmonitor/test_sshremote.py index ab648be..92a69c2 100644 --- a/tests/ocrdmonitor/test_sshremote.py +++ b/tests/ocrdmonitor/test_sshremote.py @@ -6,6 +6,7 @@ from ocrdmonitor.processstatus import ProcessState from ocrdmonitor.sshremote import SSHRemote +from tests import markers from tests.ocrdmonitor.sshcontainer import ( get_process_group_from_container, SSHConfig, @@ -17,7 +18,7 @@ @pytest.mark.asyncio @pytest.mark.integration -@pytest.mark.needs_docker +@markers.skip_if_no_docker async def test_ps_over_ssh__returns_list_of_process_status( openssh_server: DockerContainer, ) -> None: