Improve parallelisation (#25)

* Improve parallelisation * Fix mypy * Fix deps * Fix deps * Prepare for pyTMD fix * Refactor repo structure * Fix imports
GeoscienceAustralia · Nov 3, 2024 · 0b69bfd · 0b69bfd
1 parent 7cfeac2
commit 0b69bfd
Show file tree

Hide file tree

Showing 17 changed files with 8,535 additions and 734 deletions.
diff --git a/Makefile b/Makefile
@@ -50,20 +50,27 @@ test-eo: ## Test eo module with pytest
 	@export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && \
 	uv run python -m pytest tests/test_eo.py --verbose
 
+.PHONY: test-stats
+test-stats: ## Test stats module with pytest
+	@echo "🚀 Testing stats module: Running pytest"
+	@tar --skip-old-files -xzf ./tests/data/tide_models.tar.gz -C ./tests/data
+	@export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && \
+	uv run python -m pytest tests/test_stats.py --verbose
+
+.PHONY: test-utils
+test-utils: ## Test utils module with pytest
+	@echo "🚀 Testing utils module: Running pytest"
+	@tar --skip-old-files -xzf ./tests/data/tide_models.tar.gz -C ./tests/data
+	@export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && \
+	uv run python -m pytest tests/test_utils.py --verbose
+
 .PHONY: test-notebooks
 test-notebooks: ## Test notebooks with pytest
 	@echo "🚀 Testing notebooks: Running pytest"
 	@tar --skip-old-files -xzf ./tests/data/tide_models.tar.gz -C ./tests/data
 	@export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && \
     uv run python -m pytest --nbval-lax docs/notebooks/ --verbose
 
-.PHONY: test-stats
-test-stats: ## Test eo module with pytest
-	@echo "🚀 Testing eo module: Running pytest"
-	@tar --skip-old-files -xzf ./tests/data/tide_models.tar.gz -C ./tests/data
-	@export EO_TIDES_TIDE_MODELS=./tests/data/tide_models && \
-	uv run python -m pytest tests/test_stats.py --verbose
-
 .PHONY: build
 build: clean-build ## Build wheel file
 	@echo "🚀 Creating wheel file"

diff --git a/docs/notebooks/Case_study_intertidal.ipynb b/docs/notebooks/Case_study_intertidal.ipynb
@@ -51,7 +51,7 @@
     "\n",
     "from eo_tides.eo import tag_tides\n",
     "from eo_tides.stats import tide_stats\n",
-    "from eo_tides.model import list_models"
+    "from eo_tides.utils import list_models"
    ]
   },
   {

diff --git a/docs/notebooks/Model_tides.ipynb b/docs/notebooks/Model_tides.ipynb
@@ -51,7 +51,20 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "We can use the [`eo_tides.model.list_models`](../../api/#eo_tides.model.list_models) function to verify that we have some tide model data available in this directory:"
+    "<div class=\"admonition important\">\n",
+    "    <p class=\"admonition-title\">Important</p>\n",
+    "    <p>\n",
+    "        The directory above is for demo purposes only.\n",
+    "        Update the <code>directory</code> path to point to the location of your own tide model directory.\n",
+    "    </p>\n",
+    "</div>"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "We can use the [`eo_tides.utils.list_models`](../../api/#eo_tides.utils.list_models) function to verify that we have some tide model data available in this directory:"
    ]
   },
   {
@@ -79,24 +92,11 @@
     }
    ],
    "source": [
-    "from eo_tides.model import list_models\n",
+    "from eo_tides.utils import list_models\n",
     "\n",
     "list_models(directory=directory, show_supported=False);"
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {},
-   "source": [
-    "<div class=\"admonition important\">\n",
-    "    <p class=\"admonition-title\">Important</p>\n",
-    "    <p>\n",
-    "        The directory above contains demo data only.\n",
-    "        Update the <code>directory</code> path to point to the location of your own tide model directory.\n",
-    "    </p>\n",
-    "</div>"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -117,7 +117,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20\n"
+      "Modelling tides with EOT20\n"
      ]
     },
     {
@@ -294,14 +294,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20, HAMTIDE11, GOT5.5 in parallel\n"
+      "Modelling tides with EOT20, HAMTIDE11, GOT5.5 in parallel (models: 3, splits: 1)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 3/3 [00:00<00:00, 22.35it/s]\n"
+      "100%|██████████| 3/3 [00:00<00:00, 18.09it/s]\n"
      ]
     },
     {
@@ -487,14 +487,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20 in parallel\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 2/2 [00:00<00:00, 16.90it/s]\n"
+      "Modelling tides with EOT20\n"
      ]
     },
     {
@@ -691,14 +684,7 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20 in parallel\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "100%|██████████| 2/2 [00:00<00:00, 16.66it/s]\n"
+      "Modelling tides with EOT20\n"
      ]
     },
     {
@@ -791,14 +777,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20, GOT5.5, HAMTIDE11 in parallel\n"
+      "Modelling tides with EOT20, GOT5.5, HAMTIDE11 in parallel (models: 3, splits: 1)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 6/6 [00:00<00:00, 29.64it/s]\n"
+      "100%|██████████| 3/3 [00:00<00:00, 18.19it/s]\n"
      ]
     },
     {
@@ -977,14 +963,14 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20, GOT5.5, HAMTIDE11 in parallel\n"
+      "Modelling tides with EOT20, GOT5.5, HAMTIDE11 in parallel (models: 3, splits: 1)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 6/6 [00:00<00:00, 26.52it/s]\n"
+      "100%|██████████| 3/3 [00:00<00:00, 18.12it/s]\n"
      ]
     },
     {
@@ -1115,8 +1101,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20\n",
-      "Modelling tides using EOT20\n"
+      "Modelling tides with EOT20\n",
+      "Modelling tides with EOT20\n"
      ]
     },
     {
@@ -1241,28 +1227,28 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20, GOT5.5 in parallel\n"
+      "Modelling tides with EOT20, GOT5.5 in parallel (models: 2, splits: 1)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 2/2 [00:00<00:00, 15.78it/s]\n"
+      "100%|██████████| 2/2 [00:00<00:00, 12.91it/s]\n"
      ]
     },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20, GOT5.5 in parallel\n"
+      "Modelling tides with EOT20, GOT5.5 in parallel (models: 2, splits: 1)\n"
      ]
     },
     {
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "100%|██████████| 2/2 [00:00<00:00, 16.09it/s]\n"
+      "100%|██████████| 2/2 [00:00<00:00, 13.08it/s]\n"
      ]
     },
     {
@@ -1388,8 +1374,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20\n",
-      "Modelling tides using EOT20\n"
+      "Modelling tides with EOT20\n",
+      "Modelling tides with EOT20\n"
      ]
     },
     {
@@ -1508,8 +1494,8 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Modelling tides using EOT20\n",
-      "Modelling tides using EOT20\n"
+      "Modelling tides with EOT20\n",
+      "Modelling tides with EOT20\n"
      ]
     },
     {

diff --git a/docs/notebooks/Satellite_data.ipynb b/docs/notebooks/Satellite_data.ipynb