1.1: Fix kraken, migration to python 3.11 and build system

.github/workflows/test-and-deploy.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest
 
     env:
-      python-version: 3.9 # Our base image has Python 3.9
+      python-version: 3.11 # Our base image has Python 3.11
 
     steps:
     - name: Checkout
@@ -29,32 +29,22 @@ jobs:
 
     - name: Install dependencies
       run: |
-        sudo ./core/tools/mavlink_router/bootstrap.sh
-        ./core/tools/ardupilot_tools/bootstrap.sh
         sudo apt install shellcheck
-        npm install \
-          eslint@7 eslint-plugin-vue eslint-plugin-html @vue/eslint-config-typescript @typescript-eslint/parser \
-          @typescript-eslint/eslint-plugin typescript @vue/eslint-config-airbnb@^5.3.0 @vue/cli-service \
-          eslint-plugin-simple-import-sort \
-          --save-dev
         python -m pip install --upgrade pip
-        pip install wheel # wheel need to be installed before mavproxy
-        pip install mavproxy
-        pip install pyfakefs pytest-cov pytest-timeout pylint==2.17.7 pylint-pydantic mypy isort black==22.1.0 asyncmock types-requests \
-          pytest-mock nmeasim geographiclib ruff
-        pip install git+https://github.com/patrickelectric/pykson.git@master_fixes
-        ## Install our own libraries
-        python ./core/libs/bridges/setup.py install
-        python ./core/libs/commonwealth/setup.py install
-        ## We need to install loguru and appdirs since they may be used inside setup.py
-        python -m pip install --user appdirs==1.4.4 loguru==0.5.3
-        find . -type f -name "setup.py" | xargs --max-lines=1 --replace=% python % install --user
-        ## Fix Click version since its being updated to an old version by our packages
-        pip install click==8.0.0
+        pip install pipenv
+        pipenv install poetry && pipenv run poetry install
+        echo "installing mavlink-router"
+        pipenv run ./core/tools/mavlink_router/bootstrap.sh
+        echo "installing ardupilot_tools"
+        pipenv run ./core/tools/ardupilot_tools/bootstrap.sh
+        pipenv run python ./core/libs/bridges/setup.py install
+        pipenv run python ./core/libs/commonwealth/setup.py install
+        rm -rf core/libs/bridges/build
+        rm -rf core/libs/commonwealth/build
 
     - name: Run tests
       run: |
-        ./.hooks/pre-push
+        .hooks/pre-push
 
     - name: Upload coverage
       uses: actions/upload-artifact@v3

.hooks/pre-push

@@ -2,6 +2,44 @@
 
 set -e
 
+# Check if pipenv is installed
+command -v pipenv >/dev/null 2>&1 || {
+    echo >&2 "pipenv is not installed. Aborting. Please install it and try again."
+    exit 1
+}
+
+# Check if shellcheck is installed
+command -v shellcheck >/dev/null 2>&1 || {
+    echo >&2 "shellcheck is not installed. Aborting. Please install it and try again."
+    exit 1
+}
+
+pipenv install poetry
+pipenv run poetry install
+# download some required files if they are not present
+echo "installing mavlink-router"
+pipenv run ./core/tools/mavlink_router/bootstrap.sh
+echo "installing ardupilot_tools"
+pipenv run ./core/tools/ardupilot_tools/bootstrap.sh
+
+isort_extra_args="--check-only --diff"
+black_extra_args="--check --diff"
+fixing="false"
+
+for arg in "$@"; do
+  case $arg in
+    --fix)
+      isort_extra_args=""
+      black_extra_args=""
+      fixing="true"
+      ;;
+    *)
+      echo "Unknown argument: $arg"
+      exit 1
+      ;;
+  esac
+done
+
 echo "Running pre push hook!"
 repository_path=$(git rev-parse --show-toplevel)
 tag_name=$(git tag --points-at=HEAD | head -n 1)
@@ -30,25 +68,28 @@ git ls-files '*.sh' | xargs --max-lines=1 shellcheck --exclude=SC2005,SC2015,SC2
 
 echo "Running isort.."
 # Run isort for each python project
-dirname $(git ls-files "$repository_path/*/setup.py") | xargs -I {} isort --src-path="{}" --check-only --diff "{}"
+dirname $(git ls-files "$repository_path/*/setup.py") | xargs -I {} pipenv run isort --src-path="{}" ${isort_extra_args} "{}"
 
 # This will only get the python files track by git, not including submodules
 python_files=$(git ls-files '*.py')
 
 echo "Running black.."
-black --check --diff $python_files
+pipenv run black ${black_extra_args} $python_files
+
+# do not run the rest of the script if running for fixing
+[ "$fixing" = true ] && exit 0
 
 # Faster than pylint to check for issues
 echo "Running ruff.."
-ruff $python_files
+pipenv run ruff $python_files
 
 echo "Running pylint.."
-pylint $python_files
+pipenv run pylint $python_files
 
 echo "Running pytest.."
-pytest --cov="$repository_path" --cov-report html
+pipenv run pytest -n 10 --cov="$repository_path" --cov-report html
 
 echo "Running mypy.."
-git ls-files '*.py' | xargs --max-lines=1 mypy --no-incremental
+git ls-files '*setup.py' | parallel 'pipenv run mypy $(dirname {})'
 
 exit 0

core/frontend/src/components/kraken/InstalledExtensionCard.vue

@@ -204,11 +204,15 @@ export default Vue.extend({
         return false
       }
       const versions: string[] = Object.keys(this.extensionData?.versions ?? {})
-      const lastest_stable = stable.max(versions)
-      if (semver.gt(this.extension.tag, lastest_stable)) {
-        return false
+      const current_version = new semver.SemVer(this.extension.tag)
+      // if is stable (which implies major >= 1), show latest stable
+      if (stable.is(this.extension.tag) && current_version.major > 0) {
+        return stable.max(versions) === this.extension.tag ? false : stable.max(versions)
       }
-      return this.extension.tag === lastest_stable ? false : lastest_stable
+      // show the latest version regardless of stability
+      // eslint-disable-next-line no-extra-parens
+      const latest = versions.reduce((a: string, b: string) => (semver.compare(a, b) > 0 ? a : b))
+      return this.extension.tag === latest ? false : latest
     },
   },
   methods: {

core/frontend/src/views/ExtensionManagerView.vue

@@ -423,25 +423,44 @@ export default Vue.extend({
           notifier.pushBackError('EXTENSIONS_INSTALLED_FETCH_FAIL', error)
         })
     },
-    async showLogs(extension: InstalledExtensionData): Promise<void> {
+    async showLogs(extension: InstalledExtensionData) {
       this.setLoading(extension, true)
-      await back_axios({
+      const ansi = new AnsiUp()
+      this.log_output = ''
+
+      back_axios({
         method: 'get',
         url: `${API_URL}/log`,
         params: {
           container_name: this.getContainerName(extension),
         },
+        onDownloadProgress: (progressEvent) => {
+          const chunk = progressEvent.currentTarget.response
+          this.$set(this, 'log_output', ansi.ansi_to_html(chunk))
+          this.show_log = true
+          this.setLoading(extension, false)
+          this.$nextTick(() => {
+            // TODO: find a better way to scroll to bottom
+            const output = document.querySelector(
+              '#app > div.v-dialog__content.v-dialog__content--active > div',
+            ) as HTMLInputElement
+            if (!output) {
+              return
+            }
+            output.scrollTop = output.scrollHeight
+          })
+        },
         timeout: 30000,
       })
-        .then((response) => {
-          const ansi = new AnsiUp()
-          this.log_output = ansi.ansi_to_html(response.data.join(''))
-          this.show_log = true
+        .then(() => {
+          this.setLoading(extension, false)
         })
         .catch((error) => {
           notifier.pushBackError('EXTENSIONS_LOG_FETCH_FAIL', error)
         })
-      this.setLoading(extension, false)
+        .finally(() => {
+          this.setLoading(extension, false)
+        })
     },
     showModal(extension: ExtensionData) {
       this.show_dialog = true

core/libs/commonwealth/setup.py

@@ -23,7 +23,7 @@
         "aiohttp == 3.7.4",
         "appdirs == 1.4.4",
         "loguru == 0.5.3",
-        "starlette == 0.13.6",
+        "starlette == 0.27.0",
         "pykson == 1.0.2",
     ],
     dependency_links=[

core/services/ardupilot_manager/setup.py

@@ -74,8 +74,10 @@
         "appdirs == 1.4.4",
         "packaging == 20.4",
         "smbus2 == 0.3.0",
-        "starlette == 0.13.6",
-        "fastapi == 0.63.0",
+        "starlette == 0.27.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "uvicorn == 0.13.4",
         "python-multipart == 0.0.5",
         "validators == 0.18.2",

core/services/bag_of_holding/setup.py

@@ -10,7 +10,9 @@
     install_requires=[
         "appdirs == 1.4.4",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "uvicorn == 0.13.4",

core/services/bridget/setup.py

@@ -11,7 +11,9 @@
     install_requires=[
         "bridges == 0.1.0",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "uvicorn == 0.13.4",

core/services/cable_guy/setup.py

@@ -67,8 +67,10 @@ def populate_static_files() -> None:
         "appdirs == 1.4.4",
         "psutil == 5.7.2",
         "pyroute2 == 0.5.13",
-        "starlette == 0.13.6",
-        "fastapi == 0.63.0",
+        "starlette == 0.27.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "uvicorn == 0.13.4",
         "aiofiles == 0.6.0",
         "fastapi-versioning == 0.9.1",

core/services/commander/setup.py

@@ -11,7 +11,9 @@
     install_requires=[
         "appdirs == 1.4.4",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "uvicorn == 0.13.4",

core/services/helper/setup.py

@@ -67,13 +67,15 @@ def populate_static_files() -> None:
         "aiofiles == 0.6.0",
         "beautifulsoup4 == 4.9.3",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "psutil == 5.7.2",
         "requests == 2.25.1",
         "speedtest-cli == 2.1.3",
-        "starlette == 0.13.6",
+        "starlette == 0.27.0",
         "uvicorn == 0.13.4",
     ],
 )

core/services/kraken/kraken.py

@@ -279,11 +279,18 @@ async def list_containers(self) -> List[DockerContainer]:
         containers: List[DockerContainer] = await self.client.containers.list(filter='{"status": ["running"]}')  # type: ignore
         return containers
 
-    async def load_logs(self, container_name: str) -> List[str]:
+    async def stream_logs(self, container_name: str, timeout: int = 30) -> AsyncGenerator[str, None]:
         containers = await self.client.containers.list(filters={"name": {container_name: True}})  # type: ignore
         if not containers:
             raise RuntimeError(f"Container not found: {container_name}")
-        return cast(List[str], await containers[0].log(stdout=True, stderr=True))
+
+        start_time = asyncio.get_event_loop().time()
+        async for log_line in containers[0].log(stdout=True, stderr=True, follow=True, stream=True):
+            elapsed_time = asyncio.get_event_loop().time() - start_time
+            if elapsed_time > timeout:
+                break
+            yield log_line
+        logger.info(f"Finished streaming logs for {container_name}")
 
     async def load_stats(self) -> Dict[str, Any]:
         containers = await self.client.containers.list()  # type: ignore

core/services/kraken/main.py

@@ -2,12 +2,12 @@
 import argparse
 import asyncio
 import logging
-from typing import Any, List
+from typing import Any, Iterable
 
 from commonwealth.utils.apis import GenericErrorHandlingRoute
 from commonwealth.utils.logs import InterceptHandler, init_logger
 from fastapi import FastAPI, HTTPException, status
-from fastapi.responses import HTMLResponse, StreamingResponse
+from fastapi.responses import HTMLResponse, PlainTextResponse, StreamingResponse
 from fastapi_versioning import VersionedFastAPI, version
 from loguru import logger
 from pydantic import BaseModel
@@ -126,10 +126,10 @@ async def list_containers() -> Any:
     ]
 
 
-@app.get("/log", status_code=status.HTTP_200_OK)
+@app.get("/log", status_code=status.HTTP_200_OK, response_class=PlainTextResponse)
 @version(1, 0)
-async def log_containers(container_name: str) -> List[str]:
-    return await kraken.load_logs(container_name)
+async def log_containers(container_name: str) -> Iterable[bytes]:
+    return StreamingResponse(kraken.stream_logs(container_name), media_type="text/plain")  # type: ignore
 
 
 @app.get("/stats", status_code=status.HTTP_200_OK)

core/services/kraken/setup.py

@@ -12,7 +12,9 @@
         "aiodocker == 0.21.0",
         "appdirs == 1.4.4",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "uvicorn == 0.13.4",

core/services/nmea_injector/nmea_injector/tests/test_TrafficController.py

@@ -65,6 +65,7 @@ def mock_send_mavlink_message(self: MavlinkMessenger, message: Dict[str, Any]) -
 
             if sock_kind == SocketKind.UDP:
                 sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+                sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
                 sock.sendto(bytes_to_send, SERVER_ADDR)
 
             if sock_kind == SocketKind.TCP:

core/services/nmea_injector/setup.py

@@ -17,12 +17,14 @@
     install_requires=[
         "appdirs == 1.4.4",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "pynmea2 == 1.18.0",
         "pytest-mock==3.10.0",
-        "starlette == 0.13.6",
+        "starlette == 0.27.0",
         "uvicorn == 0.13.4",
         "validators == 0.18.2",
     ],

core/services/ping/setup.py

@@ -19,11 +19,13 @@
         "bluerobotics-ping == 0.1.5",
         "bridges == 0.1.0",
         "commonwealth == 0.1.0",
-        "fastapi == 0.63.0",
+        "fastapi == 0.105.0",
+        # Enforce anyio fastapi subdependency to avoid conflict with starlette
+        "anyio == 3.7.1",
         "fastapi-versioning == 0.9.1",
         "loguru == 0.5.3",
         "pyserial == 3.5",
-        "starlette == 0.13.6",
+        "starlette == 0.27.0",
         "uvicorn == 0.13.4",
     ],
 )