Merge pull request #341 from techman83/feat/update-python

Update Container + Python

.github/workflows/coverage-build.yml

@@ -16,9 +16,8 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: "3.11"
+          python-version: "3.12"
           cache: pip
-          cache-dependency-path: netkan/setup.py
       - name: Install test dependencies
         run: pip install .[test]
       - name: force our git config

.github/workflows/deploy.yml

@@ -67,9 +67,8 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
           cache: pip
-          cache-dependency-path: netkan/setup.py
       - name: Install Dependencies
         run: pip install netkan/.
       - name: Re-deploy Containers

.github/workflows/test.yml

@@ -12,9 +12,8 @@ jobs:
       - name: Setup Python
         uses: actions/setup-python@v5
         with:
-          python-version: 3.11
+          python-version: 3.12
           cache: pip
-          cache-dependency-path: netkan/setup.py
       - name: Install test dependencies
         working-directory: netkan
         run: pip install .[test]

.vscode/extensions.json

@@ -1,6 +1,7 @@
 {
     "recommendations": [
         "ms-vscode-remote.vscode-remote-extensionpack",
-        "ms-python.python"
+        "ms-python.python",
+        "tamasfe.even-better-toml"
     ]
 }

docker-compose.yml

@@ -1,5 +1,3 @@
-version: '3.4'
-
 volumes:
   certbot:
 
@@ -11,15 +9,15 @@ services:
     environment:
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
       CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
-      CKANMETA_USER: ${CKAN_METADATA_USER}
-      CKANMETA_REPOS: ${CKAN_METADATA_REPOS}
+      CKAN_USER: ${CKAN_METADATA_USER}
+      CKAN_REPOS: ${CKAN_METADATA_REPOS}
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
       GH_Token: ${CKAN_GH_Token}
       SQS_QUEUE: OutboundDev.fifo
       SQS_TIMEOUT: 30
-      STATUS_DB: DevNetKANStatus
+      STATUS_DB: DevMultiKANStatus
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
     volumes:
@@ -30,33 +28,14 @@ services:
       context: netkan/.
       target: dev
     environment:
+      GH_Token: ${CKAN_GH_Token}
       NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
       SSH_KEY: ${CKAN_NETKAN_SSHKEY}
       CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
       AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
       INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
-      GAME_ID: ksp2
-      MAX_QUEUED: 1
-      DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
-      DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
-    volumes:
-      - ./netkan:/home/netkan/netkan
-    command: scheduler --dev
-  scheduler:
-    build:
-      context: netkan/.
-      target: dev
-    environment:
-      NETKAN_REMOTES: ${NETKAN_METADATA_PATHS}
-      SSH_KEY: ${CKAN_NETKAN_SSHKEY}
-      CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
-      AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
-      AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}
-      AWS_ACCESS_KEY_ID: ${CKAN_AWS_ACCESS_KEY_ID}
-      INFLATION_QUEUES: ksp=InboundDevKsp.fifo ksp2=InboundDevKsp2.fifo
-      GAME_ID: ksp2
       MAX_QUEUED: 1
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
@@ -109,8 +88,8 @@ services:
       DISCORD_WEBHOOK_ID: ${DISCORD_WEBHOOK_ID}
       DISCORD_WEBHOOK_TOKEN: ${DISCORD_WEBHOOK_TOKEN}
       CKANMETA_REMOTES: ${CKAN_METADATA_PATHS}
-      CKANMETA_USER: ${CKAN_METADATA_USER}
-      CKANMETA_REPOS: ${CKAN_METADATA_REPOS}
+      CKAN_USER: ${CKAN_METADATA_USER}
+      CKAN_REPOS: ${CKAN_METADATA_REPOS}
     entrypoint: .local/bin/gunicorn
     command: -b 0.0.0.0:5000 --access-logfile - "netkan.webhooks:create_app()"
   adder:
@@ -145,7 +124,7 @@ services:
       context: netkan/.
       target: dev
     environment:
-      STATUS_DB: DevNetKANStatus
+      STATUS_DB: DevMultiKANStatus
       STATUS_BUCKET: ckan-test-status
       AWS_DEFAULT_REGION: ${CKAN_AWS_DEFAULT_REGION}
       AWS_SECRET_ACCESS_KEY: ${CKAN_AWS_SECRET_ACCESS_KEY}

netkan/Dockerfile

@@ -1,58 +1,52 @@
-FROM ubuntu:22.04 as ubuntu_with_python
+FROM ubuntu:24.04 AS ubuntu_with_python
 
 # Don't prompt for time zone
 ENV DEBIAN_FRONTEND=noninteractive
 
 # Install Git and Python
 RUN apt-get update \
     && apt-get install -y --no-install-recommends \
-        git libffi-dev openssh-client \
-        python3 python-is-python3 \
-    && apt-get clean
+    git libffi-dev openssh-client \
+    python3 python-is-python3 ca-certificates curl && \
+    apt-get clean && \
+    curl -sSL https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    rm /usr/lib/python3.12/EXTERNALLY-MANAGED && \
+    python3 get-pip.py --no-input && rm get-pip.py
+RUN groupmod -n 'netkan' 'ubuntu' && \
+    usermod -l 'netkan' -d /home/netkan -m ubuntu
 
-FROM ubuntu_with_python as base
-RUN apt-get install -y --no-install-recommends \
-    python3-pip python3-setuptools python3-dev
-RUN useradd -ms /bin/bash netkan
-ADD . /netkan
+FROM ubuntu_with_python AS base
+COPY . /netkan
 WORKDIR /netkan
-RUN pip install pip --upgrade
 RUN chown -R netkan:netkan /netkan
 USER netkan
 RUN pip install --user . --no-warn-script-location
 
-FROM ubuntu_with_python as production
+FROM ubuntu_with_python AS production
 COPY --from=base /home/netkan/.local /home/netkan/.local
-RUN useradd -Ms /bin/bash netkan
 RUN chown -R netkan:netkan /home/netkan
 WORKDIR /home/netkan
 USER netkan
-ADD .gitconfig .
-ENV PATH "$PATH:/home/netkan/.local/bin"
+COPY .gitconfig .
+ENV PATH="$PATH:/home/netkan/.local/bin"
 RUN /home/netkan/.local/bin/netkan --help
 ENTRYPOINT [".local/bin/netkan"]
 CMD ["--help"]
 
-FROM production as test
+FROM production AS test
 USER root
-RUN apt-get install -y --no-install-recommends \
-    python3-pip python3-setuptools python3-dev
-RUN pip install pip --upgrade
-ADD . /netkan
+COPY . /netkan
 RUN chown -R netkan:netkan /netkan
 USER netkan
 WORKDIR /netkan
 RUN pip install --user .[test]
 RUN /home/netkan/.local/bin/pytest -v
 
-FROM production as dev
+FROM production AS dev
 USER root
-RUN apt-get install -y --no-install-recommends \
-    python3-pip python3-setuptools python3-dev
-RUN pip install pip --upgrade
 ADD . /netkan
 RUN chown -R netkan:netkan /netkan
-ADD run_dev.sh /usr/local/bin/
+COPY run_dev.sh /usr/local/bin/
 USER netkan
 RUN pip install --user /netkan/.[development]
 ENTRYPOINT ["/usr/local/bin/run_dev.sh"]

netkan/MANIFEST.in

@@ -0,0 +1,2 @@
+global-include *.md *.jinja2 *.graphql
+recursive-exclude tests *

netkan/pyproject.toml

@@ -0,0 +1,110 @@
+[build-system]
+requires = ["setuptools"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "netkan"
+version = "1.0"
+dependencies = [
+    "boto3",
+    "click",
+    "gitpython",
+    "pynamodb",
+    # 2019-11-01 capping to 2.8.0 - https://github.com/boto/botocore/commit/e87e7a745fd972815b235a9ee685232745aa94f9
+    "python-dateutil>=2.1,<2.8.1",
+    "requests",
+    "flask",
+    "jinja2",
+    "internetarchive!=3.0.1",
+    "gunicorn>=19.9,!=20.0.0",
+    "discord.py>=1.6.0,<=1.7.3",
+    "PyGithub",
+    "ruamel.yaml",
+]
+requires-python = ">=3.8"
+authors = [{ name = "Leon Wright", email = "techman83@gmail.com" }]
+description = "NetKAN Infra"
+
+[project.urls]
+Repository = "https://github.com/KSP-CKAN/NetKAN-infra/"
+Issues = "https://github.com/KSP-CKAN/NetKAN-infra/issues"
+
+[project.scripts]
+netkan = "netkan.cli:netkan"
+
+[project.optional-dependencies]
+development = [
+    "ptvsd",
+    "autopep8",
+    "boto3-stubs[essential,cloudwatch]",
+    "coverage",
+    "troposphere",
+    "pytest",
+    "mypy",
+    "pytest-mypy",
+    "pylint",
+    "pytest-pylint",
+    "types-python-dateutil",
+    "types-click",
+    "types-requests",
+    "types-Flask",
+    "types-Jinja2",
+]
+test = [
+    "boto3-stubs[essential,cloudwatch]",
+    "coverage",
+    "pytest",
+    "mypy",
+    "pytest-mypy",
+    "pylint",
+    "pytest-pylint",
+    "types-python-dateutil",
+    "types-click",
+    "types-requests",
+    "types-Flask",
+    "types-Jinja2",
+]
+
+[tool.coverage.run]
+branch = true
+omit = ["tests/*"]
+
+[tool.coverage.report]
+exclude_lines = ["if TYPE_CHECKING:", "    pass"]
+
+[tool.pytest.ini_options]
+python_files = "tests/__init__.py"
+addopts = "-p no:cacheprovider --mypy --pylint"
+filterwarnings = ["ignore", "default:::netkan.*", "default:::tests.*"]
+
+[tool.mypy]
+ignore_missing_imports = true
+warn_redundant_casts = true
+show_error_context = true
+show_column_numbers = true
+show_error_codes = true
+
+[[tool.mypy.overrides]]
+module = "netkan.*"
+check_untyped_defs = true
+disallow_incomplete_defs = true
+disallow_untyped_defs = true
+disallow_any_generics = true
+warn_unreachable = true
+strict_equality = true
+
+[tool.pylint."MESSAGES CONTROL"]
+disable = """
+    line-too-long,
+    duplicate-code,
+    missing-module-docstring,
+    missing-class-docstring,
+    missing-function-docstring,
+    too-few-public-methods,
+    too-many-instance-attributes,
+    too-many-statements,
+    too-many-return-statements,
+    too-many-branches,
+    too-many-arguments,
+    consider-using-with,
+"""