Kubernetes setup

.github/workflows/docker.yml

@@ -25,7 +25,7 @@ jobs:
                 # Should match the "build" subsection's settings in docker-compose.yml
                 context: ./apps/backend
                 file: ./apps/backend/backend.Dockerfile
-                target: production
+                # target: production # this has been commented out due to simpler Dockerfile
     frontend-prod:
         name: Frontend Production Container
         runs-on: ubuntu-latest
@@ -40,34 +40,36 @@ jobs:
                 # Should match the "build" subsection's settings in docker-compose.yml
                 context: ./apps/frontend
                 file: ./apps/frontend/frontend.Dockerfile
-                target: runner
+                # target: runner # this has been commented out due to simpler Dockerfile
 
-    # Development containers get the settings from docker-compose.dev.yml layered on top of docker-compose.yml
-    backend-dev:
-      name: Backend Development Container
-      runs-on: ubuntu-latest
-      steps:
-          - name: Checkout
-            uses: actions/checkout@v3
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v2
-          - name: Build
-            uses: docker/build-push-action@v4
-            with:
-              context: ./apps/backend
-              file: ./apps/backend/backend.Dockerfile
-              target: development
-    frontend-dev:
-      name: Frontend Development Container
-      runs-on: ubuntu-latest
-      steps:
-          - name: Checkout
-            uses: actions/checkout@v3
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v2
-          - name: Build
-            uses: docker/build-push-action@v4
-            with:
-              context: ./apps/frontend
-              file: ./apps/frontend/frontend.Dockerfile
-              target: development
+    # Below has been commented out due to simpler Dockerfile, as we shift towards Kubernetes
+
+    # # Development containers get the settings from docker-compose.dev.yml layered on top of docker-compose.yml
+    # backend-dev:
+    #   name: Backend Development Container
+    #   runs-on: ubuntu-latest
+    #   steps:
+    #       - name: Checkout
+    #         uses: actions/checkout@v3
+    #       - name: Set up Docker Buildx
+    #         uses: docker/setup-buildx-action@v2
+    #       - name: Build
+    #         uses: docker/build-push-action@v4
+    #         with:
+    #           context: ./apps/backend
+    #           file: ./apps/backend/backend.Dockerfile
+    #           target: development
+    # frontend-dev:
+    #   name: Frontend Development Container
+    #   runs-on: ubuntu-latest
+    #   steps:
+    #       - name: Checkout
+    #         uses: actions/checkout@v3
+    #       - name: Set up Docker Buildx
+    #         uses: docker/setup-buildx-action@v2
+    #       - name: Build
+    #         uses: docker/build-push-action@v4
+    #         with:
+    #           context: ./apps/frontend
+    #           file: ./apps/frontend/frontend.Dockerfile
+    #           target: development

.github/workflows/python-test.yml → .github/workflows/python-test-backend.yml

@@ -1,4 +1,4 @@
-name: Python Lint and Test
+name: Python Lint and Test - Backend
 
 on:
   push:
@@ -31,24 +31,23 @@ jobs:
         python -m pip install coverage
         pip install pipenv
         pipenv install --dev --system --deploy --ignore-pipfile
-    - name: Analyze with pylint
+    - name: Analyze backend with pylint
       working-directory: ./apps/backend
       # https://pylint.readthedocs.io/en/v2.16.1/user_guide/configuration/all-options.html#fail-on
       # Uses the .pylintrc rules file in the working directory
       # Fail if any Error-level notices were produced
       # Fail if the code quality score is below 9.0
       # Recursively search the specified directories for files to analyze
       # Check the file app.py
-      # Check the directories ./modules and ./tests
-      run: pylint --fail-on=E --fail-under=9.0 --recursive=y app.py ./modules/ ./tests/
-    - name: Test with pytest + record coverage
+      run: pylint --fail-on=E --fail-under=9.0 --recursive=y app.py
+    - name: Test backend with pytest + record coverage
       working-directory: ./apps/backend
       run: bash coverage.sh
-    - name: Zip coverage report
+    - name: Zip backend coverage report
       working-directory: ./apps/backend
       run: zip -r codeCoverage.zip coverageReport/
-    - name: Upload coverage report
+    - name: Upload backend coverage report
       uses: actions/upload-artifact@v3
       with:
-        name: code-coverage-report
+        name: code-coverage-report-backend
         path: ./apps/backend/codeCoverage.zip

.github/workflows/python-test-runner.yml

@@ -0,0 +1,54 @@
+name: Python Lint and Test - Runner
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+env:
+  FIREBASE_KEY: ${{ secrets.FIREBASE_KEY }}
+  CONTACT_MONGODB_AT: ${{ secrets.CONTACT_MONGODB_AT }}
+  BACKEND_PORT: ${{ secrets.BACKEND_PORT }}
+  MONGODB_PORT: ${{ secrets.MONGODB_PORT }}
+jobs:
+  Lint_And_Test:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v3
+      with:
+        python-version: "3.8"
+    - name: Install dependencies
+      working-directory: ./apps/runner
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install coverage
+        pip install pipenv
+        pipenv install --dev --system --deploy --ignore-pipfile
+    - name: Analyze runner with pylint
+      working-directory: ./apps/runner
+      # https://pylint.readthedocs.io/en/v2.16.1/user_guide/configuration/all-options.html#fail-on
+      # Uses the .pylintrc rules file in the working directory
+      # Fail if any Error-level notices were produced
+      # Fail if the code quality score is below 9.0
+      # Recursively search the specified directories for files to analyze
+      # Check the file runner.py
+      # Check the ./modules and ./tests directory
+      run: pylint --fail-on=E --fail-under=9.0 --recursive=y runner.py ./modules/ ./tests/
+    - name: Test runner with pytest + record coverage
+      working-directory: ./apps/runner
+      run: bash coverage.sh
+    - name: Zip runner coverage report
+      working-directory: ./apps/runner
+      run: zip -r codeCoverage.zip coverageReport/
+    - name: Upload runner coverage report
+      uses: actions/upload-artifact@v3
+      with:
+        name: code-coverage-report-runner
+        path: ./apps/runner/codeCoverage.zip

README.md

@@ -6,6 +6,10 @@ The Automating Science Pipeline seeks to make it easier to run data-collecting c
 - A user should be able to download and simulate the system on their local machine with minimal effort and configuration.
 - The system is open source and designed for expansion to other languages and user-implemented functionality (ex. Output data post-processing).
 
+## Documentation
+
+Our live documentation website is hosted [here](https://automatingsciencepipeline.github.io/Monorepo). As of May 2024, old documentation is being migrated to the website.
+
 ## User Guide
 
 This user guide is meant to assist first time users in setting up a local simulation of the system. You can find it on the [wiki](https://github.com/AutomatingSciencePipeline/Monorepo/wiki/User-Guide).

apps/backend/.dockerignore

@@ -1,25 +1,5 @@
 **/__pycache__
-**/.classpath
 **/.dockerignore
-**/.env
-**/.git
 **/.gitignore
-**/.project
-**/.settings
-**/.toolstarget
-**/.vs
-**/.vscode
-**/*.*proj.user
-**/*.dbmdl
-**/*.jfm
-**/bin
-**/charts
-**/docker-compose*
-**/compose*
 **/Dockerfile*
-**/node_modules
-**/npm-debug.log
-**/obj
-**/secrets.dev.yaml
-**/values.dev.yaml
-README.md
+README.md

apps/backend/.gitignore

@@ -1,9 +1,3 @@
-# Ignore docker container volume contents
-ExperimentFiles/
-
-# Ignore log files produced by the system
-logs/
-
 # Python
 __pycache__
 .venv

apps/backend/.pylintrc

@@ -11,5 +11,5 @@ disable=
     W1203, # we want to use fstrings for log message readability, the performance cost is not significant enough for us to care. but, don't turn off W1201 because if the dev wants to use % syntax, they should use lazy logging since the syntax readability is equivalent
 
 # Load our own custom plugins
-load-plugins=
-    linters.no_print
+; load-plugins=
+;     linters.no_print

apps/backend/Pipfile

@@ -2,20 +2,12 @@
 url = "https://pypi.org/simple"
 verify_ssl = true
 name = "pypi"
-# Packages in here should be left on * unless there is a specific reason, which should be explained by a comment
 
 [packages]
-flask = "==2.0.2" # Was specifically versioned by the last team. TODO see if we can update it.
-flask-cors = "==3.0.10" # Was specifically versioned by the last team. TODO see if we can update it.
-gunicorn = "==20.1.0" # Was specifically versioned by the last team. TODO see if we can update it.
-numpy = "*"
-firebase-admin = "*"
-configparser = "*"
-python-magic = "*"
-python-dotenv = "*"
-matplotlib = "*"
-pymongo = "*"
-pydantic = "*"
+flask = "*"
+requests = "*"
+kubernetes = "*"
+flask-cors = "*"
 pytest-cov = "*"
 
 [dev-packages]