readding some scripts for CI

AutomatingSciencePipeline · May 15, 2024 · 91aa4d6 · 91aa4d6
1 parent 6969528
commit 91aa4d6
Show file tree

Hide file tree

Showing 15 changed files with 976 additions and 102 deletions.
diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml
@@ -25,7 +25,7 @@ jobs:
                 # Should match the "build" subsection's settings in docker-compose.yml
                 context: ./apps/backend
                 file: ./apps/backend/backend.Dockerfile
-                target: production
+                # target: production # this has been commented out due to simpler Dockerfile
     frontend-prod:
         name: Frontend Production Container
         runs-on: ubuntu-latest
@@ -40,34 +40,36 @@ jobs:
                 # Should match the "build" subsection's settings in docker-compose.yml
                 context: ./apps/frontend
                 file: ./apps/frontend/frontend.Dockerfile
-                target: runner
+                # target: runner # this has been commented out due to simpler Dockerfile
 
-    # Development containers get the settings from docker-compose.dev.yml layered on top of docker-compose.yml
-    backend-dev:
-      name: Backend Development Container
-      runs-on: ubuntu-latest
-      steps:
-          - name: Checkout
-            uses: actions/checkout@v3
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v2
-          - name: Build
-            uses: docker/build-push-action@v4
-            with:
-              context: ./apps/backend
-              file: ./apps/backend/backend.Dockerfile
-              target: development
-    frontend-dev:
-      name: Frontend Development Container
-      runs-on: ubuntu-latest
-      steps:
-          - name: Checkout
-            uses: actions/checkout@v3
-          - name: Set up Docker Buildx
-            uses: docker/setup-buildx-action@v2
-          - name: Build
-            uses: docker/build-push-action@v4
-            with:
-              context: ./apps/frontend
-              file: ./apps/frontend/frontend.Dockerfile
-              target: development
+    # Below has been commented out due to simpler Dockerfile, as we shift towards Kubernetes
+
+    # # Development containers get the settings from docker-compose.dev.yml layered on top of docker-compose.yml
+    # backend-dev:
+    #   name: Backend Development Container
+    #   runs-on: ubuntu-latest
+    #   steps:
+    #       - name: Checkout
+    #         uses: actions/checkout@v3
+    #       - name: Set up Docker Buildx
+    #         uses: docker/setup-buildx-action@v2
+    #       - name: Build
+    #         uses: docker/build-push-action@v4
+    #         with:
+    #           context: ./apps/backend
+    #           file: ./apps/backend/backend.Dockerfile
+    #           target: development
+    # frontend-dev:
+    #   name: Frontend Development Container
+    #   runs-on: ubuntu-latest
+    #   steps:
+    #       - name: Checkout
+    #         uses: actions/checkout@v3
+    #       - name: Set up Docker Buildx
+    #         uses: docker/setup-buildx-action@v2
+    #       - name: Build
+    #         uses: docker/build-push-action@v4
+    #         with:
+    #           context: ./apps/frontend
+    #           file: ./apps/frontend/frontend.Dockerfile
+    #           target: development
diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml
@@ -31,24 +31,44 @@ jobs:
         python -m pip install coverage
         pip install pipenv
         pipenv install --dev --system --deploy --ignore-pipfile
-    - name: Analyze with pylint
+    - name: Analyze backend with pylint
       working-directory: ./apps/backend
       # https://pylint.readthedocs.io/en/v2.16.1/user_guide/configuration/all-options.html#fail-on
       # Uses the .pylintrc rules file in the working directory
       # Fail if any Error-level notices were produced
       # Fail if the code quality score is below 9.0
       # Recursively search the specified directories for files to analyze
       # Check the file app.py
-      # Check the directories ./modules and ./tests
-      run: pylint --fail-on=E --fail-under=9.0 --recursive=y app.py ./modules/ ./tests/
-    - name: Test with pytest + record coverage
+      run: pylint --fail-on=E --fail-under=9.0 --recursive=y app.py
+    - name: Analyze runner with pylint
+      working-directory: ./apps/runner
+      # https://pylint.readthedocs.io/en/v2.16.1/user_guide/configuration/all-options.html#fail-on
+      # Uses the .pylintrc rules file in the working directory
+      # Fail if any Error-level notices were produced
+      # Fail if the code quality score is below 9.0
+      # Recursively search the specified directories for files to analyze
+      # Check the file runner.py
+      # Check the ./modules and ./tests directory
+      run: pylint --fail-on=E --fail-under=9.0 --recursive=y runner.py ./modules/ ./tests/
+    - name: Test backend with pytest + record coverage
       working-directory: ./apps/backend
       run: bash coverage.sh
-    - name: Zip coverage report
+    - name: Zip backend coverage report
       working-directory: ./apps/backend
       run: zip -r codeCoverage.zip coverageReport/
-    - name: Upload coverage report
+    - name: Test runner with pytest + record coverage
+      working-directory: ./apps/runner
+      run: bash coverage.sh
+    - name: Zip runner coverage report
+      working-directory: ./apps/runner
+      run: zip -r codeCoverage.zip coverageReport/
+    - name: Upload backend coverage report
       uses: actions/upload-artifact@v3
       with:
-        name: code-coverage-report
+        name: code-coverage-report-backend
         path: ./apps/backend/codeCoverage.zip
+    - name: Upload runner coverage report
+      uses: actions/upload-artifact@v3
+      with:
+        name: code-coverage-report-runner
+        path: ./apps/runner/codeCoverage.zip
diff --git a/apps/backend/.pylintrc b/apps/backend/.pylintrc
@@ -0,0 +1,15 @@
+# pylint python linter configuration
+
+[MASTER]
+extension-pkg-whitelist=pydantic # stop "No name 'BaseModel' in module 'pydantic'" https://github.com/pydantic/pydantic/issues/1961#issuecomment-759522422
+disable=
+    C0301, # line too long
+    C0114, # missing-module-docstring
+    C0116, # missing-function-docstring
+    C0115, # missing-class-docstring
+    C0103, # we chose to use camel case to be consistent with our frontend code
+    W1203, # we want to use fstrings for log message readability, the performance cost is not significant enough for us to care. but, don't turn off W1201 because if the dev wants to use % syntax, they should use lazy logging since the syntax readability is equivalent
+
+# Load our own custom plugins
+load-plugins=
+    linters.no_print
diff --git a/apps/backend/Pipfile b/apps/backend/Pipfile
@@ -9,5 +9,13 @@ requests = "*"
 kubernetes = "*"
 flask-cors = "*"
 
+[dev-packages]
+# libmagic dlls for windows hosts https://pypi.org/project/python-magic/ (when missing, vague errors and it crashes)
+python-magic-bin = {version = "*", markers = "platform_system == 'Windows'"}
+yapf = "*"
+pylint = "*"
+pytest = "*"
+coverage = "*"
+
 [requires]
 python_version = "3.8"