Merge branch 'main' into flak8fix

opensearch-project · Nov 15, 2024 · e23854c · e23854c
2 parents 5019545 + 62cecd1
commit e23854c
Show file tree

Hide file tree

Showing 3 changed files with 108 additions and 49 deletions.
diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml
@@ -2,7 +2,7 @@ name: CI
 
 on:
   push:
-  pull_request_target:
+  pull_request:
 
 env:
   python-version: '3.11'
@@ -11,6 +11,27 @@ env:
   node-version: '18.x'
 
 jobs:
+  workflow-info:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create JSON Artifact
+        run: |
+          echo '{' > workflow-info.json
+          echo '  "pr_number": "${{ github.event.pull_request.number }}",' >> workflow-info.json
+          echo '  "commit": "${{ github.event.pull_request.head.sha || github.event.after || github.sha }}",' >> workflow-info.json
+          echo '  "branch": "${{ github.ref }}",' >> workflow-info.json
+          echo '  "commit_parent": "${{ github.event.pull_request.base.sha || github.event.before || github.base.sha }}",' >> workflow-info.json
+          echo '  "build_url": "https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}",' >> workflow-info.json
+          echo '  "build": "${{ github.run_id }}"' >> workflow-info.json
+          echo '}' >> workflow-info.json
+          echo "workflow-info.json created"
+      - name: Upload JSON Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: workflow-info
+          path: workflow-info.json
+
+
   style-check:
     runs-on: ubuntu-latest
     steps:
@@ -42,17 +63,20 @@ jobs:
        run: |
         flake8 $(git ls-files '*.py')
 
-  python-tests:
+  console-python-tests:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
       matrix:
         py-project:
-          - ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/lib/console_link
-          - ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/cluster_tools
+          - lib/console_link
+          - cluster_tools
+          - console_api
+    env:
+      WORKING_DIR: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/${{ matrix.py-project }}
     defaults:
       run:
-        working-directory: ${{ matrix.py-project }}
+        working-directory: ${{ env.WORKING_DIR }}
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -63,38 +87,16 @@ jobs:
           pipenv install --deploy --dev
           pipenv run test
           pipenv run coverage xml
-      - name: Upload Coverage
-        uses: codecov/codecov-action@v4
-        with:
-          fail_ci_if_error: true
-          files: ./coverage.xml
-          flags: python-test
-          token: ${{ secrets.CODECOV_TOKEN }}
-          verbose: true
-
-  console-api-tests:
-    runs-on: ubuntu-latest
-    defaults:
-      run:
-        working-directory: ./TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api
-    steps:
-      - uses: actions/checkout@v4
-      - uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.python-version }}
-      - run: |
-          python3 -m pip install --upgrade pipenv
-          pipenv install --deploy --dev
-          pipenv run coverage run --source='.' manage.py test console_api
-          pipenv run coverage xml
-      - name: Upload Coverage
-        uses: codecov/codecov-action@v4
+      - name: Get Sanitized Name
+        env:
+          PY_PROJECT: ${{ matrix.py-project }}
+        run: echo "SANITIZED_PY_PROJECT=${PY_PROJECT/\//-}" >> $GITHUB_ENV
+      - name: Upload Coverage Reports
+        uses: actions/upload-artifact@v4
         with:
-          fail_ci_if_error: true
-          files: ./coverage.xml
-          flags: python-test
-          token: ${{ secrets.CODECOV_TOKEN }}
-          verbose: true
+          if-no-files-found: error
+          name: coverage-reports-python-tests-${{ env.SANITIZED_PY_PROJECT }}
+          path: ${{ env.WORKING_DIR }}/coverage.xml
 
 
   gradle-tests:
@@ -122,19 +124,16 @@ jobs:
       - uses: actions/upload-artifact@v4
         if: always()
         with:
-          name: test-reports
+          name: test-reports-gradle-tests
           path: |
             **/build/reports/tests/
             **/reports/jacoco/mergedReport/
-      - name: Upload Coverage
-        uses: codecov/codecov-action@v4
+      - name: Upload Coverage Reports
+        uses: actions/upload-artifact@v4
         with:
-          disable_search: true
-          fail_ci_if_error: true
-          files: ${{ github.workspace }}/build/reports/jacoco/mergedReport/jacocoMergedReport.xml
-          flags: gradle-test
-          token: ${{ secrets.CODECOV_TOKEN }}
-          verbose: true
+          if-no-files-found: error
+          name: coverage-reports-gradle-tests
+          path: ./**/jacocoMergedReport.xml
 
 
   python-e2e-tests:
@@ -236,15 +235,17 @@ jobs:
         env:
           GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
 
+
   all-ci-checks-pass:
     needs:
-      - style-check
-      - node-tests
+      - console-python-tests
       - gradle-tests
       - link-checker
+      - node-tests
       - python-e2e-tests
-      - python-lint
-      - python-tests
+#      Temp disable linting from blocking
+#      - python-lint
+      - style-check
     if: always()
     runs-on: ubuntu-latest
     steps:

diff --git a/.github/workflows/codecov-publish.yml b/.github/workflows/codecov-publish.yml
@@ -0,0 +1,54 @@
+name: Codecov Publish
+
+# Due to the behavior of workflow_run, changes to this file will
+# only be reflected once it is pushed up to the `default` (main) branch
+#
+# It is recommended to test changes to this file by pushing to the default branch
+# on a fork and evaluating the correctness of the action execution
+
+on:
+  workflow_run:
+    workflows: [CI]
+    types:
+      - completed
+
+jobs:
+  publish-codecov:
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download Workflow Info
+        uses: actions/download-artifact@v4
+        with:
+          name: workflow-info
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Set Environment Variables from JSON
+        run: |
+          ls
+          # Read the JSON file and export each key-value pair as an environment variable
+          for key in $(jq -r 'keys[]' ./workflow-info.json); do
+            value=$(jq -r --arg key "$key" '.[$key]' ./workflow-info.json)
+            echo "workflow-info_$key=$value"
+            echo "workflow-info_$key=$value" >> $GITHUB_ENV
+          done
+      - name: Download Coverage Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: ./coverage-reports
+          pattern: coverage-reports-*
+          run-id: ${{ github.event.workflow_run.id }}
+
+      - name: Upload coverage report
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+          files: /**/coverage-reports/**/*.*
+          flags: unittests
+          override_branch: ${{ env.workflow-info_branch }}
+          override_commit: ${{ env.workflow-info_commit }}
+          override_pr: ${{ env.workflow-info_pr_number }}
+          commit_parent: ${{ env.workflow-info_commit_parent }}
+          override_build_url: ${{ env.workflow-info_build_url }}
+          override_build:  ${{ env.workflow-info_build }}
+          token: ${{ secrets.CODECOV_TOKEN }}
+          verbose: true
diff --git a/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/Pipfile b/TrafficCapture/dockerSolution/src/main/docker/migrationConsole/console_api/Pipfile
@@ -19,3 +19,7 @@ moto = {version = "*", extras = ["sts"]}
 
 [requires]
 python_version = "3.11"
+
+[scripts]
+test = "coverage run --source='.' manage.py test console_api"
+coverage = "coverage"