Parallelize pytest execution jobs

.dockerignore

@@ -9,3 +9,4 @@ docs/
 .git/
 .idea/
 .DS_Store
+.cov/

.github/actions/setup-test-image/action.yml

@@ -11,6 +11,7 @@ runs:
         images: ibet-wallet-api
         tags: |
           type=sha,format=short
+    - run: ls -al
     - uses: docker/build-push-action@v6
       with:
         context: .

.github/workflows/pr.yml

@@ -14,58 +14,116 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - uses: chartboost/ruff-action@v1
-  unit-test-postgres:
-    name: 'Unit tests (PostgreSQL)'
+  unit-test:
+    name: 'Unit tests'
     runs-on: ubuntu-latest
     needs: build
+    strategy:
+      fail-fast: true
+      matrix:
+        db: [postgres, mysql]
+        test_target: [
+          'tests/app/admin_* tests/app/company_* tests/app/config_* tests/app/contract_*',
+          'tests/app/dex_*',
+          'tests/app/e2e_* tests/app/eth_*',
+          'tests/app/events_*',
+          'tests/app/messaging_* tests/app/node_info_* tests/app/notification_* tests/app/openapi_*',
+          'tests/app/position_ListAllCouponConsumptions_*',
+          'tests/app/position_Position_*',
+          'tests/app/position_PositionCoupon* tests/app/position_PositionMembership*',
+          'tests/app/position_PositionShare*',
+          'tests/app/position_PositionStraightBond*',
+          'tests/app/token_*',
+          'tests/app/userinfo_*',
+          'tests/batch/indexer_Block_* tests/batch/indexer_CompanyList_* tests/batch/indexer_Consume_*',
+          'tests/batch/indexer_DEX_*',
+          'tests/batch/indexer_Position_Bond_*',
+          'tests/batch/indexer_Position_Coupon_*',
+          'tests/batch/indexer_Position_Membership_*',
+          'tests/batch/indexer_Position_Share_*',
+          'tests/batch/indexer_Token_Detail_*',
+          'tests/batch/indexer_Token_Holders_*',
+          'tests/batch/indexer_Token_List_*',
+          'tests/batch/indexer_Transfer_*',
+          'tests/batch/processor_*',
+        ]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Change owner of output directory
+        run: |
+          sudo chown runner:docker /home/runner/work/ibet-Wallet-API/ibet-Wallet-API/cov
+          sudo chmod 777 /home/runner/work/ibet-Wallet-API/ibet-Wallet-API/cov
+      - uses: ./.github/actions/setup-test-image
+      - run: ls -al
+      - name: run unit test using ${{ matrix.db }}
+        run: docker compose run -e TEST_TARGET="${{ matrix.test_target }}" ibet-wallet-api-${{ matrix.db }}
+      - run: mv cov/.coverage cov/.coverage-${{ strategy.job-index }}
+      - uses: actions/upload-artifact@v4
+        if: matrix.db == 'postgres'
+        with:
+          name: .coverage-${{ strategy.job-index }}
+          path: cov/.coverage-${{ strategy.job-index }}
+          include-hidden-files: true
+      - run: mv cov/pytest.xml cov/pytest-${{ strategy.job-index }}.xml
+      - uses: actions/upload-artifact@v4
+        if: matrix.db == 'postgres'
+        with:
+          name: pytest-${{ strategy.job-index }}
+          path: cov/pytest-${{ strategy.job-index }}.xml
+  collect_coverage:
+    runs-on: ubuntu-latest
+    needs: unit-test
     permissions:
       actions: read
       contents: read
       packages: read
       pull-requests: write
     steps:
-      - name: Collect Workflow Telemetry
-        uses: catchpoint/workflow-telemetry-action@v2
-        with:
-          comment_on_pr: false
       - uses: actions/checkout@v4
+      - uses: ./.github/actions/setup-test-image
       - name: Change owner of output directory
         run: |
-          sudo chown -R runner:docker /home/runner/work/ibet-Wallet-API/ibet-Wallet-API/cov
+          sudo chown runner:docker /home/runner/work/ibet-Wallet-API/ibet-Wallet-API/cov
           sudo chmod 777 /home/runner/work/ibet-Wallet-API/ibet-Wallet-API/cov
-      - name: run unit test using postgres
-        run: docker compose run ibet-wallet-api-postgres
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: '.coverage-*'
+          path: cov/
+          merge-multiple: true
+      - name: combine coverage files
+        run: docker compose run -w /app/ibet-Wallet-API/cov ibet-wallet-api-postgres bash --login -c 'poetry run coverage combine .coverage-*'
+      - name: generate coverage xml file
+        run: docker compose run -w /app/ibet-Wallet-API ibet-wallet-api-postgres bash --login -c 'poetry run coverage xml --data-file=cov/.coverage -o cov/coverage.xml'
+      - uses: actions/download-artifact@v4
+        with:
+          pattern: 'pytest-*'
+          path: cov/
+          merge-multiple: true
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install junit-report-merger
+        run: npm install -g junit-report-merger
+      - name: Merge reports
+        run: jrm cov/pytest.xml "cov/pytest-*.xml"
       - name: Pytest coverage comment
         id: coverageComment
         uses: MishaKav/pytest-coverage-comment@main
         with:
+          report-only-changed-files: true
           pytest-xml-coverage-path: cov/coverage.xml
           junitxml-path: cov/pytest.xml
-          report-only-changed-files: true
-  unit-test-mysql:
-    name: 'Unit tests (MySQL)'
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-test-image
-      - name: run unit test using mysql
-        run: docker compose run ibet-wallet-api-mysql
-  migration-test-postgres:
-    name: 'Migration tests (PostgreSQL)'
-    runs-on: ubuntu-latest
-    needs: build
-    steps:
-      - uses: actions/checkout@v4
-      - uses: ./.github/actions/setup-test-image
-      - name: run unit test using postgres
-        run: docker compose run ibet-wallet-api-postgres bash --login -c "cd /app/ibet-Wallet-API && poetry run pytest -vv --test-alembic -m 'alembic'"
-  migration-test-mysql:
-    name: 'Migration tests (MySQL)'
+  migration-test:
+    name: 'Migration tests'
     runs-on: ubuntu-latest
     needs: build
+    strategy:
+      fail-fast: true
+      matrix:
+        db: [postgres, mysql]
     steps:
       - uses: actions/checkout@v4
       - uses: ./.github/actions/setup-test-image
-      - name: run unit test using mysql
-        run: docker compose run ibet-wallet-api-mysql bash --login -c "cd /app/ibet-Wallet-API && poetry run pytest -vv --test-alembic -m 'alembic'"
+      - run: ls -al
+      - name: run unit test using ${{ matrix.db }}
+        run: docker compose run ibet-wallet-api-${{ matrix.db }} bash --login -c "cd /app/ibet-Wallet-API && poetry run pytest -vv --test-alembic -m 'alembic'"

.gitignore

@@ -25,4 +25,5 @@ node_modules/
 
 # Project
 cov/coverage.xml
-cov/pytest.xml
+cov/pytest.xml
+cov/.coverage

pyproject.toml

@@ -97,6 +97,13 @@ combine-as-imports = true
 [tool.coverage.run]
 branch = true
 concurrency = ["thread", "greenlet"]
+include = [
+    "app/api/**/*",
+    "app/model/**/*",
+    "app/utils/**/*",
+    "batch/**/*",
+    "cmd/**/*"
+]
 
 [build-system]
 requires = ["poetry-core>=1.0.0"]

tests/qa.sh

@@ -21,16 +21,19 @@ source ~/.bash_profile
 
 cd /app/ibet-Wallet-API
 
+TEST_TARGET=${TEST_TARGET:-tests/}
+
 sleep 10
 
 # test
-poetry run pytest -v --cov=app/api/ --cov=app/model/ --cov=app/utils/ --cov=batch/ --cov=cmd/ --junitxml=pytest.xml --cov-report=xml --cov-report=term-missing:skip-covered --cov-branch tests/
+poetry run pytest -v --cov=. --junitxml=pytest.xml --cov-report=xml --cov-report=term-missing:skip-covered --cov-branch $TEST_TARGET
 
 status_code=$?
 
 # カバレッジファイルの移動
 mv coverage.xml cov/
 mv pytest.xml cov/
+mv .coverage cov/
 
 if [ $status_code -ne 0 ]; then
   exit 1