Conformance Testing #856
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Conformance Testing | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| # Every day at midnight | |
| - cron: '0 0 * * *' | |
| jobs: | |
| # conformance-run works through a matrix of actors and runs the conformance | |
| # test suite for each of them. Results are stored in a `reports` artifact for | |
| # later compilation and publishing. | |
| conformance-run: | |
| name: Run Conformance Suite | |
| runs-on: ubuntu-latest | |
| strategy: | |
| # Job must not stop iterating through matrix on failure | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: "Mavennet" | |
| actor: "MAVENNET_STAGING" | |
| - name: "mesur.io" | |
| actor: "MESUR_IO_PRODUCTION" | |
| - name: "GS1US" | |
| actor: "GS1US" | |
| - name: 'BCGov' | |
| actor: 'BCGOV' | |
| steps: | |
| # Check out repo, set up node, and install dependencies. | |
| # @see https://github.com/actions/setup-node#usage | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.15.1 | |
| cache: 'npm' | |
| - run: npm ci | |
| - name: Run Tests | |
| env: | |
| organization_did_web: ${{secrets[format('{0}_ORGANIZATION_DID_WEB', matrix.actor)]}} | |
| client_id: ${{secrets[format('{0}_CLIENT_ID', matrix.actor)]}} | |
| client_secret: ${{secrets[format('{0}_CLIENT_SECRET', matrix.actor)]}} | |
| token_audience: ${{secrets[format('{0}_TOKEN_AUDIENCE', matrix.actor)]}} | |
| token_endpoint: ${{secrets[format('{0}_TOKEN_ENDPOINT', matrix.actor)]}} | |
| api_base_url: ${{secrets[format('{0}_API_BASE_URL', matrix.actor)]}} | |
| run: | | |
| npx newman run ./tests/conformance_suite.postman_collection.json \ | |
| --env-var ORGANIZATION_DID_WEB=$organization_did_web \ | |
| --env-var CLIENT_ID=$client_id \ | |
| --env-var CLIENT_SECRET=$client_secret \ | |
| --env-var TOKEN_AUDIENCE=$token_audience \ | |
| --env-var TOKEN_ENDPOINT=$token_endpoint \ | |
| --env-var API_BASE_URL=$api_base_url \ | |
| --reporters cli,htmlextra,json \ | |
| --reporter-htmlextra-skipSensitiveData \ | |
| --reporter-htmlextra-export "newman/${{format('{0}-{1}-{2}.html', github.run_id, github.job, matrix.name)}}" \ | |
| --reporter-json-export "newman/${{format('{0}-{1}-{2}.json', github.run_id, github.job, matrix.name)}}" | |
| # Write sanitized Newman output to `./docs/reports`. | |
| - name: Sanitize Report Output | |
| if: always() # Run even when postman tests fail | |
| run: npm run report:sanitize | |
| # Sanitized reports are needed by subsequent jobs | |
| - uses: actions/upload-artifact@v3 | |
| if: always() # Run even when postman tests fail | |
| with: | |
| name: reports | |
| path: docs/reports/ | |
| # conformance-compile compiles the sanitized raw report output into an | |
| # interactive HTML conformance report suitable for publishing. | |
| conformance-compile: | |
| name: Compile Reports | |
| runs-on: ubuntu-latest | |
| # Run even when postman tests fail | |
| if: always() | |
| # Wait for jobs which generate reporting input. | |
| needs: | |
| - conformance-run | |
| steps: | |
| # Check out repo, setup node, and install dependencies. | |
| # @see https://github.com/actions/setup-node#usage | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: 16.15.1 | |
| cache: 'npm' | |
| - run: npm ci | |
| # Download 'reports' artifact | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: reports | |
| path: docs/reports/ | |
| # Build report index | |
| - name: Build JSON Index | |
| run: npm run report:index -- --folder conformance | |
| # Set up Python 3.10 | |
| - name: Set up Python 3.10 | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10" | |
| cache: pip | |
| - run: pip install -r ./reporting/requirements.txt | |
| # Compile interactive report | |
| - name: Build Conformance Report | |
| working-directory: ./reporting | |
| run: ./reporter.py --mode ci --conformance | |
| # Compiled report must be added to artifact | |
| - uses: actions/upload-artifact@v3 | |
| with: | |
| name: reports | |
| path: docs/reports/ | |
| # conformance-publish publishes sanitized raw and interactive HTML reports to | |
| # the `reports/conformance` directory in GitHub pages. This job is restricted | |
| # to commits on the `main` branch to prevent feature branches from overwriting | |
| # production reports. | |
| conformance-publish: | |
| name: Publish Report | |
| runs-on: ubuntu-latest | |
| # Publishing must only happen when triggered on the 'main' branch to prevent | |
| # tests on other branches from polluting the published reports. Note that | |
| # `always()`` is a special flag that is REQUIRED in order to get this job to | |
| # run if jobs listed in the `needs:` section fail. | |
| if: github.ref == 'refs/heads/main' && always() | |
| # Report must be compiled before publishing | |
| needs: | |
| - conformance-compile | |
| steps: | |
| # Check out repo | |
| - uses: actions/checkout@v3 | |
| # Download 'reports' artifact | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: reports | |
| path: docs/reports/ | |
| # Publish report subfolder to GitHub Pages | |
| - name: Publish Conformance Report | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./docs/reports | |
| destination_dir: reports/conformance | |
| # Prepare an archive folder name and update the index.json appropriately | |
| - name: Prepare Conformance Archive | |
| id: archive | |
| run: | | |
| FOLDER=conformance-$(date +'%s') | |
| sed -i "s#reports/conformance/#reports/$FOLDER/#" ./docs/reports/index.json | |
| echo "::set-output name=folder::$FOLDER" | |
| # Publish report archive subfolder to GitHub Pages | |
| - name: Publish Conformance Archive | |
| uses: peaceiris/actions-gh-pages@v3 | |
| with: | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| publish_dir: ./docs/reports | |
| destination_dir: reports/${{ steps.archive.outputs.folder }} |