API Test #54
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: API Test | |
| on: | |
| workflow_call: | |
| inputs: | |
| environment: | |
| type: string | |
| required: true | |
| description: "where to test" | |
| workflow_dispatch: | |
| inputs: | |
| environment: | |
| type: choice | |
| required: true | |
| description: "where to test" | |
| default: "tools" | |
| options: | |
| - tools | |
| - production | |
| repository_dispatch: | |
| types: | |
| - webhook | |
| jobs: | |
| define-test-matrix: | |
| runs-on: ubuntu-latest | |
| environment: ${{ inputs.environment }} | |
| outputs: | |
| tests: ${{ steps.tests.outputs.tests }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - id: tests | |
| working-directory: bin/si-api-test/tests | |
| run: | | |
| # Find .ts files, remove ./ prefix, and format as JSON array | |
| files=$(find "." -mindepth 1 -maxdepth 1 -type f -name "*.ts" | sed -r "s/\.\/(.*)\.ts/\1/" | sort) | |
| # Get the number of tests | |
| test_count=$(echo "$files" | wc -l) | |
| # Get the list of workspace IDs from the environment variable | |
| workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}" | |
| echo "workspace_ids found to be $workspace_ids" | |
| workspace_count=$(echo "$workspace_ids" | tr ',' '\n' | wc -l) | |
| # Validate that the number of workspace IDs is greater than or equal to the number of tests | |
| if [ "$workspace_count" -lt "$test_count" ]; then | |
| echo "Error: The number of workspace IDs ($workspace_count) is less than the number of tests ($test_count)." | |
| exit 1 | |
| fi | |
| # Format files as JSON array with correct numbering and sorted order | |
| indexed_files=$(echo "$files" | awk '{print "{\"name\": \"" $0 "\", \"index\": " NR-1 "}"}' | jq -s .) | |
| # Ensure indexed_files are formatted correctly | |
| test_output=$(echo "$indexed_files" | jq -c '.') | |
| echo "tests=$test_output" >> "$GITHUB_OUTPUT" | |
| echo "$test_output" | |
| api-test: | |
| name: API Test SDF | |
| environment: ${{ inputs.environment }} | |
| runs-on: ubuntu-latest | |
| needs: define-test-matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| tests: ${{ fromJSON(needs.define-test-matrix.outputs.tests) }} | |
| env: | |
| SDF_API_URL: ${{ vars.SDF_API_URL }} | |
| AUTH_API_URL: ${{ vars.AUTH_API_URL }} | |
| outputs: | |
| last_exit_code: ${{ steps.capture-exit-code.outputs.last_exit_code }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Install Deno | |
| uses: denoland/setup-deno@v1 | |
| with: | |
| deno-version: v1.x | |
| - name: Push failure of exit 53 to test code randomly | |
| run: | | |
| [[ "$(( ( RANDOM % 5 ) + 1 ))" == "3" ]] && exit 53 | |
| exit 0 | |
| - name: Run the deno exec with retry | |
| run: | | |
| cd bin/si-api-test | |
| echo "Running test ${{ matrix.tests.name }} with index: ${{ matrix.tests.index }}" | |
| # Split the workspace IDs into an array | |
| workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}" | |
| IFS=',' read -r -a workspace_array <<< "$workspace_ids" | |
| # Pick the correct workspace ID based on the index | |
| workspace_id=${workspace_array[${{ matrix.tests.index }}]} | |
| echo "Using workspace ID: $workspace_id" | |
| # Retry loop with 5 attempts | |
| n=0 | |
| max_retries=5 | |
| exit_code=0 | |
| until [ $n -ge $max_retries ] | |
| do | |
| unset exit_code || echo "exit_code not set" | |
| # Run the deno task and store exit code in a variable | |
| deno task run \ | |
| --workspaceId "$workspace_id" \ | |
| --userId ${{ secrets.API_TEST_EMAIL }} \ | |
| --password ${{ secrets.API_TEST_PASSWORD }} \ | |
| --tests ${{ matrix.tests.name }} || exit_code=$? | |
| # Check the exit code | |
| if [ -z "$exit_code" ]; then | |
| echo "Deno task succeeded!" | |
| break | |
| fi | |
| n=$((n+1)) | |
| echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..." | |
| sleep 60 | |
| done | |
| if [ $n -ge $max_retries ]; then | |
| echo "All $max_retries attempts failed." | |
| exit_code=0 | |
| fi | |
| echo "last_exit_code=$exit_code" >> "$GITHUB_OUTPUT" | |
| exit $last_exit_code | |
| on-failure: | |
| runs-on: ubuntu-latest | |
| needs: api-test | |
| environment: ${{ inputs.environment }} | |
| if: ${{ failure() }} #&& github.ref == 'refs/heads/main' }} #TODO(johnrwatson): disable comment before merging | |
| steps: | |
| - run: | | |
| VALID_FAILURE_EXIT_CODES="53" | |
| echo "consolidated:" | |
| echo "{{ needs.api-test.outputs.exit_code | join(",") }}" | |
| echo "alt:" | |
| echo "{{ needs.api-test.outputs.exit_code }}" | |
| exit_codes='[{{ needs.api-test.outputs.exit_code | join(",") }}] | jq -r '.[]'' | |
| # Check if any exit code is "bad" | |
| for code in $exit_codes; do | |
| if echo "$VALID_FAILURE_EXIT_CODES" | grep -qw "$code"; then | |
| echo "Bad exit code detected: $code" | |
| curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \ | |
| --header "Content-Type: application/json" \ | |
| --data "{ | |
| \"summary\": \"API ${{ inputs.environment }} Tests Fail\", | |
| \"body\": \"API Tests have failed for ${{ inputs.environment }}.\", | |
| \"links\": [ | |
| { | |
| \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\", | |
| \"text\": \"E2E Test Run ${{ inputs.environment }}\" | |
| } | |
| ], | |
| \"tags\": [ | |
| \"service:github\" | |
| ] | |
| }" | |
| fi | |
| done | |
| - run: | | |
| curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \ | |
| --header 'Content-type: application/json' \ | |
| --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}" |