run-api-test.yml

name: API Test

on:
  workflow_call:
    inputs:
      environment:
        type: string
        required: true
        description: "where to test"
  workflow_dispatch:
    inputs:
      environment:
        type: choice
        required: true
        description: "where to test"
        default: "tools"
        options:
          - tools
          - production
  repository_dispatch:
    types:
      - webhook

jobs:
  define-test-matrix:
    runs-on: ubuntu-latest
    environment: ${{ inputs.environment }}
    outputs:
      tests: ${{ steps.tests.outputs.tests }}
    steps:
      - uses: actions/checkout@v4
      - id: tests
        working-directory: bin/si-api-test/tests
        run: |
          # Find .ts files, remove ./ prefix, and format as JSON array
          files=$(find "." -mindepth 1 -maxdepth 1 -type f -name "*.ts" | sed -r "s/\.\/(.*)\.ts/\1/" | sort)
          # Get the number of tests
          test_count=$(echo "$files" | wc -l)

          # Get the list of workspace IDs from the environment variable
          workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}"
          echo "workspace_ids found to be $workspace_ids"
          workspace_count=$(echo "$workspace_ids" | tr ',' '\n' | wc -l)

          # Validate that the number of workspace IDs is greater than or equal to the number of tests
          if [ "$workspace_count" -lt "$test_count" ]; then
              echo "Error: The number of workspace IDs ($workspace_count) is less than the number of tests ($test_count)."
              exit 1
          fi

          # Format files as JSON array with correct numbering and sorted order
          indexed_files=$(echo "$files" | awk '{print "{\"name\": \"" $0 "\", \"index\": " NR-1 "}"}' | jq -s .)

          # Ensure indexed_files are formatted correctly
          test_output=$(echo "$indexed_files" | jq -c '.')
          echo "tests=$test_output" >> "$GITHUB_OUTPUT"
          echo "$test_output"

  api-test:
    name: API Test SDF
    environment: ${{ inputs.environment }}
    runs-on: ubuntu-latest
    needs: define-test-matrix
    strategy:
      fail-fast: false
      matrix:
        tests: ${{ fromJSON(needs.define-test-matrix.outputs.tests) }}
    env:
      SDF_API_URL: ${{ vars.SDF_API_URL }}
      AUTH_API_URL: ${{ vars.AUTH_API_URL }}
    outputs:
      last_exit_code: ${{ steps.capture-exit-code.outputs.last_exit_code }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Install Deno
        uses: denoland/setup-deno@v1
        with:
          deno-version: v1.x

      - name: Push failure of exit 53 to test code
        run: |
          exit 53

      - name: Run the deno exec with retry
        run: |
          cd bin/si-api-test
          
          echo "Running test ${{ matrix.tests.name }} with index: ${{ matrix.tests.index }}"

          # Split the workspace IDs into an array
          workspace_ids="${{ vars.API_TEST_WORKSPACE_IDS }}"
          IFS=',' read -r -a workspace_array <<< "$workspace_ids"
          
          # Pick the correct workspace ID based on the index
          workspace_id=${workspace_array[${{ matrix.tests.index }}]}

          echo "Using workspace ID: $workspace_id"

          # Retry loop with 5 attempts
          n=0
          max_retries=5
          exit_code=0
          
          until [ $n -ge $max_retries ]
          do
            unset exit_code || echo "exit_code not set"

            # Run the deno task and store exit code in a variable
            deno task run \
            --workspaceId "$workspace_id" \
            --userId ${{ secrets.API_TEST_EMAIL }} \
            --password ${{ secrets.API_TEST_PASSWORD }} \
            --tests ${{ matrix.tests.name }} || exit_code=$?
            
            # Check the exit code
            if [ -z "$exit_code" ]; then
              echo "Deno task succeeded!"
              break
            fi

            n=$((n+1))
            echo "Attempt $n/$max_retries failed with exit code $exit_code! Retrying..."
            sleep 60
          done

          if [ $n -ge $max_retries ]; then
            echo "All $max_retries attempts failed."
            exit_code=5  # Set your custom exit code
          fi

          echo "last_exit_code=$exit_code" >> "$GITHUB_OUTPUT"
          exit $last_exit_code

  on-failure:
    runs-on: ubuntu-latest
    needs: api-test
    environment: ${{ inputs.environment }}
    if: ${{ failure() }} #&& github.ref == 'refs/heads/main' }} #TODO(johnrwatson): disable comment before merging
    steps:
      - run: |
          VALID_FAILURE_EXIT_CODES="53"
          if echo "$VALID_FAILURE_EXIT_CODES" | grep -qw "$LAST_EXIT_CODE"; then
            curl --location "${{ secrets.FIREHYDRANT_WEBHOOK_URL }}" \
            --header "Content-Type: application/json" \
            --data "{
              \"summary\": \"API ${{ inputs.environment }} Tests Fail\",
              \"body\": \"API Tests have failed for ${{ inputs.environment }}.\",
              \"links\": [
                {
                  \"href\": \"https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID\",
                  \"text\": \"E2E Test Run ${{ inputs.environment }}\"
                }
              ],
              \"tags\": [
                \"service:github\"
              ]
            }" 
          else # Page someone as it's not explicitly excluded
            echo "Exit code $LAST_EXIT_CODE is in the excluded list, doing nothing."
          fi
      - run: |
          curl --location "${{ secrets.SLACK_WEBHOOK_URL }}" -X POST \
            --header 'Content-type: application/json' \
            --data "{\"text\": \":si: Failed API Tests for ${{ inputs.environment }}: <https://github.com/systeminit/si/actions/runs/$GITHUB_RUN_ID|:test_tube: Link>\"}"