nightly-eval.yml

name: Nightly Evaluation

on:
  schedule:
    - cron: '0 0 * * *'
  push:
    branches:
      - main
    paths:
      - "python/sglang/version.py"
  workflow_dispatch:

concurrency:
  group: nightly-eval-${{ github.ref }}
  cancel-in-progress: true

jobs:
  nightly-eval-2-gpu:
    if: github.repository == 'sgl-project/sglang' || github.event_name == 'pull_request'
    runs-on: 2-gpu-runner
    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Install dependencies
        run: |
          bash scripts/ci_install_dependency.sh
          pip install --upgrade "evalplus[vllm] @ git+https://github.com/evalplus/evalplus"

      - name: Test gsm8k
        timeout-minutes: 120
        run: |
          cd test/srt
          python3 test_nightly_gsm8k_eval.py

      - name: Test human eval
        timeout-minutes: 120
        run: |
          cd test/srt
          python3 test_nightly_human_eval.py