test.yaml

# We use self-hosted runners for performance: we can rely on docker caching
# on the runner machines for 0s startup times for our tests.
# There is an issue with self-hosted runners where if you use a mix of
# host and container jobs, the host jobs will fail to start
# (https://github.com/actions/checkout/issues/273).
# So we create a separate "builder" tagged runner to do the build job on
# the host, and then all the tests run on "runner" tagged runners.

name: test

on:
  push:

jobs:
  build-container:
    name: Build test container
    timeout-minutes: 30
    runs-on: [self-hosted, builder]
    # runs-on: ubuntu-latest
    env:
      REGISTRY: us-east4-docker.pkg.dev/weave-support-367421/weave-images
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 2

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          registry: us-east4-docker.pkg.dev
          username: _json_key
          password: ${{ secrets.gcp_sa_key }}

      # this script is hardcoded to build for linux/amd64
      - name: Prune docker cache
        run: docker system prune -f
      - name: Build unit test image
        run: python3 weave/docker/docker_build.py build_deps weave-test-python-client builder . weave/docker/Dockerfile.python-client.test
      - name: Build integration test image
        run: python3 weave/docker/docker_build.py build weave-integration-test . weave/legacy/Dockerfile.test

  lint:
    name: Python lint
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install tox
      - name: Run tox
        run: tox -e lint

  weavejs-lint-compile:
    name: WeaveJS Lint and Compile
    runs-on: ubuntu-latest
    # runs-on: [self-hosted, gke-runner]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/setup-node@v1
        with:
          node-version: '18.x'
      - run: |
          set -e
          cd weave-js
          yarn install --frozen-lockfile
          yarn generate
          yarn eslint --max-warnings=0
          yarn tslint
          yarn prettier
          yarn run tsc

  trace-tests:
    name: Trace tox tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version-major: ['3']
        python-version-minor: [
            '9',
            '10',
            '11',
            # '12',  # TODO: We have actual failing tests in 3.12, but commenting for simplicity for now.
            #
          ]
        tox-shard:
          [
            'trace',
            'trace_server',
            'anthropic',
            'cerebras',
            'cohere',
            'dspy',
            'groq',
            'langchain',
            'litellm',
            'llamaindex',
            'mistral0',
            'mistral1',
            'openai',
          ]
      fail-fast: false
    services:
      wandbservice:
        image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master
        credentials:
          username: _json_key
          password: ${{ secrets.gcp_wb_sa_key }}
        env:
          CI: 1
          WANDB_ENABLE_TEST_CONTAINER: true
        ports:
          - '8080:8080'
          - '8083:8083'
          - '9015:9015'
        options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s
      weave_clickhouse:
        image: clickhouse/clickhouse-server
        ports:
          - '8123:8123'
        options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s

    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install tox
      - name: Run tox (Clickhouse Only)
        env:
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WB_SERVER_HOST: http://wandbservice
          WF_CLICKHOUSE_HOST: weave_clickhouse
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
        run: |
          tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} -- \
            -m "weave_client and not skip_clickhouse_client" \
            --weave-server=clickhouse
      - name: Run tox
        env:
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WB_SERVER_HOST: http://wandbservice
          WF_CLICKHOUSE_HOST: weave_clickhouse
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
        run: |
          tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }}

  test:
    name: Python unit tests
    timeout-minutes: 15 # do not raise! running longer than this indicates an issue with the tests. fix there.
    needs:
      - build-container
    # runs-on: [self-hosted, gke-runner]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        job_num: [0, 1]

    # runs-on: ubuntu-latest
    container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-client:${{ github.sha }}
    services:
      wandbservice:
        image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master
        credentials:
          username: _json_key
          password: ${{ secrets.gcp_wb_sa_key }}
        env:
          CI: 1
          WANDB_ENABLE_TEST_CONTAINER: true
        ports:
          - '8080:8080'
          - '8083:8083'
          - '9015:9015'
        options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s
      weave_clickhouse:
        image: clickhouse/clickhouse-server
        ports:
          - '8123:8123'
        options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s
    steps:
      # - uses: datadog/agent-github-action@v1.3
      #   with:
      #     api_key: ${{ secrets.DD_API_KEY }}
      - uses: actions/checkout@v2
      - name: Verify wandb server is running
        run: curl -s http://wandbservice:8080/healthz
      - name: Run Legacy (Query Service) Python Unit Tests
        env:
          DD_SERVICE: weave-python
          DD_ENV: ci
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WB_SERVER_HOST: http://wandbservice
          WF_CLICKHOUSE_HOST: weave_clickhouse
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
        run: |
          source /root/venv/bin/activate && \
          pip install 'dspy-ai==2.4.13' && \
          pip install pydantic -U && \
          cd weave && \
          pytest \
          --job-num=${{ matrix.job_num }} \
          --timeout=90 \
          --ddtrace \
          --durations=5 \
          ./legacy