test.yaml

# We use self-hosted runners for performance: we can rely on docker caching
# on the runner machines for 0s startup times for our tests.
# There is an issue with self-hosted runners where if you use a mix of
# host and container jobs, the host jobs will fail to start
# (https://github.com/actions/checkout/issues/273).
# So we create a separate "builder" tagged runner to do the build job on
# the host, and then all the tests run on "runner" tagged runners.

name: test

on:
  push:

jobs:
  build-container:
    name: Build test container
    timeout-minutes: 30
    runs-on: [self-hosted, builder]
    # runs-on: ubuntu-latest
    env:
      REGISTRY: us-east4-docker.pkg.dev/weave-support-367421/weave-images
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 2

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          registry: us-east4-docker.pkg.dev
          username: _json_key
          password: ${{ secrets.gcp_sa_key }}

      # this script is hardcoded to build for linux/amd64
      - name: Prune docker cache
        run: docker system prune -f
      - name: Build unit test image
        run: python3 weave/docker/docker_build.py build_deps weave-test-python-client builder . weave/docker/Dockerfile.python-client.test
      - name: Build integration test image
        run: python3 weave/docker/docker_build.py build weave-integration-test . weave/legacy/Dockerfile.test

  lint:
    name: Python lint
    timeout-minutes: 7
    needs:
      - build-container
    # runs-on: [self-hosted, gke-runner]
    runs-on: ubuntu-latest
    container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-client:${{ github.sha }}
    steps:
      - uses: actions/checkout@v2
      - run: source /root/venv/bin/activate && pre-commit run --hook-stage=pre-push --all-files

  weavejs-lint-compile:
    name: WeaveJS Lint and Compile
    runs-on: ubuntu-latest
    # runs-on: [self-hosted, gke-runner]
    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0
          token: ${{ secrets.GITHUB_TOKEN }}
      - uses: actions/setup-node@v1
        with:
          node-version: "18.x"
      - run: |
          set -e
          cd weave-js
          yarn install --frozen-lockfile
          yarn generate
          yarn eslint --max-warnings=0
          yarn tslint
          yarn prettier
          yarn run tsc

  test:
    name: Python unit tests
    timeout-minutes: 15 # do not raise! running longer than this indicates an issue with the tests. fix there.
    needs:
      - build-container
    # runs-on: [self-hosted, gke-runner]
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        job_num: [0, 1]

    # runs-on: ubuntu-latest
    container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-client:${{ github.sha }}
    services:
      wandbservice:
        image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master
        credentials:
          username: _json_key
          password: ${{ secrets.gcp_wb_sa_key }}
        env:
          CI: 1
          WANDB_ENABLE_TEST_CONTAINER: true
        ports:
          - "8080:8080"
          - "8083:8083"
          - "9015:9015"
        options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s
      weave_clickhouse:
        image: clickhouse/clickhouse-server
        ports:
          - "8123:8123"
        options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s
    steps:
      # - uses: datadog/agent-github-action@v1.3
      #   with:
      #     api_key: ${{ secrets.DD_API_KEY }}
      - uses: actions/checkout@v2
      - name: Verify wandb server is running
        run: curl -s http://wandbservice:8080/healthz
      - name: Run Python Unit Tests (Clickhouse Client Only)
        env:
          DD_SERVICE: weave-python
          DD_ENV: ci
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WB_SERVER_HOST: http://wandbservice
          WF_CLICKHOUSE_HOST: weave_clickhouse
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
        # This runner specifically runs the tests that use the `client` fixture (those that support clickhouse client tests)
        # However, we skip tests marked with `skip_clickhouse_client`. These should be considered TODOs and an exception
        run: |
          source /root/venv/bin/activate && \
          cd weave && \
          pytest -m "weave_client and not skip_clickhouse_client" \
          --weave-server=clickhouse \
          --job-num=${{ matrix.job_num }} \
          --timeout=90 \
          --ddtrace \
          --durations=5 \
          ./integrations ./legacy ./trace_server ./trace ./tests
      - name: Run Python Unit Tests
        env:
          DD_SERVICE: weave-python
          DD_ENV: ci
          WEAVE_SENTRY_ENV: ci
          CI: 1
          WB_SERVER_HOST: http://wandbservice
          WF_CLICKHOUSE_HOST: weave_clickhouse
          WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
        run: |
          source /root/venv/bin/activate && \
          pip install 'dspy-ai==2.4.13' && \
          pip install pydantic -U && \
          cd weave && \
          pytest \
          --job-num=${{ matrix.job_num }} \
          --timeout=90 \
          --ddtrace \
          --durations=5 \
          ./integrations ./legacy ./trace_server ./trace ./tests

  # nbmake:
  #   name: Run notebooks with nbmake
  #   runs-on: self-hosted
  #   container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test:latest
  #   steps:
  #     - uses: actions/checkout@v2

  #     - name: Run notebooks
  #       run: source /root/venv/bin/activate && export PYTHONPATH=$(pwd) && pytest -n=4 --nbmake --overwrite examples
  #     - name: Upload executed notebooks
  #       uses: actions/upload-artifact@v3
  #       if: always()
  #       with:
  #         name: notebooks
  #         path: examples

  cypress-run:
    name: Notebook and UI tests
    timeout-minutes: 25 # 15 minute timeout routinely trips on rerun
    needs:
      - build-container
      # - lint
      # - test
    if: always()
    # runs-on: [self-hosted, gke-runner]
    runs-on: ubuntu-latest

    container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-integration-test:${{ github.sha }}

    strategy:
      fail-fast: false
      matrix:
        containers: [1, 2, 3, 4, 5, 6]
    steps:
      - uses: actions/checkout@v3
      - name: Setup W&B API key
        run: echo "WANDB_API_KEY=${{ secrets.WANDB_API_KEY }}" >> $GITHUB_ENV
      - name: Setup Replicate API token
        run: echo "REPLICATE_API_TOKEN=${{ secrets.REPLICATE_API_TOKEN }}" > $GITHUB_ENV
      - name: Copy node_modules from container to checkout
        run: cp -R /root/integration_test/node_modules ./integration_test/
      - name: Activate venv
        run: source /root/venv/bin/activate && echo "PATH=$PATH" >> $GITHUB_ENV
      - name: Make Log Dir
        run: mkdir -p /tmp/weave/log
      - name: Copy over built assets
        run: cp -r /root/weave-js-build/* ./weave/frontend
      - name: Start weave server
        # github actions does something funky with the std file descriptors, they end up
        # being closed. tqdm (for example) raises an exception when the descriptor it
        # wants to write to is closed.
        run: nohup ./scripts/weave_server_test.sh < /dev/null &> /tmp/weave/log/stdout.log &
        shell: bash
      - name: Cypress run
        # Use the following to run just a single test
        # run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium --spec "cypress/e2e/notebooks/Ops that return images.cy.ts"
        # run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium
        run: export PYTHONPATH=$(pwd) && cd integration_test &&  npx cypress run --browser chrome
        # uses: cypress-io/github-action@v4
        # with:
        #   working-directory: ./integration_testG
        #   record: true
        #   parallel: true
        env:
          REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          SPLIT: ${{ strategy.job-total }}
          SPLIT_INDEX: ${{ strategy.job-index }}
          # - uses: actions/upload-artifact@v3
      - name: Upload logs
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: weave-server-logs
          path: /tmp/weave/log
          retention-days: 3