Skip to content
test

merge #14360

Sign in to view logs

merge

merge #14360

Workflow file for this run

.github/workflows/test.yaml at f5499e6
# We use self-hosted runners for performance: we can rely on docker caching
# on the runner machines for 0s startup times for our tests.
# There is an issue with self-hosted runners where if you use a mix of
# host and container jobs, the host jobs will fail to start
# (https://github.com/actions/checkout/issues/273).
# So we create a separate "builder" tagged runner to do the build job on
# the host, and then all the tests run on "runner" tagged runners.
name: test
on:
push:
jobs:
build-container:
name: Build test container
timeout-minutes: 30
runs-on: [self-hosted, builder]
# runs-on: ubuntu-latest
env:
REGISTRY: us-east4-docker.pkg.dev/weave-support-367421/weave-images
steps:
- uses: actions/checkout@v3
with:
fetch-depth: 2
- name: Login to Docker Hub
uses: docker/login-action@v2
with:
registry: us-east4-docker.pkg.dev
username: _json_key
password: ${{ secrets.gcp_sa_key }}
# this script is hardcoded to build for linux/amd64
- name: Prune docker cache
run: docker system prune -f
- name: Build unit test image
run: python3 weave/docker/docker_build.py build_deps weave-test-python-client builder . weave/docker/Dockerfile.python-client.test
- name: Build integration test image
run: python3 weave/docker/docker_build.py build weave-integration-test . weave/legacy/Dockerfile.test
lint:
name: Python lint
timeout-minutes: 7
needs:
- build-container
# runs-on: [self-hosted, gke-runner]
runs-on: ubuntu-latest
container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-client:${{ github.sha }}
steps:
- uses: actions/checkout@v2
- run: source /root/venv/bin/activate && pre-commit run --hook-stage=pre-push --all-files
weavejs-lint-compile:
name: WeaveJS Lint and Compile
runs-on: ubuntu-latest
# runs-on: [self-hosted, gke-runner]
steps:
- uses: actions/checkout@v2
with:
fetch-depth: 0
token: ${{ secrets.GITHUB_TOKEN }}
- uses: actions/setup-node@v1
with:
node-version: "18.x"
- run: |
set -e
cd weave-js
yarn install --frozen-lockfile
yarn generate
yarn eslint --max-warnings=0
yarn tslint
yarn prettier
yarn run tsc
test:
name: Python unit tests
timeout-minutes: 15 # do not raise! running longer than this indicates an issue with the tests. fix there.
needs:
- build-container
# runs-on: [self-hosted, gke-runner]
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
job_num: [0, 1]
# runs-on: ubuntu-latest
container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-client:${{ github.sha }}
services:
wandbservice:
image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master
credentials:
username: _json_key
password: ${{ secrets.gcp_wb_sa_key }}
env:
CI: 1
WANDB_ENABLE_TEST_CONTAINER: true
ports:
- "8080:8080"
- "8083:8083"
- "9015:9015"
options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s
weave_clickhouse:
image: clickhouse/clickhouse-server
ports:
- "8123:8123"
options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s
steps:
# - uses: datadog/agent-github-action@v1.3
# with:
# api_key: ${{ secrets.DD_API_KEY }}
- uses: actions/checkout@v2
- name: Verify wandb server is running
run: curl -s http://wandbservice:8080/healthz
- name: Run Python Unit Tests (Clickhouse Client Only)
env:
DD_SERVICE: weave-python
DD_ENV: ci
WEAVE_SENTRY_ENV: ci
CI: 1
WB_SERVER_HOST: http://wandbservice
WF_CLICKHOUSE_HOST: weave_clickhouse
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
# This runner specifically runs the tests that use the `client` fixture (those that support clickhouse client tests)
# However, we skip tests marked with `skip_clickhouse_client`. These should be considered TODOs and an exception
run: |
source /root/venv/bin/activate && \
cd weave && \
pytest -m "weave_client and not skip_clickhouse_client" \
--weave-server=clickhouse \
--job-num=${{ matrix.job_num }} \
--timeout=90 \
--ddtrace \
--durations=5 \
./integrations ./legacy ./trace_server ./trace ./tests
- name: Run Python Unit Tests
env:
DD_SERVICE: weave-python
DD_ENV: ci
WEAVE_SENTRY_ENV: ci
CI: 1
WB_SERVER_HOST: http://wandbservice
WF_CLICKHOUSE_HOST: weave_clickhouse
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1
run: |
source /root/venv/bin/activate && \
pip install 'dspy-ai==2.4.13' && \
pip install pydantic -U && \
cd weave && \
pytest \
--job-num=${{ matrix.job_num }} \
--timeout=90 \
--ddtrace \
--durations=5 \
./integrations ./legacy ./trace_server ./trace ./tests
# nbmake:
# name: Run notebooks with nbmake
# runs-on: self-hosted
# container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test:latest
# steps:
# - uses: actions/checkout@v2
# - name: Run notebooks
# run: source /root/venv/bin/activate && export PYTHONPATH=$(pwd) && pytest -n=4 --nbmake --overwrite examples
# - name: Upload executed notebooks
# uses: actions/upload-artifact@v3
# if: always()
# with:
# name: notebooks
# path: examples
cypress-run:
name: Notebook and UI tests
timeout-minutes: 25 # 15 minute timeout routinely trips on rerun
needs:
- build-container
# - lint
# - test
if: always()
# runs-on: [self-hosted, gke-runner]
runs-on: ubuntu-latest
container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-integration-test:${{ github.sha }}
strategy:
fail-fast: false
matrix:
containers: [1, 2, 3, 4, 5, 6]
steps:
- uses: actions/checkout@v3
- name: Setup W&B API key
run: echo "WANDB_API_KEY=${{ secrets.WANDB_API_KEY }}" >> $GITHUB_ENV
- name: Setup Replicate API token
run: echo "REPLICATE_API_TOKEN=${{ secrets.REPLICATE_API_TOKEN }}" > $GITHUB_ENV
- name: Copy node_modules from container to checkout
run: cp -R /root/integration_test/node_modules ./integration_test/
- name: Activate venv
run: source /root/venv/bin/activate && echo "PATH=$PATH" >> $GITHUB_ENV
- name: Make Log Dir
run: mkdir -p /tmp/weave/log
- name: Copy over built assets
run: cp -r /root/weave-js-build/* ./weave/frontend
- name: Start weave server
# github actions does something funky with the std file descriptors, they end up
# being closed. tqdm (for example) raises an exception when the descriptor it
# wants to write to is closed.
run: nohup ./scripts/weave_server_test.sh < /dev/null &> /tmp/weave/log/stdout.log &
shell: bash
- name: Cypress run
# Use the following to run just a single test
# run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium --spec "cypress/e2e/notebooks/Ops that return images.cy.ts"
# run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser replay-chromium
run: export PYTHONPATH=$(pwd) && cd integration_test && npx cypress run --browser chrome
# uses: cypress-io/github-action@v4
# with:
# working-directory: ./integration_testG
# record: true
# parallel: true
env:
REPLAY_API_KEY: ${{ secrets.REPLAY_API_KEY }}
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
SPLIT: ${{ strategy.job-total }}
SPLIT_INDEX: ${{ strategy.job-index }}
# - uses: actions/upload-artifact@v3
- name: Upload logs
uses: actions/upload-artifact@v3
if: failure()
with:
name: weave-server-logs
path: /tmp/weave/log
retention-days: 3