test #14364
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
# We use self-hosted runners for performance: we can rely on docker caching | |
# on the runner machines for 0s startup times for our tests. | |
# There is an issue with self-hosted runners where if you use a mix of | |
# host and container jobs, the host jobs will fail to start | |
# (https://github.com/actions/checkout/issues/273). | |
# So we create a separate "builder" tagged runner to do the build job on | |
# the host, and then all the tests run on "runner" tagged runners. | |
name: test | |
on: | |
push: | |
jobs: | |
build-container: | |
name: Build test container | |
timeout-minutes: 30 | |
runs-on: [self-hosted, builder] | |
# runs-on: ubuntu-latest | |
env: | |
REGISTRY: us-east4-docker.pkg.dev/weave-support-367421/weave-images | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
fetch-depth: 2 | |
- name: Login to Docker Hub | |
uses: docker/login-action@v2 | |
with: | |
registry: us-east4-docker.pkg.dev | |
username: _json_key | |
password: ${{ secrets.gcp_sa_key }} | |
# this script is hardcoded to build for linux/amd64 | |
- name: Prune docker cache | |
run: docker system prune -f | |
- name: Build unit test image | |
run: python3 weave/docker/docker_build.py build_deps weave-test-python-client builder . weave/docker/Dockerfile.python-client.test | |
- name: Build integration test image | |
run: python3 weave/docker/docker_build.py build weave-integration-test . weave/legacy/Dockerfile.test | |
lint: | |
name: Python lint | |
runs-on: ubuntu-latest | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Set up Python | |
uses: actions/setup-python@v4 | |
with: | |
python-version: 3.9 | |
- name: Install dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install tox | |
- name: Run tox | |
run: tox -e lint | |
weavejs-lint-compile: | |
name: WeaveJS Lint and Compile | |
runs-on: ubuntu-latest | |
# runs-on: [self-hosted, gke-runner] | |
steps: | |
- uses: actions/checkout@v2 | |
with: | |
fetch-depth: 0 | |
token: ${{ secrets.GITHUB_TOKEN }} | |
- uses: actions/setup-node@v1 | |
with: | |
node-version: '18.x' | |
- run: | | |
set -e | |
cd weave-js | |
yarn install --frozen-lockfile | |
yarn generate | |
yarn eslint --max-warnings=0 | |
yarn tslint | |
yarn prettier | |
yarn run tsc | |
trace-tests: | |
name: Trace tox tests | |
runs-on: ubuntu-latest | |
strategy: | |
matrix: | |
python-version-major: ['3'] | |
python-version-minor: [ | |
'9', | |
'10', | |
'11', | |
# '12', # TODO: We have actual failing tests in 3.12, but commenting for simplicity for now. | |
# | |
] | |
tox-shard: | |
[ | |
'trace', | |
'trace_server', | |
'anthropic', | |
'cerebras', | |
'cohere', | |
'dspy', | |
'groq', | |
'langchain', | |
'litellm', | |
'llamaindex', | |
'mistral0', | |
'mistral1', | |
'openai', | |
] | |
fail-fast: false | |
services: | |
wandbservice: | |
image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master | |
credentials: | |
username: _json_key | |
password: ${{ secrets.gcp_wb_sa_key }} | |
env: | |
CI: 1 | |
WANDB_ENABLE_TEST_CONTAINER: true | |
ports: | |
- '8080:8080' | |
- '8083:8083' | |
- '9015:9015' | |
options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s | |
weave_clickhouse: | |
image: clickhouse/clickhouse-server | |
ports: | |
- '8123:8123' | |
options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s | |
steps: | |
- name: Checkout | |
uses: actions/checkout@v3 | |
- name: Set up Python ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }} | |
uses: actions/setup-python@v4 | |
with: | |
python-version: ${{ matrix.python-version-major }}.${{ matrix.python-version-minor }} | |
- name: Install dependencies | |
run: | | |
python -m pip install --upgrade pip | |
pip install tox | |
- name: Run tox (Clickhouse Only) | |
env: | |
WEAVE_SENTRY_ENV: ci | |
CI: 1 | |
WB_SERVER_HOST: http://wandbservice | |
WF_CLICKHOUSE_HOST: weave_clickhouse | |
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 | |
run: | | |
tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} -- \ | |
-m "weave_client and not skip_clickhouse_client" \ | |
--weave-server=clickhouse | |
- name: Run tox | |
env: | |
WEAVE_SENTRY_ENV: ci | |
CI: 1 | |
WB_SERVER_HOST: http://wandbservice | |
WF_CLICKHOUSE_HOST: weave_clickhouse | |
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 | |
run: | | |
tox -e ${{ matrix.tox-shard }}-py${{ matrix.python-version-major }}${{ matrix.python-version-minor }} | |
test: | |
name: Python unit tests | |
timeout-minutes: 15 # do not raise! running longer than this indicates an issue with the tests. fix there. | |
needs: | |
- build-container | |
# runs-on: [self-hosted, gke-runner] | |
runs-on: ubuntu-latest | |
strategy: | |
fail-fast: false | |
matrix: | |
job_num: [0, 1] | |
# runs-on: ubuntu-latest | |
container: us-east4-docker.pkg.dev/weave-support-367421/weave-images/weave-test-python-client:${{ github.sha }} | |
services: | |
wandbservice: | |
image: us-central1-docker.pkg.dev/wandb-production/images/local-testcontainer:master | |
credentials: | |
username: _json_key | |
password: ${{ secrets.gcp_wb_sa_key }} | |
env: | |
CI: 1 | |
WANDB_ENABLE_TEST_CONTAINER: true | |
ports: | |
- '8080:8080' | |
- '8083:8083' | |
- '9015:9015' | |
options: --health-cmd "curl --fail http://localhost:8080/healthz || exit 1" --health-interval=5s --health-timeout=3s | |
weave_clickhouse: | |
image: clickhouse/clickhouse-server | |
ports: | |
- '8123:8123' | |
options: --health-cmd "wget -nv -O- 'http://localhost:8123/ping' || exit 1" --health-interval=5s --health-timeout=3s | |
steps: | |
# - uses: datadog/agent-github-action@v1.3 | |
# with: | |
# api_key: ${{ secrets.DD_API_KEY }} | |
- uses: actions/checkout@v2 | |
- name: Verify wandb server is running | |
run: curl -s http://wandbservice:8080/healthz | |
- name: Run Legacy (Query Service) Python Unit Tests | |
env: | |
DD_SERVICE: weave-python | |
DD_ENV: ci | |
WEAVE_SENTRY_ENV: ci | |
CI: 1 | |
WB_SERVER_HOST: http://wandbservice | |
WF_CLICKHOUSE_HOST: weave_clickhouse | |
WEAVE_SERVER_DISABLE_ECOSYSTEM: 1 | |
run: | | |
source /root/venv/bin/activate && \ | |
pip install 'dspy-ai==2.4.13' && \ | |
pip install pydantic -U && \ | |
cd weave && \ | |
pytest \ | |
--job-num=${{ matrix.job_num }} \ | |
--timeout=90 \ | |
--ddtrace \ | |
--durations=5 \ | |
./legacy |