-
Notifications
You must be signed in to change notification settings - Fork 590
262 lines (245 loc) · 9.51 KB
/
pythonpackage.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
name: Python package
on:
push:
branches:
- master
pull_request:
branches:
- master
- docs
schedule:
- cron: "0 2 * * 1-5"
# to make bash use ~/.bashrc, see https://github.com/marketplace/actions/setup-miniconda#example-5-custom-installer
defaults:
run:
shell: bash -l {0}
concurrency:
group: ${{ github.workflow }}-${{ github.ref || github.run_id }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
main:
env:
CACHE_NUMBER_COMPILE: 6
MAMBA_ROOT_PREFIX: /dev/shm/mambaroot
runs-on: ${{ matrix.os }}
strategy:
fail-fast: false
max-parallel: 12
matrix:
os: [ubuntu-latest, windows-latest, macOS-12]
python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
steps:
- name: disk space
if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }}
run: |
df -h
- uses: maxim-lobanov/setup-xcode@v1
# alternative would be to upgrade tapi to 1100.0.11, we can possibly remove this in the future
if: matrix.os == 'macOS-12'
with:
xcode-version: "13.2.1"
- uses: actions/checkout@v2
with:
submodules: true
- uses: actions/checkout@v2
if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }}
with:
repository: vaexio/vaex-enterprise
token: ${{ secrets.PAT_PULL_ENTERPRISE }}
path: vaex-enterprise
- name: install micromamba
uses: mamba-org/setup-micromamba@v1
with:
micromamba-version: 1.1.0-1
environment-name: vaex-dev
environment-file: ci/conda-env.yml
create-args: >-
python=${{ matrix.python-version }}
- name: Extra non-windows installs
if: matrix.os != 'windows-latest'
run: |
echo "not needed for now"
# - name: Extra windows installs
# if: matrix.os == 'windows-latest'
# run: |
# # not sure why, but the conda env is strange (old dask) if we put this constraint in the conda solver
# # also might solve scipy issues
# micromamba update scipy ipython dask
# micromamba install scipy=1.7.1
# - name: free disk space
# if: matrix.os == 'ubuntu-latest'
# run: |
# sudo swapoff -a
# sudo rm -f /swapfile
# sudo apt clean
# docker rmi $(docker image ls -aq)
# df -h
- name: Install OpenMP runtime (Mac-only)
if: matrix.os == 'macOS-12'
run: |
brew install libomp
- name: Cache compiled binaries
# this fails for this combination, leading to binaries filled with 0's
if: matrix.os != 'macOS-12'
id: cache-compiled-binaries
uses: actions/cache@v2
with:
path: |
packages/vaex-core/build/lib*/vaex/vaexfast*
packages/vaex-core/build/lib*/vaex/super*
key: ${{ runner.OS }}-${{ matrix.python-version }}-${{ hashFiles('packages/vaex-core/src/*') }}-${{ env.CACHE_NUMBER_COMPILE }}
- name: Fix cache timestamp
run: |
touch packages/vaex-core/build/lib*/vaex/vaexfast* || echo "ok to fail"
touch packages/vaex-core/build/lib*/vaex/super* || echo "ok to fail"
ls -alh packages/vaex-core/build/lib*/vaex/ || echo "ok to fail"
- name: Install vaex
run: |
pip install myst_parser
pip install -e . -v
# ./ci/03-install-vaex.sh micromamba
# - name: Install vaex-enterprise
# if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }}
# run: |
# cd vaex-enterprise
# micromamba install -c conda-forge distributed
# pip install ray
# pip install -e .
- name: Copy dll
if: (matrix.os == 'windows-latest')
uses: ./ci/actions/windll
- name: Build Python packages
run: |
mkdir dist
(cd packages/vaex-core && python setup.py bdist_wheel); cp packages/vaex-core/dist/* dist
(cd packages/vaex-graphql && python setup.py bdist_wheel); cp packages/vaex-graphql/dist/* dist
(cd packages/vaex-jupyter && python setup.py bdist_wheel); cp packages/vaex-jupyter/dist/* dist
(cd packages/vaex-ml && python setup.py bdist_wheel); cp packages/vaex-ml/dist/* dist
(cd packages/vaex-viz && python setup.py bdist_wheel); cp packages/vaex-viz/dist/* dist
(cd packages/vaex-astro && python setup.py bdist_wheel); cp packages/vaex-astro/dist/* dist
(cd packages/vaex-hdf5 && python setup.py bdist_wheel); cp packages/vaex-hdf5/dist/* dist
(cd packages/vaex-server && python setup.py bdist_wheel); cp packages/vaex-server/dist/* dist
(cd packages/vaex-meta && python setup.py bdist_wheel); cp packages/vaex-server/dist/* dist
- name: Upload builds
uses: actions/upload-artifact@v2
with:
name: dist-${{ github.run_number }}-${{ matrix.python-version }}-${{ matrix.os }}
path: ./dist
# - name: Lint with flake8
# run: |
# echo "soon"
# # pip install flake8
# # stop the build if there are Python syntax errors or undefined names
# # flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
# # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
# # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
- name: Install pytest-asyncio correct version
if: matrix.python-version != '3.10'
run: |
# 3.10 doesn't seem to work
pip install "pytest-asyncio<0.14"
- name: Test with pytest
run: |
./ci/04-run-test-suite.sh
- name: Test with pytest (with cache on)
if: matrix.python-version == '3.9' && matrix.os == 'ubuntu-latest'
run: |
VAEX_CACHE_RESULTS=1 ./ci/04-run-test-suite.sh
- name: Check ml spec
# no catboost for py39 and py37, xgboost fails on 3.6
if: matrix.python-version != '3.9' && matrix.os != 'windows-latest'
run: |
python -m vaex.ml.spec packages/vaex-ml/vaex/ml/spec_new.json
diff packages/vaex-ml/vaex/ml/spec_new.json packages/vaex-ml/vaex/ml/spec.json
# - name: Test with pytest (vaex-enterprise)
# if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && (matrix.os == 'ubuntu-latest') }}
# run: |
# py.test vaex-enterprise/tests --timeout=1000
- name: Test notebooks
if: matrix.os != 'windows-latest'
run: |
./ci/05-run-notebooks.sh
- name: Authenticate Google Cloud Platform
if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest')) }}
uses: google-github-actions/setup-gcloud@v0
with:
project_id: ${{ secrets.GCP_PROJECT_ID_VAEX }}
service_account_key: ${{ secrets.GCP_SA_KEY_VAEX }}
export_default_credentials: true
- name: Test vaex-contrib
# do not run in a PR from someone else, skip windows
if: ${{ (github.event.pull_request.head.repo.full_name == 'vaexio/vaex') && !((matrix.os == 'windows-latest')) }}
env:
PROJECT_ID: ${{ secrets.GCP_PROJECT_ID_VAEX }}
run: |
pip install -e 'packages/vaex-contrib[gcp]'
./ci/06-run-contrib-tests.sh
install:
runs-on: ${{ matrix.os }}
needs: [main]
strategy:
fail-fast: false
matrix:
os: [ubuntu-latest, windows-latest, macOS-12]
python-version: [3.8, 3.9]
steps:
- uses: actions/checkout@v2
- uses: actions/download-artifact@v2
with:
name: dist-${{ github.run_number }}-${{ matrix.python-version }}-${{ matrix.os }}
path: ./dist
- name: Install micromamba
uses: mamba-org/setup-micromamba@v1
with:
micromamba-version: 1.5.6-0
environment-name: vaex-test
environment-file: ci/conda-base-minimal.yml
create-args: >-
python=${{ matrix.python-version }}
pcre
rich
# it shouldn't be needed to use pcre, maybe we should make the wheels similar to how we make distributions
- name: Cache pip
uses: actions/cache@v2
env:
# Increase this value to reset cache
CACHE_NUMBER: 0
with:
path: |
~/.cache/pip
key:
pip-${{ runner.os }}-${{ matrix.python-version }}-${{ env.CACHE_NUMBER }}
- name: Install core packages
run: |
pip install -vv dist/vaex*core* dist/vaex*hdf5*
- name: Test core import
run: |
python -c "import vaex; df = vaex.example()"
- name: Install all packages
run: |
pip install -vv dist/*
- name: Test import
run: |
python -c "import vaex; df = vaex.example()"
- name: Test comand line
run: |
vaex convert ~/.vaex/data/helmi-dezeeuw-2000-FeH-v2-10percent.hdf5 test.parquet
pip install rich
VAEX_PROGRESS_TYPE=rich vaex convert ~/.vaex/data/helmi-dezeeuw-2000-FeH-v2-10percent.hdf5 test.parquet
vaex settings yaml
vaex settings md
- name: Test server
if: matrix.os != 'windows-latest'
run: |
vaex server --add-example --port 9999&
- name: Wait for Vaex server
if: matrix.os != 'windows-latest'
uses: ifaxity/wait-on-action@v1
with:
resource: http-get://localhost:9999/hello
timeout: 5000
- name: Use the Vaex server
if: matrix.os != 'windows-latest'
run: |
python -c "import vaex; df = vaex.open('ws://localhost:9999/example'); df.x.sum()"
curl -i http://localhost:9999/histogram/example/x