forked from apache/libcloud
-
Notifications
You must be signed in to change notification settings - Fork 0
/
tox.ini
383 lines (347 loc) · 14.1 KB
/
tox.ini
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
[tox]
envlist = py{pypy3,3.6,3.7,3.8,3.9,3.10,pyjion},checks,lint,pylint,mypy,docs,coverage,integration-storage
skipsdist = true
requires =
wheel
[testenv]
passenv = TERM CI GITHUB_* DOCKER_*
deps =
-r{toxinidir}/requirements-tests.txt
fasteners
libvirt-python==7.9.0
basepython =
pypypy3: pypy3
pypyjion: pyjion
{py3.6,py3.6-dist,py3.6-dist-wheel}: python3.6
{py3.7,docs,checks,black,lint,pylint,mypy,micro-benchmarks,coverage,docs,py3.7-dist,py3.7-dist-wheel}: python3.7
{py3.8,py3.8-windows,integration-storage,py3.8-dist,py3.8-dist-wheel}: python3.8
{py3.9,py3.9-dist,py3.9-dist-wheel}: python3.9
{py3.10,py3.10-dist,py3.10-dist-wheel}: python3.10
setenv =
CRYPTOGRAPHY_ALLOW_OPENSSL_102=1
# NOTE: By default we run tests on CI in parallel to speed up the build
# To avoid per-test function process safety issues we run all tests in a single
# file in the same worker process.
# python setup.py test
# NOTE: tee-sys is not supported by pytest which still supports Python 3.5
# pytest -rsx -vvv -o log_cli=True --durations=10 -n auto --dist loadfile
commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
pytest -rsx -vvv --capture=tee-sys -o log_cli=True --durations=10 -n auto --dist loadfile --ignore libcloud/test/benchmarks/
whitelist_externals = cp
bash
echo
scripts/*.sh
[testenv:py3.8-windows]
deps =
-r{toxinidir}/requirements-tests.txt
fasteners
[testenv:py3.6-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"
[testenv:py3.6-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"
[testenv:py3.7-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"
[testenv:py3.7-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"
[testenv:py3.8-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"
[testenv:py3.8-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"
[testenv:py3.9-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"
[testenv:py3.9-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"
[testenv:py3.10-dist]
# Verify library installs without any dependencies when using python setup.py
# install
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_install_check.sh"
[testenv:py3.10-dist-wheel]
# Verify library installs without any dependencies when using built wheel
skipdist = True
recreate = True
# NOTE: We intentionally set empty deps to ensure it works on a clean
# environment without any dependencies
deps =
commands = bash -c "./scripts/dist_wheel_install_check.sh"
[testenv:docs]
deps = pyopenssl==21.0.0
fasteners
rstcheck
requests_mock
changedir = docs
commands = pip install sphinx==4.2.0
rstcheck --report warning ../CHANGES.rst
python ../contrib/generate_provider_feature_matrix_table.py
sphinx-apidoc -d 4 ../libcloud/ -o apidocs/
/bin/bash -c "ls apidocs/modules.rst && (grep orphan apidocs/modules.rst || sed -i '1i :orphan:\n' apidocs/modules.rst) || (exit 0)"
# TODO: Add -W back when we fix all the warnings in docstrings
sphinx-build -j auto -b html -d {envtmpdir}/doctrees . _build/html
[testenv:docs-ci]
# Note: We don't build API docs on Travis since it causes build failures because
# those API docs files are not included anywhere.
deps = pyopenssl==21.0.0
fasteners
rstcheck
requests_mock
# Pinned due to a bug in a newer version
changedir = docs
commands = pip install sphinx==4.2.0
rstcheck --report warning ../README.rst
rstcheck --report warning ../CHANGES.rst
rstcheck --report warning ../CONTRIBUTING.rst
python ../contrib/generate_provider_feature_matrix_table.py
/bin/bash -c "ls apidocs/modules.rst && (grep orphan apidocs/modules.rst || sed -i '1i :orphan:\n' apidocs/modules.rst) || (exit 0)"
# TODO: Add -W back when we fix all the warnings in docstrings
sphinx-build -j auto -b html -d {envtmpdir}/doctrees . _build/html
[testenv:provider-tables]
basepython: python3.7
deps = typing
pyopenssl==21.0.0
fasteners
rstcheck
requests
commands = python ./contrib/generate_provider_feature_matrix_table.py
[testenv:scrape-and-publish-provider-prices]
basepython: python3.7
deps = requests
jsonnet
setenv =
PYTHONPATH={toxinidir}
passenv = TERM GCE_API_KEY PRICING_DATA_BUCKET_NAME AWS_REGION AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_SECRET
commands =
python contrib/scrape-gce-prices.py --all
python contrib/scrape-gce-prices.py
python contrib/scrape-ec2-prices.py
python contrib/scrape-azure-prices.py
# We also store the SHa512 sum so users can check if something has changed
# by caching and checking the value of the shasum file
bash -c "(cd libcloud/data/ ; sha256sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha256)"
bash -c "(cd libcloud/data/ ; sha512sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha512)"
python contrib/upload-pricing-data-to-s3.py libcloud/data/pricing.json
echo ""
echo "Pricing data should now be available at"
echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json"
echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha256"
echo "https://libcloud-pricing-data.s3.amazonaws.com/pricing.json.sha512"
[testenv:scrape-provider-prices]
basepython: python3.7
deps = requests
jsonnet
passenv = TERM GCE_API_KEY
whitelist_externals = sha512sum
bash
commands =
python contrib/scrape-gce-prices.py --all
python contrib/scrape-gce-prices.py
python contrib/scrape-ec2-prices.py
python contrib/scrape-azure-prices.py
# We also store the SHa512 sum so users can check if something has changed
# by caching and checking the value of the shasum file
bash -c "(cd libcloud/data/ ; sha256sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha256)"
bash -c "(cd libcloud/data/ ; sha512sum pricing.json > {toxinidir}/libcloud/data/pricing.json.sha512)"
[testenv:scrape-ec2-prices]
basepython: python3.7
deps = requests
jsonnet
tqdm
commands = python contrib/scrape-ec2-prices.py
[testenv:scrape-ec2-sizes]
basepython: python3.7
deps = requests
ijson
tqdm
commands =
bash -c 'echo "Scrapping EC2 sizes, this may take up to 10 minutes or more since the actual JSON data we download and scrape is very large"'
bash -c 'python contrib/scrape-ec2-sizes.py'
[testenv:pylint]
deps = -r{toxinidir}/requirements-tests.txt
bottle
fasteners
paramiko==2.9.2
pysphere
setenv =
PYTHONPATH={toxinidir}
commands = pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/common/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/compute/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/container/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/backup/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/dns/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/storage/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc libcloud/utils/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc demos/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc contrib/
pylint -E --load-plugins=pylint_plugins.driver_class --rcfile=./.pylintrc pylint_plugins/
[testenv:lint]
deps = -r{toxinidir}/requirements-tests.txt
fasteners
rstcheck
commands = flake8 --config ./.flake8 libcloud/
flake8 --config ./.flake8 libcloud/test/
flake8 --config ./.flake8 demos/
flake8 --config ./.flake8 integration/
flake8 --config ./.flake8 scripts/
flake8 --config ./.flake8 docs/examples/
flake8 --config ./.flake8 contrib/
python -mjson.tool libcloud/data/pricing.json /dev/null
rstcheck --report warning README.rst
rstcheck --report warning CHANGES.rst
rstcheck --report warning CONTRIBUTING.rst
[testenv:black]
deps = -r{toxinidir}/requirements-tests.txt
black==22.1.0; python_version >= '3.6' and implementation_name == "cpython"
# NOTE: We need to use bash -c otherwise black will quote the argument and it
# won't expand to the list of files when the command runs
commands =
bash -c "black --config pyproject.toml *.py"
black --config pyproject.toml libcloud/
black --config pyproject.toml docs/examples/
black --config pyproject.toml demos/
black --config pyproject.toml contrib/
black --config pyproject.toml pylint_plugins/
[testenv:black-check]
deps = -r{toxinidir}/requirements-tests.txt
black==22.1.0; python_version >= '3.6' and implementation_name == "cpython"
# NOTE: We need to use bash -c otherwise black will quote the argument and it
# won't expand to the list of files when the command runs
commands =
bash -c "black --config pyproject.toml *.py"
black --config pyproject.toml --check libcloud/
black --config pyproject.toml --check docs/examples/
black --config pyproject.toml --check demos/
black --config pyproject.toml --check contrib/
black --config pyproject.toml --check pylint_plugins/
[testenv:checks]
commands =
bash ./scripts/check_file_names.sh
python ./scripts/check_asf_license_headers.py .
[testenv:integration-compute]
deps = -r{toxinidir}/integration/compute/requirements.txt
commands = python -m integration.compute
[testenv:integration-storage]
passenv = AZURE_CLIENT_SECRET AWS_ACCESS_KEY_ID AWS_ACCESS_KEY_SECRET GITHUB_*
setenv =
AZURE_CLIENT_ID=16cd65a3-dfa2-4272-bcdb-842cbbedb1b7
AZURE_TENANT_ID=982317c6-fb7e-4e92-abcd-196557e41c5b
AZURE_SUBSCRIPTION_ID=d6d608a6-e0c8-42ae-a548-2f41793709d2
# Actual secret token is defined as part of Github repo secret
deps =
pytest==6.2.5
-r{toxinidir}/integration/storage/requirements.txt
commands = pytest -rsx -vvv --capture=tee-sys -o log_cli=True --durations=10 integration/storage
[testenv:coverage]
deps =
-r{toxinidir}/requirements-tests.txt
paramiko==2.9.2
pyopenssl==21.0.0
python-dateutil
libvirt-python==7.9.0
fasteners
setenv =
CRYPTOGRAPHY_ALLOW_OPENSSL_102=1
commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
coverage run --source=libcloud setup.py test
[testenv:coverage-ci]
passenv = TERM TOXENV CI GITHUB_*
deps =
-r{toxinidir}/requirements-tests.txt
paramiko==2.9.2
pyopenssl==21.0.0
libvirt-python==7.9.0
fasteners
setenv =
CRYPTOGRAPHY_ALLOW_OPENSSL_102=1
commands = cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
coverage run --source=libcloud setup.py test
codecov
[testenv:mypy]
deps =
typing
# Mypy requires typed-ast, which is broken on PyPy 3.7 (could work in PyPy 3.8).
mypy==0.910; implementation_name == "cpython"
types-simplejson
types-certifi
types-requests
types-urllib3
commands =
mypy --no-incremental libcloud/common/
mypy --no-incremental libcloud/compute/
mypy --no-incremental libcloud/storage/
mypy --no-incremental libcloud/dns/
mypy --no-incremental libcloud/container/
mypy --no-incremental example_compute.py
mypy --no-incremental example_storage.py
mypy --no-incremental example_dns.py
mypy --no-incremental example_container.py
[testenv:micro-benchmarks]
deps =
-r{toxinidir}/requirements-tests.txt
fasteners
commands =
cp libcloud/test/secrets.py-dist libcloud/test/secrets.py
pytest -s -v --benchmark-only --benchmark-name=short --benchmark-columns=min,max,mean,stddev,median,ops,rounds --benchmark-histogram=benchmark_histograms/benchmark --benchmark-group-by=group,param:sort_objects libcloud/test/benchmarks/test_list_objects_filtering_performance.py
[testenv:import-timings]
basepython: python3.7
setenv =
PYTHONPATH={toxinidir}
deps =
requests
profimp==0.1.0
commands =
bash -c "./scripts/time_imports.sh"