forked from spacetx/starfish
-
Notifications
You must be signed in to change notification settings - Fork 0
/
Makefile
335 lines (277 loc) · 11 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
SHELL := /bin/bash
EDITOR?=vi
MPLBACKEND?=Agg
export MPLBACKEND
MODULES=starfish examples/data_formatting/format*
DOCKER_IMAGE?=spacetx/starfish
DOCKER_BUILD?=1
VERSION=$(shell sh -c "git describe --exact --dirty 2> /dev/null")
define print_help
@printf " %-28s $(2)\n" $(1)
endef
define create_venv
if [[ "$(TRAVIS)" = "" ]]; then \
python -m venv $(1); \
else \
virtualenv -p $$(which python) $(1); \
fi
endef
all: fast
### UNIT #####################################################
#
fast: lint mypy fast-test docs-html
lint: lint-non-init lint-init
lint-non-init:
flake8 --ignore 'E252, E301, E302, E305, E401, W503, E731, F811' --exclude='*__init__.py' $(MODULES)
lint-init:
flake8 --ignore 'E252, E301, E302, E305, E401, F401, W503, E731, F811' --filename='*__init__.py' $(MODULES)
# note that napari tests shouldn't be run in parallel because Qt seems to intermittently fail when multiple QApplications are spawned on threads.
test:
pytest -v -n 8 --cov starfish -m 'not napari'
pytest -v --cov starfish -m 'napari'
fast-test:
pytest -v -n 8 --cov starfish -m 'not (slow or napari)'
slow-test:
pytest -v -n 8 --cov starfish -m 'slow and (not napari)'
# note that this shouldn't be run in parallel because Qt seems to intermittently fail when multiple QApplications are spawned on threads.
napari-test:
pytest -v --cov starfish -m 'napari'
mypy:
mypy --ignore-missing-imports $(MODULES)
help-unit:
$(call print_help, all, alias for fast)
.PHONY: all fast lint lint-non-init lint-init test mypy help-unit
#
##############################################################
### DOCS #####################################################
#
docs-%:
make -C docs $*
help-docs:
$(call print_help, docs-TASK, alias for 'make TASK' in the docs subdirectory)
.PHONY: help-docs
#
##############################################################
### REQUIREMENTS #############################################
#
check-requirements:
if [[ $$(git status --porcelain REQUIREMENTS* requirements/REQUIREMENTS* starfish/REQUIREMENTS*) ]]; then \
echo "Modifications found in REQUIREMENTS files"; exit 2; \
fi
refresh-all-requirements: export TARGETS=starfish/REQUIREMENTS-STRICT.txt requirements/REQUIREMENTS-CI.txt requirements/REQUIREMENTS-NAPARI-CI.txt
refresh-all-requirements: export SOURCES=REQUIREMENTS.txt requirements/REQUIREMENTS-CI.txt.in requirements/REQUIREMENTS-NAPARI-CI.txt.in
refresh-all-requirements:
@for target in $(TARGETS); do \
echo -n '' >| $$target; \
done
@if [ $$(uname -s) == "Darwin" ]; then sleep 1; fi # this is require because Darwin HFS+ only has second-resolution for timestamps.
@touch $(SOURCES)
$(MAKE) $(TARGETS)
starfish/REQUIREMENTS-STRICT.txt : REQUIREMENTS.txt
[ ! -e .$<-env ] || exit 1
$(call create_venv, .$<-env)
.$<-env/bin/pip install -r $@
.$<-env/bin/pip install -r $<
echo "# You should not edit this file directly. Instead, you should edit one of the following files ($^) and run make $@" >| $@
.$<-env/bin/pip freeze >> $@
rm -rf .$<-env
requirements/REQUIREMENTS-%.txt : requirements/REQUIREMENTS-%.txt.in REQUIREMENTS.txt
[ ! -e .$<-env ] || exit 1
$(call create_venv, .$<-env)
.$<-env/bin/pip install -r $@
for src in $^; do \
.$<-env/bin/pip install -r $$src; \
done
echo "# You should not edit this file directly. Instead, you should edit one of the following files ($<) and run make $@" >| $@
.$<-env/bin/pip freeze >> $@
rm -rf .$<-env
help-requirements:
$(call print_help, refresh_all_requirements, regenerate requirements files)
$(call print_help, check_requirements, fail if requirements files have been modified)
.PHONY: refresh_all_requirements
#
##############################################################
### INTEGRATION ##############################################
#
include notebooks/subdir.mk
include examples/pipelines/subdir.mk
test-examples: export TESTING=1
test-examples: run-examples
slow: fast run-notebooks test-examples docker
docker:
docker build -f docker/Dockerfile -t $(DOCKER_IMAGE) .
docker run -ti --rm $(DOCKER_IMAGE) starfish --help
help-integration:
$(call print_help, slow, alias for 'fast run-notebooks docker')
$(call print_help, run-notebooks, run all files matching 'notebooks/py/*.py')
$(call print_help, docker, build docker and run a simple container)
.PHONY: slow docker
#
##############################################################
### INSTALL ##################################################
#
install-dev:
pip install --force-reinstall --upgrade -r requirements/REQUIREMENTS-CI.txt
pip install -e .
pip freeze
install-src:
pip install --force-reinstall --upgrade -e .
pip freeze
install-released-notebooks-support:
pip install -r requirements/REQUIREMENTS-CI.txt
pip install starfish
pip freeze
help-install:
$(call print_help, install-dev, pip install from the current directory with pinned requirements and tooling for CI)
$(call print_help, install-src, pip install from the current directory)
$(call print_help, install-released-notebooks, pip install tooling to run notebooks against the released version of starfish)
.PHONY: install-dev install-src install-released-notebooks-support help-install
#
###############################################################
### Deployment ################################################
#
# General release steps:
# --------------------------------------------------------------
#
# (0) Pull master and check out the latest version of master for
# releasing.
#
# (1) `make release-changelog` to print a suggested update to
# CHANGELOG.md. Replace "XXX" with your intended tag and
# perform other edits. Important is that each section is
# separated by a line beginning with '##'.
#
# Remove changelog entries that do not pertain to end users.
#
# (2) Commit all files and remove any untracked files.
# `git status` should show nothing.
#
# (3) Push master to origin, i.e., `git push origin master`.
#
# (4) Run `make release-tag TAG=x.y.z` to tag your release.
#
# (5) Run `make release-prep` which:
# - checks the tag
# - creates a virtualenv
# - builds and installs the sdist
#
# (6) Run `make release-verify` which:
# - runs tests
# - builds docker
#
# (7) Run `make release-upload` and execute the commands.
#
# If anything goes wrong, rollback the various steps:
# - delete on docker hub
# - delete local docker image
# - delete tag locally
# - make clean
## Sections: 1 - 3
# public: print a changelog to stdout
release-changelog:
@if test -n "$(VERSION)"; then \
echo VERSION is set to $(VERSION) \
echo Create your changelog before tagging.\
exit 102; \
fi;
@echo "##" "[XXX]" - $(shell sh -c "date +'%Y-%m-%d'")
@git log $(shell sh -c "git describe --tags --abbrev=0")..HEAD --pretty=format:"- %s"
@printf "\n\n"
@cat CHANGELOG.md; echo "[XXX]: https://github.com/spacetx/starfish/releases/tag/XXX"
# public: generate a tag from the current commit & changelog
release-tag:
@if test -z "$(TAG)"; then \
echo TAG is not set. Use: \
echo make TAG=x.y.z release-tag; \
exit 104; \
fi && \
printf "Tag $(TAG)\n\n" > release-msg && \
sed -n -e '/^##/,/^##/{ /^##/d; /^##/d; p; }' CHANGELOG.md >> release-msg && \
$(EDITOR) release-msg && \
git tag -a -F release-msg "$(TAG)" && \
rm release-msg
## Sections: 4
# private: assert a clean tag on the current commit
release-check:
@if test -z "$(VERSION)"; then \
echo VERSION is not set\!; \
echo Is the current commit tagged?; \
echo If not, create a tag for the current version.;\
exit 100; \
elif [[ "$(VERSION)" == *"dirty"* ]] ; then \
echo VERSION is dirty.; \
echo Please commit all files and re-tag.; \
exit 101; \
else \
echo "==============================="; \
echo "Releasing version: $(VERSION)"; \
echo "==============================="; \
fi;
# private: assert release-env and release-msg don't exist
release-ready:
@if compgen -G release-* > /dev/null; then \
echo "Previous release found."; \
echo "Run 'make clean'"; \
exit 103; \
fi;
# private: create a virtualenv for testing the release
release-env: release-env/bin/activate release-env/bin/make_shell
# private: call virtualenv and pip install
release-env/bin/activate:
$(call create_venv, release-env)
release-env/bin/pip install -r requirements/REQUIREMENTS-CI.txt
touch release-env/bin/activate
# private: create make_shell for activating the virtualenv below
release-env/bin/make_shell:
echo '#!/bin/bash' > $@
echo 'source release-env/bin/activate' >> $@
echo 'bash "$$@"' >> $@
chmod a+x $@
# public: generate the release build
release-prep: release-check release-ready release-env
release-env/bin/python setup.py clean
release-env/bin/python setup.py sdist
release-env/bin/pip install dist/starfish-$(VERSION).tar.gz
## Sections: 5 - 6
# public: run tests on the current release build
release-verify: export SHELL=release-env/bin/make_shell
release-verify: release-check slow release-docker
# public: tag the docker images
release-docker: release-check
docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(VERSION)
docker tag $(DOCKER_IMAGE) $(DOCKER_IMAGE):$(VERSION)-$(DOCKER_BUILD)
# public: print commands for uploading artifacts
release-upload: release-check
@printf '\n# Please execute the following steps\n'
@echo git push origin $(VERSION)
@echo docker push $(DOCKER_IMAGE)
@echo docker push $(DOCKER_IMAGE):$(VERSION)
@echo docker push $(DOCKER_IMAGE):$(VERSION)-$(DOCKER_BUILD)
@echo twine upload dist/starfish-$(VERSION).tar.gz
clean:
rm -rf release-env
rm -rf release-msg
rm -rf starfish.egg-info
rm -rf dist
rm -rf build
rm -rf .eggs
rm -f .cover*
help-deployment:
$(call print_help, release-changelog, Print changelog for updating CHANGELOG.md)
$(call print_help, release-tag, Tag current commit with update changelog)
$(call print_help, release-prep, Builds and installs the current tagged version)
$(call print_help, release-verify, Runs tests on the tagged version)
$(call print_help, release-upload, Prints commands for uploading release artifacts)
$(call print_help, clean, Deletes build-related directories)
.PHONY: clean release-prep release-verify release-upload release-check
#
###############################################################
help: help-main help-parts
help-main:
@echo Main starfish make targets:
@echo =======================================================================================
$(call print_help, help, print this text)
help-parts: help-unit help-docs help-requirements help-integration help-install help-deployment
@echo =======================================================================================
@echo Default: all
.PHONY: help help-unit help-requirements help-integration help-install help-release