make-wheel-builder

#!/usr/bin/env python
""" Script to create a wheel builder for a project
"""
from __future__ import print_function

import os
from os.path import isdir

import argparse # Python 2.6, really?

from subprocess import check_call

TERRYFY_URL = 'git://github.com/MacPython/terryfy.git'
LATEST_PY27 = "2.7.10"
LATEST_PY33 = "3.3.5"
LATEST_PY34 = "3.4.3"
LATEST_PY35 = "3.5.1"

VERSION_BIT_NO_NUMPY = \
"""  - VERSION={}
  - VERSION={}
  - VERSION={}
  - VERSION={}""".format(
      LATEST_PY27, LATEST_PY33, LATEST_PY34, LATEST_PY35)

VERSION_BIT_NUMPY = \
"""  - VERSION={} NUMPY_VERSION=1.6.2 # tune to taste
  - VERSION={} NUMPY_VERSION=1.7.1
  - VERSION={} NUMPY_VERSION=1.7.1
  - VERSION={} NUMPY_VERSION=1.7.1""".format(
      LATEST_PY27, LATEST_PY33, LATEST_PY34, LATEST_PY35)

TRAVIS_TEMPLATE = \
"""language:
- objective-c
env:
  global:
  - BUILD_COMMIT='latest-tag'  # comment out to build version in submodule
  - REPO_DIR={project_dir}
  matrix:
{version_bit}
install:
  - source terryfy/travis_tools.sh
  - get_python_environment macpython $VERSION venv
  # Update to latest wheel package for Python 3.5
  - pip install -U wheel
  - pip install {pip_depends}
  - if [ -n "$BUILD_COMMIT" ]; then
        checkout_commit $REPO_DIR $BUILD_COMMIT;
    fi
  - cd {project_dir}
  - python setup.py bdist_wheel
  - delocate-listdeps dist/*.whl # lists library dependencies
  - delocate-wheel dist/*.whl # copies library dependencies into wheel
  - delocate-addplat --rm-orig -x 10_9 -x 10_10 dist/*.whl
  - pip install dist/*.whl
  - cd ..
script:
  # install dependencies for tests e.g - pip install nose
  - mkdir tmp_for_test
  - cd tmp_for_test
  # Run the tests
  - cd ..
before_deploy:
  - cd $REPO_DIR/dist
deploy:
  provider: cloudfiles
  username: travis-worker
  api_key:
    secure: ALZrkCwjz3nY/9GfTNPgVQHUmUpCc5SnZTaqIUlJrhVyOBu6aktXaRmQi+sTXzFn7a/PQsYDnoZ8nibCZ472oUVDGUOSbtjIEkPodU+aisLN2dSLNueQsUU7Dmt0SaeyJVGovzfyUiavIYLuAfOtCfUKrwAsi7XTVNQpdyWCQDs=
  region: ORD
  container: wheels
  skip_cleanup: true
"""

GITIGNORE_TEMPLATE = \
"""*.pyc
tmp_for_test/
working/
build/
"""

README_TEMPLATE = \
"""##################################
Building and uploading {project_dir} wheels
##################################

*******
For OSX
*******

We automate OSX wheel building using this custom github repository that builds
on the travis-ci OSX machines.

The travis-ci interface for the builds is :
https://travis-ci.org/MacPython/{project_dir}-wheels

The driving github repository is :
https://github.com/MacPython/{project_dir}-wheels

How it works
============

The wheel-building repository:

* does a fresh build of the required C / C++ libraries;
* builds a {project_dir} wheel, linking against these fresh builds;
* processes the wheel using [delocate](https://pypi.python.org/pypi/delocate).
  `delocate` copies the required dynamic libraries into the wheel and relinks
  the extension modules against the copied libraries;
* uploads the built wheel to http://wheels.scipy.org (a Rackspace container
  kindly donated by Rackspace to scikit-learn).

The resulting wheel is therefore self-contained and does not need any external
dynamic libraries apart from those provided as standard by OSX.

The ``.travis.yml`` file in this repository has a line containing the API key
for the Rackspace container encrypted with an RSA key that is unique to the
repository - see http://docs.travis-ci.com/user/encryption-keys.  This
encrypted key gives the travis build permission to upload to the Rackspace
directory pointed to by http://wheels.scipy.org.

Triggering a build
==================

You will need write permission to the github repository to trigger new builds
on the travis-ci interface.  Contact us on the mailing list if you need this.

You can trigger a build by:

* making a commit to the `{project_dir}-wheels` repository (e.g. with `git
  commit --allow-empty`); or
* clicking on the circular arrow icon towards the top right of the travis-ci
  page, to rerun the previous build.

In general, it is better to trigger a build with a commit, because this makes
a new set of build products and logs, keeping the old ones for reference.
Keeping the old build logs helps us keep track of previous problems and
successful builds.

Which {project_dir} commit does the repository build?
============================================

By default, the `{project_dir}-wheels` repository is usually set up to build
the latest git tag.  By "latest" we mean the tag on the branch most recently
branched from master - see http://stackoverflow.com/a/24557377/1939576. To
check whether you are building the latest tag have a look around line 5 of
`.travis.yml` in the `{project_dir}-wheels` repository.  You should see something
like::

    - BUILD_COMMIT='latest-tag'

If this is commented out, then the repository is set up to build the current
commit in the `{project_dir}` submodule of the repository.  If it is set to
another value then it will be specifying a commit to build.

You can therefore build any arbitrary commit by specifying the commit hash or
branch name or tag name in this line of the `.travis.yml` file.

Uploading the built wheels to pypi
==================================

Be careful, http://wheels.scipy.org points to a container on a distributed
content delivery network.  It can take up to 15 minutes for the new wheel file
to get updated into the container at http://wheels.scipy.org.

When the wheels are updated, you can of course just download them to your
machine manually, and then upload them manually to pypi, or by using
twine_.  You can also use a script for doing this, housed at :
https://github.com/MacPython/terryfy/blob/master/wheel-uploader

You'll need twine and `beautiful soup 4 <bs4>`_.

You will typically have a directory on your machine where you store wheels,
called a `wheelhouse`.   The typical call for `wheel-uploader` would then
be something like::

    wheel-uploader -v -w ~/wheelhouse {project_dir} 1.0.3

where:

* `-v` means give verbose messages;
* `-w ~/wheelhouse` means download the wheels from https://wheels.scipy.org to
  the directory `~/wheelhouse`;
* `{project_dir}` is the root name of the wheel(s) to download / upload;
* `1.0.3` is the version to download / upload.

So, in this case, `wheel-uploader` will download all wheels starting with
`{project_dir}-1.0.3-` from http://wheels.scipy.org to `~/wheelhouse`, then upload
them to pypi.

Of course, you will need permissions to upload to pypi, for this to work.

{numpy_bit}

.. _twine: https://pypi.python.org/pypi/twine
.. _bs4: https://pypi.python.org/pypi/beautifulsoup4
"""

NUMPY_BIT_TEMPLATE = \
"""Maintaining the build repo
==========================

* Check minimum numpy versions to build against in ``.travis.yml`` file.  You
  need to build against the earliest numpy that {project_dir} is compatible with; see
  `forward, backward numpy compatibility
  <http://stackoverflow.com/questions/17709641/valueerror-numpy-dtype-has-the-wrong-size-try-recompiling/18369312#18369312>`_
"""


def repo2project_dir(url):
    """ Get project name from git repo
    """
    if url.endswith('.git'):
        url = url[:-4]
    return url.split('/')[-1]


def main():
    parser = argparse.ArgumentParser(
        description='Create project to build wheels on travis')
    parser.add_argument('github_url',
                        help = 'Github repo URL')
    parser.add_argument('--needs-numpy', action='store_true')
    args = parser.parse_args()
    project_dir = repo2project_dir(args.github_url)
    wheels_project = project_dir + '-wheels'
    os.mkdir(wheels_project)
    os.chdir(wheels_project)
    check_call(['git', 'init'])
    check_call(['git', 'submodule', 'add', args.github_url])
    project_dir = repo2project_dir(args.github_url)
    if not isdir(project_dir):
        raise RuntimeError('Expecting {} directory'.format(project_dir))
    check_call(['git', 'submodule', 'add', TERRYFY_URL])
    version_bit = (VERSION_BIT_NUMPY if args.needs_numpy
                   else VERSION_BIT_NO_NUMPY)
    pip_depends = 'delocate'
    if args.needs_numpy:
        pip_depends += ' numpy==$NUMPY_VERSION'
    with open('.travis.yml', 'wt') as fobj:
        fobj.write(TRAVIS_TEMPLATE.format(project_dir=project_dir,
                                          version_bit=version_bit,
                                          pip_depends=pip_depends))
    with open('.gitignore', 'wt') as fobj:
        fobj.write(GITIGNORE_TEMPLATE)
    if args.needs_numpy:
        numpy_bit = NUMPY_BIT_TEMPLATE.format(project_dir=project_dir)
    else:
        numpy_bit = ''
    with open('README.rst', 'wt') as fobj:
        fobj.write(README_TEMPLATE.format(project_dir=project_dir,
                                          numpy_bit=numpy_bit))
    check_call(['git', 'add', '*', '.gitignore', '.travis.yml'])
    check_call(['git', 'commit', '-m', 'Initial commit'])
    print('Created ' + wheels_project)
    print(
"""Now push to github, setup travis building and encrypt the rackspace key.

I use "hub" [1] and the travis command line client [2], but you can do most of
this with the web GUIs.

    cd {wheels_project}
    hub create <your-username>/{wheels_project} # create project on github
    travis sync # sync list of github repos known to travis
    travis enable # enable travis testing of this repo
    travis encrypt <the rackspace API key> # see README about this key

The last line generates something like:

  secure: "fC8a017zdgsbkxZRcJrIrEa35LyzJJEyxTHkbMG/gVJKLhzMLKdw7F6FnsUyxu/RHA5FHE5D2S7X471dNU/xfCYJjWkzvHlrZrwQ5vwJ0KGIRE46KiO0URBGoGFQWI1WPhOeirIJ0ZLaahTYLbXYnQRhM2mOjCmIY74jPSTrrvg="

Replace the current .travis.yml 'secure:' line with the result and commit.

You might also want to fill in the test command for the project in the
.travis.yml file (see comment placeholder)

Finally, push to github to initiate the first travis build.

If your package has external library dependencies, there are some utilities to
help you build them; see
https://github.com/matthew-brett/h5py-wheels/blob/master/.travis.yml for an
example of their use.

[1] https://github.com/github/hub
[2] https://github.com/travis-ci/travis.rb
""".format(wheels_project=wheels_project))


if __name__ == '__main__':
    main()