diff --git a/.gitignore b/.gitignore index f928b2f7..3ab26439 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ target/ # Generated by test script *.zip wheelhoust-* +tests/testpackage/testpackage/testprogram diff --git a/auditwheel/repair.py b/auditwheel/repair.py index e342b0f9..2b7c8e99 100644 --- a/auditwheel/repair.py +++ b/auditwheel/repair.py @@ -41,6 +41,11 @@ def repair_wheel(wheel_path: str, abi: str, lib_sdir: str, out_dir: str, update_tags: bool) -> Optional[str]: external_refs_by_fn = get_wheel_elfdata(wheel_path)[1] + + # Do not repair a pure wheel, i.e. has no external refs + if not external_refs_by_fn: + return + soname_map = {} # type: Dict[str, str] if not isabs(out_dir): out_dir = abspath(out_dir) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index 241ebada..29d70e67 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -1,3 +1,4 @@ +import itertools import json import logging import functools @@ -25,6 +26,7 @@ @functools.lru_cache() def get_wheel_elfdata(wheel_fn: str): full_elftree = {} + nonpy_elftree = {} full_external_refs = {} versioned_symbols = defaultdict(lambda: set()) # type: Dict[str, Set[str]] uses_ucs2_symbols = False @@ -43,19 +45,44 @@ def get_wheel_elfdata(wheel_fn: str): log.info('processing: %s', fn) elftree = lddtree(fn) - full_elftree[fn] = elftree - if is_py_ext: - uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) + for key, value in elf_find_versioned_symbols(elf): + log.debug('key %s, value %s', key, value) versioned_symbols[key].add(value) + # If the ELF is a Python extention, we definitely need to include + # its external dependencies. if is_py_ext: + full_elftree[fn] = elftree + uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) if py_ver == 2: uses_ucs2_symbols |= any( True for _ in elf_find_ucs2_symbols(elf)) - full_external_refs[fn] = lddtree_external_references(elftree, - ctx.path) - + full_external_refs[fn] = lddtree_external_references(elftree, + ctx.path) + else: + # If the ELF is not a Python extension, it might be included in + # the wheel already because auditwheel repair vendored it, so + # we will check whether we should include its internal + # references later. + nonpy_elftree[fn] = elftree + + # Get a list of all external libraries needed by ELFs in the wheel. + needed_libs = { + lib + for elf in itertools.chain(full_elftree.values(), + nonpy_elftree.values()) + for lib in elf['needed'] + } + + for fn in nonpy_elftree.keys(): + # If a non-pyextension ELF file is not needed by something else + # inside the wheel, then it was not checked by the logic above and + # we should walk its elftree. + if basename(fn) not in needed_libs: + full_elftree[fn] = nonpy_elftree[fn] + full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn], + ctx.path) log.debug(json.dumps(full_elftree, indent=4)) diff --git a/auditwheel/wheeltools.py b/auditwheel/wheeltools.py index f51149e3..1038ae90 100644 --- a/auditwheel/wheeltools.py +++ b/auditwheel/wheeltools.py @@ -185,11 +185,10 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): platform tags to remove to the wheel filename and WHEEL tags, e.g. ``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added """ + definitely_not_purelib = False + info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL') info = read_pkg_info(info_fname) - if info['Root-Is-Purelib'] == 'true': - print('No need to add platforms to pure wheel - Skipping {}'.format(wheel_ctx.in_wheel)) - return # Check what tags we have if wheel_ctx.out_wheel is not None: @@ -203,11 +202,16 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): fparts = parsed_fname.groupdict() original_fname_tags = fparts['plat'].split('.') print('Previous filename tags:', ', '.join(original_fname_tags)) - fname_tags = [tag for tag in original_fname_tags - if tag not in remove_platforms] - for platform in platforms: - if platform not in fname_tags: - fname_tags.append(platform) + fname_tags = {tag for tag in original_fname_tags + if tag not in remove_platforms} + fname_tags |= set(platforms) + + # Can't be 'any' and another platform + if 'any' in fname_tags and len(fname_tags) > 1: + fname_tags.remove('any') + remove_platforms.append('any') + definitely_not_purelib = True + if fname_tags != original_fname_tags: print('New filename tags:', ', '.join(fname_tags)) else: @@ -232,11 +236,15 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): for tup in product(pyc_apis, remove_platforms)] updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags] updated_tags += new_tags - needs_write = updated_tags != in_info_tags - if needs_write: + if updated_tags != in_info_tags: del info['Tag'] for tag in updated_tags: info.add_header('Tag', tag) + + if definitely_not_purelib: + info['Root-Is-Purelib'] = 'False' + print('Changed wheel type to Platlib') + print('New WHEEL info tags:', ', '.join(info.get_all('Tag'))) write_pkg_info(info_fname, info) else: diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 498e6970..d4ca1a47 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -16,7 +16,7 @@ '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin') WHEEL_CACHE_FOLDER = op.expanduser('~/.cache/auditwheel_tests') ORIGINAL_NUMPY_WHEEL = 'numpy-1.11.0-cp35-cp35m-linux_x86_64.whl' -ORIGINAL_TESTPACKAGE_WHEEL = 'testpackage-0.0.1-cp35-cp35m-linux_x86_64.whl' +ORIGINAL_SIX_WHEEL = 'six-1.11.0-py2.py3-none-any.whl' def find_src_folder(): @@ -85,7 +85,7 @@ def docker_container(): volumes={'/io': io_folder, '/auditwheel_src': src_folder}, env_variables={'PATH': PATH}) # Install the development version of auditwheel from source: - docker_exec(manylinux_id, 'pip install -U pip setuptools wheel') + docker_exec(manylinux_id, 'pip install -U pip setuptools') docker_exec(manylinux_id, 'pip install -U /auditwheel_src') # Launch a docker container with a more recent userland to check that @@ -170,10 +170,10 @@ def test_build_wheel_with_binary_executable(docker_container): manylinux_id, python_id, io_folder = docker_container docker_exec(manylinux_id, 'yum install -y gsl-devel') - docker_exec(manylinux_id, 'cd /auditwheel_src/test/testpackage && python setup.py bdist_wheel -d /io') + docker_exec(manylinux_id, ['bash', '-c', 'cd /auditwheel_src/tests/testpackage && python setup.py bdist_wheel -d /io']) filenames = os.listdir(io_folder) - assert filenames == [ORIGINAL_TESTPACKAGE_WHEEL] + assert filenames == ['testpackage-0.0.1-py3-none-any.whl'] orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel @@ -182,11 +182,11 @@ def test_build_wheel_with_binary_executable(docker_container): filenames = os.listdir(io_folder) assert len(filenames) == 2 repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] - assert repaired_wheels == ['testpackage-0.0.1-cp35-cp35m-manylinux1_x86_64.whl'] + assert repaired_wheels == ['testpackage-0.0.1-py3-none-manylinux1_x86_64.whl'] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'testpackage-0.0.1-cp35-cp35m-manylinux1_x86_64.whl is consistent' + 'testpackage-0.0.1-py3-none-manylinux1_x86_64.whl is consistent' ' with the following platform tag: "manylinux1_x86_64"' ) in output.replace('\n', ' ') @@ -194,5 +194,40 @@ def test_build_wheel_with_binary_executable(docker_container): # on a modern linux image. docker_exec(python_id, 'pip install /io/' + repaired_wheel) output = docker_exec( - python_id, 'python -c "from testpackage import runit; print(runit(1.5))"').strip() + python_id, ['python', '-c', 'from testpackage import runit; print(runit(1.5))']).strip() assert output.strip() == '2.25' + + +def test_build_repair_pure_wheel(docker_container): + manylinux_id, python_id, io_folder = docker_container + + if op.exists(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL)): + # If six has already been built and put in cache, let's reuse this. + shutil.copy2(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL), + op.join(io_folder, ORIGINAL_SIX_WHEEL)) + else: + docker_exec(manylinux_id, + 'pip wheel -w /io --no-binary=:all: six==1.11.0') + shutil.copy2(op.join(io_folder, ORIGINAL_SIX_WHEEL), + op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL)) + + filenames = os.listdir(io_folder) + assert filenames == [ORIGINAL_SIX_WHEEL] + orig_wheel = filenames[0] + assert 'manylinux' not in orig_wheel + + # Repair the wheel using the manylinux1 container + docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) + filenames = os.listdir(io_folder) + assert len(filenames) == 1 # no new wheels + assert filenames == [ORIGINAL_SIX_WHEEL] + + output = docker_exec(manylinux_id, 'auditwheel show /io/' + filenames[0]) + assert ''.join([ + ORIGINAL_SIX_WHEEL, + ' is consistent with the following platform tag: ', + '"manylinux1_x86_64". ', + 'The wheel references no external versioned symbols from system- ', + 'provided shared libraries. ', + 'The wheel requires no external shared libraries! :)', + ]) in output.replace('\n', ' ') diff --git a/tests/testpackage/setup.py b/tests/testpackage/setup.py index d16d860f..1034e150 100644 --- a/tests/testpackage/setup.py +++ b/tests/testpackage/setup.py @@ -1,7 +1,7 @@ from setuptools import setup import subprocess -cmd = 'gcc testpackage/testprogram.c -lgsl -o testpackage/testprogram' +cmd = 'gcc testpackage/testprogram.c -lgsl -lgslcblas -o testpackage/testprogram' subprocess.check_call(cmd.split()) setup( diff --git a/tests/testpackage/testpackage/__init__.py b/tests/testpackage/testpackage/__init__.py index 2e933192..06a5ad48 100644 --- a/tests/testpackage/testpackage/__init__.py +++ b/tests/testpackage/testpackage/__init__.py @@ -5,4 +5,4 @@ def runit(x): filename = pkg_resources.resource_filename(__name__, 'testprogram') output = subprocess.check_output([filename, str(x)]) - return float(x) + return float(output)