From 03d9d769c09d264d1146e029d3915992624fd246 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Fri, 19 Oct 2018 22:45:10 -0400 Subject: [PATCH 1/8] Handle non-extension wheels with binaries --- auditwheel/wheel_abi.py | 27 ++++++++++++++++++----- auditwheel/wheeltools.py | 27 ++++++++++++++--------- tests/test_manylinux.py | 15 ++++++------- tests/testpackage/setup.py | 2 +- tests/testpackage/testpackage/__init__.py | 2 +- 5 files changed, 47 insertions(+), 26 deletions(-) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index 241ebada..aa753c60 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -1,3 +1,4 @@ +import itertools import json import logging import functools @@ -25,6 +26,7 @@ @functools.lru_cache() def get_wheel_elfdata(wheel_fn: str): full_elftree = {} + nonpy_elftree = {} full_external_refs = {} versioned_symbols = defaultdict(lambda: set()) # type: Dict[str, Set[str]] uses_ucs2_symbols = False @@ -43,19 +45,32 @@ def get_wheel_elfdata(wheel_fn: str): log.info('processing: %s', fn) elftree = lddtree(fn) - full_elftree[fn] = elftree - if is_py_ext: - uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) + for key, value in elf_find_versioned_symbols(elf): + log.debug('key %s, value %s', key, value) versioned_symbols[key].add(value) if is_py_ext: + full_elftree[fn] = elftree + uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) if py_ver == 2: uses_ucs2_symbols |= any( True for _ in elf_find_ucs2_symbols(elf)) - full_external_refs[fn] = lddtree_external_references(elftree, - ctx.path) - + full_external_refs[fn] = lddtree_external_references(elftree, + ctx.path) + else: + nonpy_elftree[fn] = elftree + + needed_libs = [elf['needed'] for elf + in itertools.chain(full_elftree.values(), + nonpy_elftree.values())] + uniq_needs = set(sum(needed_libs, [])) # concatenate needed libs together, get uniq + + for fn in nonpy_elftree.keys(): + if basename(fn) not in uniq_needs: + full_elftree[fn] = nonpy_elftree[fn] + full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn], + ctx.path) log.debug(json.dumps(full_elftree, indent=4)) diff --git a/auditwheel/wheeltools.py b/auditwheel/wheeltools.py index f51149e3..ec7eb5e7 100644 --- a/auditwheel/wheeltools.py +++ b/auditwheel/wheeltools.py @@ -185,11 +185,10 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): platform tags to remove to the wheel filename and WHEEL tags, e.g. ``('linux_x86_64',)`` when ``('manylinux_x86_64')`` is added """ + definitely_not_purelib = False + info_fname = pjoin(_dist_info_dir(wheel_ctx.path), 'WHEEL') info = read_pkg_info(info_fname) - if info['Root-Is-Purelib'] == 'true': - print('No need to add platforms to pure wheel - Skipping {}'.format(wheel_ctx.in_wheel)) - return # Check what tags we have if wheel_ctx.out_wheel is not None: @@ -203,11 +202,15 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): fparts = parsed_fname.groupdict() original_fname_tags = fparts['plat'].split('.') print('Previous filename tags:', ', '.join(original_fname_tags)) - fname_tags = [tag for tag in original_fname_tags - if tag not in remove_platforms] - for platform in platforms: - if platform not in fname_tags: - fname_tags.append(platform) + fname_tags = {tag for tag in original_fname_tags + if tag not in remove_platforms} + fname_tags |= set(platforms) + + # Can't be 'any' and another platform + if 'any' in fname_tags and len(fname_tags) > 1: + fname_tags.remove('any') + definitely_not_purelib = True + if fname_tags != original_fname_tags: print('New filename tags:', ', '.join(fname_tags)) else: @@ -232,11 +235,15 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): for tup in product(pyc_apis, remove_platforms)] updated_tags = [tag for tag in in_info_tags if tag not in unwanted_tags] updated_tags += new_tags - needs_write = updated_tags != in_info_tags - if needs_write: + if updated_tags != in_info_tags: del info['Tag'] for tag in updated_tags: info.add_header('Tag', tag) + + if definitely_not_purelib: + info['Root-Is-Purelib'] = 'False' + print('Changed wheel type to Platlib') + print('New WHEEL info tags:', ', '.join(info.get_all('Tag'))) write_pkg_info(info_fname, info) else: diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 498e6970..b94fd22e 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -16,7 +16,6 @@ '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin') WHEEL_CACHE_FOLDER = op.expanduser('~/.cache/auditwheel_tests') ORIGINAL_NUMPY_WHEEL = 'numpy-1.11.0-cp35-cp35m-linux_x86_64.whl' -ORIGINAL_TESTPACKAGE_WHEEL = 'testpackage-0.0.1-cp35-cp35m-linux_x86_64.whl' def find_src_folder(): @@ -85,7 +84,7 @@ def docker_container(): volumes={'/io': io_folder, '/auditwheel_src': src_folder}, env_variables={'PATH': PATH}) # Install the development version of auditwheel from source: - docker_exec(manylinux_id, 'pip install -U pip setuptools wheel') + docker_exec(manylinux_id, 'pip install -U pip setuptools') docker_exec(manylinux_id, 'pip install -U /auditwheel_src') # Launch a docker container with a more recent userland to check that @@ -140,7 +139,7 @@ def test_build_repair_numpy(docker_container): repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl'] repaired_wheel = repaired_wheels[0] - output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) + output = docker_exec(manylinux_id, 'auditwheel -vv show /io/' + repaired_wheel) assert ( 'numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl is consistent' ' with the following platform tag: "manylinux1_x86_64"' @@ -170,10 +169,10 @@ def test_build_wheel_with_binary_executable(docker_container): manylinux_id, python_id, io_folder = docker_container docker_exec(manylinux_id, 'yum install -y gsl-devel') - docker_exec(manylinux_id, 'cd /auditwheel_src/test/testpackage && python setup.py bdist_wheel -d /io') + docker_exec(manylinux_id, ['bash', '-c', 'cd /auditwheel_src/tests/testpackage && python setup.py bdist_wheel -d /io']) filenames = os.listdir(io_folder) - assert filenames == [ORIGINAL_TESTPACKAGE_WHEEL] + assert filenames == ['testpackage-0.0.1-py3-none-any.whl'] orig_wheel = filenames[0] assert 'manylinux' not in orig_wheel @@ -182,11 +181,11 @@ def test_build_wheel_with_binary_executable(docker_container): filenames = os.listdir(io_folder) assert len(filenames) == 2 repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] - assert repaired_wheels == ['testpackage-0.0.1-cp35-cp35m-manylinux1_x86_64.whl'] + assert repaired_wheels == ['testpackage-0.0.1-py3-none-manylinux1_x86_64.whl'] repaired_wheel = repaired_wheels[0] output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( - 'testpackage-0.0.1-cp35-cp35m-manylinux1_x86_64.whl is consistent' + 'testpackage-0.0.1-py3-none-manylinux1_x86_64.whl is consistent' ' with the following platform tag: "manylinux1_x86_64"' ) in output.replace('\n', ' ') @@ -194,5 +193,5 @@ def test_build_wheel_with_binary_executable(docker_container): # on a modern linux image. docker_exec(python_id, 'pip install /io/' + repaired_wheel) output = docker_exec( - python_id, 'python -c "from testpackage import runit; print(runit(1.5))"').strip() + python_id, ['python', '-c', 'from testpackage import runit; print(runit(1.5))']).strip() assert output.strip() == '2.25' diff --git a/tests/testpackage/setup.py b/tests/testpackage/setup.py index d16d860f..1034e150 100644 --- a/tests/testpackage/setup.py +++ b/tests/testpackage/setup.py @@ -1,7 +1,7 @@ from setuptools import setup import subprocess -cmd = 'gcc testpackage/testprogram.c -lgsl -o testpackage/testprogram' +cmd = 'gcc testpackage/testprogram.c -lgsl -lgslcblas -o testpackage/testprogram' subprocess.check_call(cmd.split()) setup( diff --git a/tests/testpackage/testpackage/__init__.py b/tests/testpackage/testpackage/__init__.py index 2e933192..06a5ad48 100644 --- a/tests/testpackage/testpackage/__init__.py +++ b/tests/testpackage/testpackage/__init__.py @@ -5,4 +5,4 @@ def runit(x): filename = pkg_resources.resource_filename(__name__, 'testprogram') output = subprocess.check_output([filename, str(x)]) - return float(x) + return float(output) From 8fad909dc6391d4676fb2bba09d2ca425788cc8a Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Fri, 19 Oct 2018 23:05:18 -0400 Subject: [PATCH 2/8] Remove the extra verbosity from tests --- tests/test_manylinux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index b94fd22e..71c74af9 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -139,7 +139,7 @@ def test_build_repair_numpy(docker_container): repaired_wheels = [fn for fn in filenames if 'manylinux1' in fn] assert repaired_wheels == ['numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl'] repaired_wheel = repaired_wheels[0] - output = docker_exec(manylinux_id, 'auditwheel -vv show /io/' + repaired_wheel) + output = docker_exec(manylinux_id, 'auditwheel show /io/' + repaired_wheel) assert ( 'numpy-1.11.0-cp35-cp35m-manylinux1_x86_64.whl is consistent' ' with the following platform tag: "manylinux1_x86_64"' From aad726c652ce353966c080982ada53e5ad94af45 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Fri, 19 Oct 2018 23:29:35 -0400 Subject: [PATCH 3/8] Remove 'any' platform tag in non-pure non-extension wheels --- auditwheel/wheeltools.py | 1 + 1 file changed, 1 insertion(+) diff --git a/auditwheel/wheeltools.py b/auditwheel/wheeltools.py index ec7eb5e7..1038ae90 100644 --- a/auditwheel/wheeltools.py +++ b/auditwheel/wheeltools.py @@ -209,6 +209,7 @@ def add_platforms(wheel_ctx, platforms, remove_platforms=()): # Can't be 'any' and another platform if 'any' in fname_tags and len(fname_tags) > 1: fname_tags.remove('any') + remove_platforms.append('any') definitely_not_purelib = True if fname_tags != original_fname_tags: From d6e29c657b6d0e59e9dc1cc83fc97fbc2eedd610 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 27 Oct 2018 14:22:33 -0400 Subject: [PATCH 4/8] Ignore output from the non-extension binary wheel build --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index f928b2f7..3ab26439 100644 --- a/.gitignore +++ b/.gitignore @@ -62,3 +62,4 @@ target/ # Generated by test script *.zip wheelhoust-* +tests/testpackage/testpackage/testprogram From 88f9e5d1cd74bf7f335a9f4999e25f80a80511d4 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 27 Oct 2018 14:51:05 -0400 Subject: [PATCH 5/8] Add 'control' test with a pure wheel, fix #47 --- auditwheel/repair.py | 5 +++++ tests/test_manylinux.py | 32 ++++++++++++++++++++++++++++++++ 2 files changed, 37 insertions(+) diff --git a/auditwheel/repair.py b/auditwheel/repair.py index e342b0f9..2b7c8e99 100644 --- a/auditwheel/repair.py +++ b/auditwheel/repair.py @@ -41,6 +41,11 @@ def repair_wheel(wheel_path: str, abi: str, lib_sdir: str, out_dir: str, update_tags: bool) -> Optional[str]: external_refs_by_fn = get_wheel_elfdata(wheel_path)[1] + + # Do not repair a pure wheel, i.e. has no external refs + if not external_refs_by_fn: + return + soname_map = {} # type: Dict[str, str] if not isabs(out_dir): out_dir = abspath(out_dir) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index 71c74af9..febd4be6 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -16,6 +16,7 @@ '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin') WHEEL_CACHE_FOLDER = op.expanduser('~/.cache/auditwheel_tests') ORIGINAL_NUMPY_WHEEL = 'numpy-1.11.0-cp35-cp35m-linux_x86_64.whl' +ORIGINAL_SIX_WHEEL = 'six-1.11.0-py2.py3-none-any.whl' def find_src_folder(): @@ -195,3 +196,34 @@ def test_build_wheel_with_binary_executable(docker_container): output = docker_exec( python_id, ['python', '-c', 'from testpackage import runit; print(runit(1.5))']).strip() assert output.strip() == '2.25' + + +def test_build_repair_pure_wheel(docker_container): + manylinux_id, python_id, io_folder = docker_container + + if op.exists(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL)): + # If six has already been built and put in cache, let's reuse this. + shutil.copy2(op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL), + op.join(io_folder, ORIGINAL_SIX_WHEEL)) + else: + docker_exec(manylinux_id, + 'pip wheel -w /io --no-binary=:all: six==1.11.0') + shutil.copy2(op.join(io_folder, ORIGINAL_SIX_WHEEL), + op.join(WHEEL_CACHE_FOLDER, ORIGINAL_SIX_WHEEL)) + + filenames = os.listdir(io_folder) + assert filenames == [ORIGINAL_SIX_WHEEL] + orig_wheel = filenames[0] + assert 'manylinux' not in orig_wheel + + # Repair the wheel using the manylinux1 container + docker_exec(manylinux_id, 'auditwheel repair -w /io /io/' + orig_wheel) + filenames = os.listdir(io_folder) + assert len(filenames) == 1 # no new wheels + assert filenames == [ORIGINAL_SIX_WHEEL] + + output = docker_exec(manylinux_id, 'auditwheel show /io/' + filenames[0]) + assert ( + ORIGINAL_SIX_WHEEL + + ' is consistent with the following platform tag: "manylinux1_x86_64"' + ) in output.replace('\n', ' ') From 274bc45639b7bcbbfb7de4cae59f399301aa46a1 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 27 Oct 2018 15:09:36 -0400 Subject: [PATCH 6/8] Refactor logic for getting needed_libs --- auditwheel/wheel_abi.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index aa753c60..b6673c65 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -61,13 +61,15 @@ def get_wheel_elfdata(wheel_fn: str): else: nonpy_elftree[fn] = elftree - needed_libs = [elf['needed'] for elf - in itertools.chain(full_elftree.values(), - nonpy_elftree.values())] - uniq_needs = set(sum(needed_libs, [])) # concatenate needed libs together, get uniq + needed_libs = { + lib + for elf in itertools.chain(full_elftree.values(), + nonpy_elftree.values()) + for lib in elf['needed'] + } for fn in nonpy_elftree.keys(): - if basename(fn) not in uniq_needs: + if basename(fn) not in needed_libs: full_elftree[fn] = nonpy_elftree[fn] full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn], ctx.path) From 12cf71fd75c456fcbb18d1d5688173c5e85b83ae Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 27 Oct 2018 15:30:51 -0400 Subject: [PATCH 7/8] Improve pure wheel test to assert the wheel is indeed pure --- tests/test_manylinux.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/test_manylinux.py b/tests/test_manylinux.py index febd4be6..d4ca1a47 100644 --- a/tests/test_manylinux.py +++ b/tests/test_manylinux.py @@ -223,7 +223,11 @@ def test_build_repair_pure_wheel(docker_container): assert filenames == [ORIGINAL_SIX_WHEEL] output = docker_exec(manylinux_id, 'auditwheel show /io/' + filenames[0]) - assert ( - ORIGINAL_SIX_WHEEL + - ' is consistent with the following platform tag: "manylinux1_x86_64"' - ) in output.replace('\n', ' ') + assert ''.join([ + ORIGINAL_SIX_WHEEL, + ' is consistent with the following platform tag: ', + '"manylinux1_x86_64". ', + 'The wheel references no external versioned symbols from system- ', + 'provided shared libraries. ', + 'The wheel requires no external shared libraries! :)', + ]) in output.replace('\n', ' ') From 687c3bc0ded3fec50a4a6740ed91f94c6b541aa9 Mon Sep 17 00:00:00 2001 From: Elana Hashman Date: Sat, 27 Oct 2018 15:44:13 -0400 Subject: [PATCH 8/8] Add explanatory comments for the logic updates in wheel_abi.py --- auditwheel/wheel_abi.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/auditwheel/wheel_abi.py b/auditwheel/wheel_abi.py index b6673c65..29d70e67 100644 --- a/auditwheel/wheel_abi.py +++ b/auditwheel/wheel_abi.py @@ -50,6 +50,8 @@ def get_wheel_elfdata(wheel_fn: str): log.debug('key %s, value %s', key, value) versioned_symbols[key].add(value) + # If the ELF is a Python extention, we definitely need to include + # its external dependencies. if is_py_ext: full_elftree[fn] = elftree uses_PyFPE_jbuf |= elf_references_PyFPE_jbuf(elf) @@ -59,8 +61,13 @@ def get_wheel_elfdata(wheel_fn: str): full_external_refs[fn] = lddtree_external_references(elftree, ctx.path) else: + # If the ELF is not a Python extension, it might be included in + # the wheel already because auditwheel repair vendored it, so + # we will check whether we should include its internal + # references later. nonpy_elftree[fn] = elftree + # Get a list of all external libraries needed by ELFs in the wheel. needed_libs = { lib for elf in itertools.chain(full_elftree.values(), @@ -69,6 +76,9 @@ def get_wheel_elfdata(wheel_fn: str): } for fn in nonpy_elftree.keys(): + # If a non-pyextension ELF file is not needed by something else + # inside the wheel, then it was not checked by the logic above and + # we should walk its elftree. if basename(fn) not in needed_libs: full_elftree[fn] = nonpy_elftree[fn] full_external_refs[fn] = lddtree_external_references(nonpy_elftree[fn],