Skip to content

Commit

Permalink
Add support for data_files python-poetry#890
Browse files Browse the repository at this point in the history
  • Loading branch information
delphyne committed Feb 19, 2019
1 parent 4a82287 commit d4b4d6d
Show file tree
Hide file tree
Showing 17 changed files with 186 additions and 8 deletions.
53 changes: 53 additions & 0 deletions docs/docs/pyproject.md
Original file line number Diff line number Diff line change
Expand Up @@ -133,6 +133,59 @@ packages = [
Poetry is clever enough to detect Python subpackages.

Thus, you only have to specify the directory where your root package resides.

## data_files

A list of files to be installed using the [data_files](https://docs.python.org/2/distutils/setupscript.html#installing-additional-files)
installation mechanism. Data files are particularly useful when shipping non-code artifacts that may need to be used
by other packages at a well-known location. Example uses include distribution of Protobuf proto definition files and
Avro avsc schemas.

```toml
[tool.poetry.data_files]
my_package_name = ["a.txt", "subdir/subsubdir/b.txt"]
"my_target_dir.has.dots" = ["**/*.txt"]
"my_package_name/with/subdirectories" = ["subdir/subsubdir/b.txt"]
```

### Effect on produced wheel archives

The above TOML snippet will result in the addition of a [.data](https://www.python.org/dev/peps/pep-0427/#the-data-directory)
directory to your wheel. For example, given my package, `my_package_name`, and a version of `2.3.4`, the wheel will now
contain the following:

```
my_package_name-2.3.4.data/my_package_name/a.txt
my_package_name-2.3.4.data/my_package_name/b.txt
my_package_name-2.3.4.data/my_target_dir.has.dots/a.txt
my_package_name-2.3.4.data/my_target_dir.has.dots/b.txt
my_package_name-2.3.4.data/my_packge_name/with/subdirectories/b.txt
```

All of the entries added to the wheel are also added to the RECORD with their appropriate secure hashes.

### Effect on produced sdist archives

The above TOML snippet will result in the addition of the following `data_files` element to the generated setup.py:

```python
# [...]
data_files = \
[('my_package_name', ['a.txt', 'subdir/subsubdir/b.txt']),
('my_target_dir.has.dots', ['a.txt', 'subdir/subsubdir/b.txt']),
('my_package_name/with/subdirectories', ['subdir/subsubdir/b.txt'])]

setup_kwargs = {
# [...]
'data_files': data_files,
}
```


!!!note

The path information in the files or globs is discarded during installation. If you need your files to be placed
in a nested directory, it must be a part of the "name" of the `data_files` element.

## include and exclude

Expand Down
1 change: 1 addition & 0 deletions poetry/masonry/builders/builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ def __init__(self, poetry, env, io):
self._path.as_posix(),
packages=self._package.packages,
includes=self._package.include,
data_files=self._package.data_files,
)
self._meta = Metadata.from_package(self._package)

Expand Down
18 changes: 16 additions & 2 deletions poetry/masonry/builders/sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from poetry.utils._compat import to_str

from ..utils.helpers import normalize_file_permissions
from ..utils.data_file_include import DataFileInclude
from ..utils.package_include import PackageInclude

from .builder import Builder
Expand Down Expand Up @@ -119,6 +120,7 @@ def build_setup(self): # type: () -> bytes
modules = []
packages = []
package_data = {}
data_files = {}
for include in self._module.includes:
if isinstance(include, PackageInclude):
if include.is_package():
Expand All @@ -137,6 +139,10 @@ def build_setup(self): # type: () -> bytes

if module not in modules:
modules.append(module)
elif isinstance(include, DataFileInclude):
data_files.setdefault(include.data_file_path_prefix, []).extend(
str(element.relative_to(self._path)) for element in include.elements
)
else:
pass

Expand All @@ -153,8 +159,16 @@ def build_setup(self): # type: () -> bytes
extra.append("'package_data': package_data,")

if modules:
before.append("modules = \\\n{}".format(pformat(modules)))
extra.append("'py_modules': modules,".format())
before.append("modules = \\\n{}\n".format(pformat(modules)))
extra.append("'py_modules': modules,")

if data_files:
before.append(
"data_files = \\\n{}\n".format(
pformat([(k, v) for k, v in data_files.items()])
)
)
extra.append("'data_files': data_files,")

dependencies, extras = self.convert_dependencies(
self._package, self._package.requires
Expand Down
18 changes: 15 additions & 3 deletions poetry/masonry/builders/wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,13 +17,15 @@
from poetry.semver import parse_constraint

from ..utils.helpers import normalize_file_permissions
from ..utils.data_file_include import DataFileInclude
from ..utils.package_include import PackageInclude
from ..utils.tags import get_abbr_impl
from ..utils.tags import get_abi_tag
from ..utils.tags import get_impl_ver
from ..utils.tags import get_platform
from .builder import Builder

from poetry.utils._compat import Path

wheel_file_template = """\
Wheel-Version: 1.0
Expand Down Expand Up @@ -136,6 +138,14 @@ def _copy_module(self, wheel):

if isinstance(include, PackageInclude) and include.source:
rel_file = file.relative_to(include.base)
elif isinstance(include, DataFileInclude):
rel_file = Path(
self.wheel_meta_dir_name(
self._package.name, self._meta.version, "data"
),
include.data_file_path_prefix,
file.name,
)
else:
rel_file = file.relative_to(self._path)

Expand Down Expand Up @@ -192,7 +202,7 @@ def find_excluded_files(self): # type: () -> Set

@property
def dist_info(self): # type: () -> str
return self.dist_info_name(self._package.name, self._meta.version)
return self.wheel_meta_dir_name(self._package.name, self._meta.version)

@property
def wheel_filename(self): # type: () -> str
Expand All @@ -207,11 +217,13 @@ def supports_python2(self):
parse_constraint(">=2.0.0 <3.0.0")
)

def dist_info_name(self, distribution, version): # type: (...) -> str
def wheel_meta_dir_name(
self, distribution, version, suffix="dist-info"
): # type: (...) -> str
escaped_name = re.sub(r"[^\w\d.]+", "_", distribution, flags=re.UNICODE)
escaped_version = re.sub(r"[^\w\d.]+", "_", version, flags=re.UNICODE)

return "{}-{}.dist-info".format(escaped_name, escaped_version)
return "{}-{}.{}".format(escaped_name, escaped_version, suffix)

@property
def tag(self):
Expand Down
16 changes: 16 additions & 0 deletions poetry/masonry/utils/data_file_include.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
from .include import Include

# noinspection PyProtectedMember
from poetry.utils._compat import Path


class DataFileInclude(Include):
def __init__(
self, base, include, data_file_path_prefix
): # type: (Path, str, str) -> None
super(DataFileInclude, self).__init__(base, include)
self._data_file_path_prefix = data_file_path_prefix

@property
def data_file_path_prefix(self): # type: () -> str
return self._data_file_path_prefix
2 changes: 1 addition & 1 deletion poetry/masonry/utils/include.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ def __init__(self, base, include): # type: (Path, str) -> None
self._base = base
self._include = str(include)

self._elements = sorted(list(self._base.glob(str(self._include))))
self._elements = sorted(list(self._base.glob(self._include)))

@property
def base(self): # type: () -> Path
Expand Down
10 changes: 9 additions & 1 deletion poetry/masonry/utils/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
from poetry.utils.helpers import module_name

from .include import Include
from .data_file_include import DataFileInclude
from .package_include import PackageInclude


Expand All @@ -11,14 +12,17 @@ class ModuleOrPackageNotFound(ValueError):


class Module:
def __init__(self, name, directory=".", packages=None, includes=None):
def __init__(
self, name, directory=".", packages=None, includes=None, data_files=None
):
self._name = module_name(name)
self._in_src = False
self._is_package = False
self._path = Path(directory)
self._includes = []
packages = packages or []
includes = includes or []
data_files = data_files or {}

if not packages:
# It must exist either as a .py file or a directory, but not both
Expand Down Expand Up @@ -62,6 +66,10 @@ def __init__(self, name, directory=".", packages=None, includes=None):
PackageInclude(self._path, package["include"], package.get("from"))
)

for base, globs in data_files.items():
for glob in globs:
self._includes.append(DataFileInclude(self._path, glob, base))

for include in includes:
self._includes.append(Include(self._path, include))

Expand Down
2 changes: 1 addition & 1 deletion poetry/masonry/utils/package_include.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ def source(self): # type: () -> str
def is_package(self): # type: () -> bool
return self._is_package

def is_module(self): # type: ()
def is_module(self): # type: () -> bool
return self._is_module

def refresh(self): # type: () -> PackageInclude
Expand Down
1 change: 1 addition & 0 deletions poetry/packages/project_package.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ def __init__(self, name, version, pretty_version=None):
self.packages = []
self.include = []
self.exclude = []
self.data_files = {}

if self._python_versions == "*":
self._python_constraint = parse_constraint("~2.7 || >=3.4")
Expand Down
3 changes: 3 additions & 0 deletions poetry/poetry.py
Original file line number Diff line number Diff line change
Expand Up @@ -184,6 +184,9 @@ def create(cls, cwd): # type: (Path) -> Poetry
if "packages" in local_config:
package.packages = local_config["packages"]

if "data_files" in local_config:
package.data_files = local_config["data_files"]

# Moving lock if necessary (pyproject.lock -> poetry.lock)
lock = poetry_file.parent / "poetry.lock"
if not lock.exists():
Expand Down
Empty file.
Empty file.
10 changes: 10 additions & 0 deletions tests/masonry/builders/fixtures/data_files/pyproject.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[tool.poetry]
name = "data_files_example"
version = "0.1.0"
description = "An example TOML file describing a package with data_files"
authors = ["delphyne <delphyne@gmail.com>"]

[tool.poetry.data_files]
easy = ["a.txt", "**/c.csv"]
"a.little.more.difficult" = ["**/*.txt"]
"nested/directories" = ["subdir/subsubdir/b.txt"]
Empty file.
Empty file.
35 changes: 35 additions & 0 deletions tests/masonry/builders/test_sdist.py
Original file line number Diff line number Diff line change
Expand Up @@ -510,3 +510,38 @@ def test_proper_python_requires_if_three_digits_precision_version_specified():
parsed = p.parsestr(to_str(pkg_info))

assert parsed["Requires-Python"] == "==2.7.15"


def test_with_data_files():
poetry = Poetry.create(project("data_files"))
builder = SdistBuilder(poetry, NullEnv(), NullIO())

# Check setup.py
setup = builder.build_setup()
setup_ast = ast.parse(setup)

setup_ast.body = [n for n in setup_ast.body if isinstance(n, ast.Assign)]
ns = {}
exec(compile(setup_ast, filename="setup.py", mode="exec"), ns)

assert [
("easy", ["a.txt", "subdir/subsubdir/c.csv"]),
("a.little.more.difficult", ["a.txt", "subdir/subsubdir/b.txt"]),
("nested/directories", ["subdir/subsubdir/b.txt"]),
] == ns.get("data_files")

assert ns.get("setup_kwargs", {}).get("data_files") == ns.get("data_files")

builder.build()

sdist = fixtures_dir / "data_files" / "dist" / "data_files_example-0.1.0.tar.gz"

assert sdist.exists()

with tarfile.open(str(sdist), "r") as tar:
names = tar.getnames()
assert "data_files_example-0.1.0/data_files_example/__init__.py" in names
assert "data_files_example-0.1.0/a.txt" in names
assert "data_files_example-0.1.0/subdir/subsubdir/b.txt" in names
assert "data_files_example-0.1.0/subdir/subsubdir/c.csv" in names
assert "data_files_example-0.1.0/setup.py" in names
25 changes: 25 additions & 0 deletions tests/masonry/builders/test_wheel.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,3 +144,28 @@ def test_write_metadata_file_license_homepage_default(mocker):
# Assertion
mocked_file.write.assert_any_call("Home-page: UNKNOWN\n")
mocked_file.write.assert_any_call("License: UNKNOWN\n")


def test_with_data_files():
module_path = fixtures_dir / "data_files"
p = Poetry.create(str(module_path))
WheelBuilder.make(p, NullEnv(), NullIO())
whl = module_path / "dist" / "data_files_example-0.1.0-py2.py3-none-any.whl"
assert whl.exists()

with zipfile.ZipFile(str(whl)) as z:
names = z.namelist()
with z.open("data_files_example-0.1.0.dist-info/RECORD") as record_file:
record = record_file.readlines()

def validate(path):
assert path in names
assert (
len([r for r in record if r.decode("utf-8").startswith(path)]) == 1
)

validate("data_files_example-0.1.0.data/easy/a.txt")
validate("data_files_example-0.1.0.data/easy/c.csv")
validate("data_files_example-0.1.0.data/a.little.more.difficult/a.txt")
validate("data_files_example-0.1.0.data/a.little.more.difficult/b.txt")
validate("data_files_example-0.1.0.data/nested/directories/b.txt")

0 comments on commit d4b4d6d

Please sign in to comment.