Skip to content
This repository has been archived by the owner on Apr 26, 2024. It is now read-only.

Use importlib.metadata to read requirements #12088

Merged
merged 9 commits into from
Mar 1, 2022
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/12088.misc
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Inspect application dependencies using `importlib.metadata` or its backport.
6 changes: 3 additions & 3 deletions synapse/app/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,13 @@
import sys
from typing import Container

from synapse import python_dependencies # noqa: E402
from synapse.util import check_dependencies

logger = logging.getLogger(__name__)

try:
python_dependencies.check_requirements()
except python_dependencies.DependencyException as e:
check_dependencies.check_requirements()
except check_dependencies.DependencyException as e:
sys.stderr.writelines(
e.message # noqa: B306, DependencyException.message is a property
)
Expand Down
2 changes: 1 addition & 1 deletion synapse/app/homeserver.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@
from synapse.http.site import SynapseSite
from synapse.logging.context import LoggingContext
from synapse.metrics import METRICS_PREFIX, MetricsResource, RegistryProxy
from synapse.python_dependencies import check_requirements
from synapse.replication.http import REPLICATION_PREFIX, ReplicationRestResource
from synapse.replication.tcp.resource import ReplicationStreamProtocolFactory
from synapse.rest import ClientRestResource
Expand All @@ -70,6 +69,7 @@
from synapse.rest.well_known import well_known_resource
from synapse.server import HomeServer
from synapse.storage import DataStore
from synapse.util.check_dependencies import check_requirements
from synapse.util.httpresourcetree import create_resource_tree
from synapse.util.module_loader import load_module

Expand Down
2 changes: 1 addition & 1 deletion synapse/config/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@

import attr

from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.check_dependencies import DependencyException, check_requirements

from ._base import Config, ConfigError

Expand Down
2 changes: 1 addition & 1 deletion synapse/config/metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@

import attr

from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.check_dependencies import DependencyException, check_requirements

from ._base import Config, ConfigError

Expand Down
2 changes: 1 addition & 1 deletion synapse/config/oidc.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,11 @@

from synapse.config._util import validate_config
from synapse.config.sso import SsoAttributeRequirement
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.module_loader import load_module
from synapse.util.stringutils import parse_and_validate_mxc_uri

from ..util.check_dependencies import DependencyException, check_requirements
from ._base import Config, ConfigError, read_file

DEFAULT_USER_MAPPING_PROVIDER = "synapse.handlers.oidc.JinjaOidcMappingProvider"
Expand Down
2 changes: 1 addition & 1 deletion synapse/config/redis.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
# limitations under the License.

from synapse.config._base import Config
from synapse.python_dependencies import check_requirements
from synapse.util.check_dependencies import check_requirements


class RedisConfig(Config):
Expand Down
2 changes: 1 addition & 1 deletion synapse/config/repository.py
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@
import attr

from synapse.config.server import DEFAULT_IP_RANGE_BLACKLIST, generate_ip_set
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.check_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module

from ._base import Config, ConfigError
Expand Down
2 changes: 1 addition & 1 deletion synapse/config/saml2.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@
from typing import Any, List, Set

from synapse.config.sso import SsoAttributeRequirement
from synapse.python_dependencies import DependencyException, check_requirements
from synapse.types import JsonDict
from synapse.util.check_dependencies import DependencyException, check_requirements
from synapse.util.module_loader import load_module, load_python_module

from ._base import Config, ConfigError
Expand Down
2 changes: 1 addition & 1 deletion synapse/config/tracer.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@

from typing import Set

from synapse.python_dependencies import DependencyException, check_requirements
from synapse.util.check_dependencies import DependencyException, check_requirements

from ._base import Config, ConfigError

Expand Down
107 changes: 3 additions & 104 deletions synapse/python_dependencies.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,7 @@

import itertools
import logging
from typing import List, Set

from pkg_resources import (
DistributionNotFound,
Requirement,
VersionConflict,
get_provider,
)
from typing import Set

logger = logging.getLogger(__name__)

Expand Down Expand Up @@ -90,6 +83,8 @@
# ijson 3.1.4 fixes a bug with "." in property names
"ijson>=3.1.4",
"matrix-common~=1.1.0",
# For runtime introspection of our dependencies
"packaging~=21.3",
]

CONDITIONAL_REQUIREMENTS = {
Expand Down Expand Up @@ -144,102 +139,6 @@ def list_requirements():
return list(set(REQUIREMENTS) | ALL_OPTIONAL_REQUIREMENTS)


class DependencyException(Exception):
@property
def message(self):
return "\n".join(
[
"Missing Requirements: %s" % (", ".join(self.dependencies),),
"To install run:",
" pip install --upgrade --force %s" % (" ".join(self.dependencies),),
"",
]
)

@property
def dependencies(self):
for i in self.args[0]:
yield '"' + i + '"'


def check_requirements(for_feature=None):
deps_needed = []
errors = []

if for_feature:
reqs = CONDITIONAL_REQUIREMENTS[for_feature]
else:
reqs = REQUIREMENTS

for dependency in reqs:
try:
_check_requirement(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
"Needed %s, got %s==%s"
% (
dependency,
e.dist.project_name, # type: ignore[attr-defined] # noqa
e.dist.version, # type: ignore[attr-defined] # noqa
)
)
except DistributionNotFound:
deps_needed.append(dependency)
if for_feature:
errors.append(
"Needed %s for the '%s' feature but it was not installed"
% (dependency, for_feature)
)
else:
errors.append("Needed %s but it was not installed" % (dependency,))

if not for_feature:
# Check the optional dependencies are up to date. We allow them to not be
# installed.
OPTS: List[str] = sum(CONDITIONAL_REQUIREMENTS.values(), [])

for dependency in OPTS:
try:
_check_requirement(dependency)
except VersionConflict as e:
deps_needed.append(dependency)
errors.append(
"Needed optional %s, got %s==%s"
% (
dependency,
e.dist.project_name, # type: ignore[attr-defined] # noqa
e.dist.version, # type: ignore[attr-defined] # noqa
)
)
except DistributionNotFound:
# If it's not found, we don't care
pass

if deps_needed:
for err in errors:
logging.error(err)

raise DependencyException(deps_needed)


def _check_requirement(dependency_string):
"""Parses a dependency string, and checks if the specified requirement is installed

Raises:
VersionConflict if the requirement is installed, but with the the wrong version
DistributionNotFound if nothing is found to provide the requirement
"""
req = Requirement.parse(dependency_string)

# first check if the markers specify that this requirement needs installing
if req.marker is not None and not req.marker.evaluate():
# not required for this environment
return

get_provider(req)


if __name__ == "__main__":
import sys

Expand Down
129 changes: 129 additions & 0 deletions synapse/util/check_dependencies.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
import logging
from typing import Iterable, NamedTuple, Optional

try:
from importlib import metadata
except ImportError:
import importlib_metadata as metadata # type: ignore[no-redef]

from packaging.requirements import Requirement


class DependencyException(Exception):
@property
def message(self) -> str:
return "\n".join(
[
"Missing Requirements: %s" % (", ".join(self.dependencies),),
"To install run:",
" pip install --upgrade --force %s" % (" ".join(self.dependencies),),
"",
]
)

@property
def dependencies(self) -> Iterable[str]:
for i in self.args[0]:
yield '"' + i + '"'


EXTRAS = {
v
for k, v in metadata.distribution("matrix-synapse").metadata.items()
if k == "Provides-Extra"
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

set(metadata.distribution("matrix-synapse").metadata.get_all("Provides-Extra")) perhaps?
Or even set(metadata.metadata("matrix-synapse").get_all("Provides-Extra"))

Disclaimer: I've only confirmed it works on my machine and not in general

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That seems neater---will check this out!

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.



class Dependency(NamedTuple):
requirement: Requirement
must_be_installed: bool


def _generic_dependencies() -> Iterable[Dependency]:
"""Yield pairs (requirement, must_be_installed)."""
requirements = metadata.requires("matrix-synapse")
assert requirements is not None
for raw_requirement in requirements:
req = Requirement(raw_requirement)
# https://packaging.pypa.io/en/latest/markers.html#usage notes that
# > Evaluating an extra marker with no environment is an error
# so we pass in a dummy empty extra value here.
must_be_installed = req.marker is None or req.marker.evaluate({"extra": ""})
yield Dependency(req, must_be_installed)


def _dependencies_for_extra(extra: str) -> Iterable[Dependency]:
"""Yield additional dependencies needed for a given `extra`."""
requirements = metadata.requires("matrix-synapse")
assert requirements is not None
for raw_requirement in requirements:
req = Requirement(raw_requirement)
# Exclude mandatory deps by only selecting deps needed with this extra.
if (
req.marker is not None
and req.marker.evaluate({"extra": extra})
and not req.marker.evaluate({"extra": ""})
):
yield Dependency(req, True)


def _not_installed(requirement: Requirement, extra: Optional[str] = None) -> str:
if extra:
return f"Need {requirement.name} for {extra}, but it is not installed"
else:
return f"Need {requirement.name}, but it is not installed"


def _incorrect_version(
requirement: Requirement, got: str, extra: Optional[str] = None
) -> str:
if extra:
return f"Need {requirement} for {extra}, but got {requirement.name}=={got}"
else:
return f"Need {requirement}, but got {requirement.name}=={got}"


def check_requirements(extra: Optional[str] = None) -> None:
"""Check Synapse's dependencies are present and correctly versioned.

If provided, `extra` must be the name of an pacakging extra (e.g. "saml2" in
`pip install matrix-synapse[saml2]`).

If `extra` is None, this function checks that
- all mandatory dependencies are installed and correctly versioned, and
- each optional dependency that's installed is correctly versioned.

If `extra` is not None, this function checks that
- the dependencies needed for that extra are installed and correctly versioned.

:raises DependencyException: if a dependency is missing or incorrectly versioned.
:raises ValueError: if this extra does not exist.
"""
# First work out which dependencies are required, and which are optional.
if extra is None:
dependencies = _generic_dependencies()
elif extra in EXTRAS:
dependencies = _dependencies_for_extra(extra)
else:
raise ValueError(f"Synapse does not provide the feature '{extra}'")

deps_needed = []
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: needed is a tad misleading. deps_unfulfilled maybe?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can do. This came from the original implementation, but finding the right words for all of this stuff is tricky.

errors = []

for (requirement, must_be_installed) in dependencies:
try:
dist: metadata.Distribution = metadata.distribution(requirement.name)
except metadata.PackageNotFoundError:
if must_be_installed:
deps_needed.append(requirement.name)
errors.append(_not_installed(requirement, extra))
else:
if not requirement.specifier.contains(dist.version):
deps_needed.append(requirement.name)
errors.append(_incorrect_version(requirement, dist.version, extra))

if deps_needed:
for err in errors:
logging.error(err)

raise DependencyException(deps_needed)
Loading