Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add types to the public API #244

Merged
merged 4 commits into from
Feb 4, 2022
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions MANIFEST.in
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
include tldextract/.tld_set_snapshot
include tldextract/py.typed
include LICENSE
recursive-include tests *.py *.dat
3 changes: 2 additions & 1 deletion tldextract/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,5 @@

from .tldextract import extract, TLDExtract

from ._version import version as __version__
from . import _version
__version__: str = _version.version
4 changes: 2 additions & 2 deletions tldextract/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
_DID_LOG_UNABLE_TO_CACHE = False


def get_pkg_unique_identifier():
def get_pkg_unique_identifier() -> str:
"""
Generate an identifier unique to the python version, tldextract version, and python instance

Expand Down Expand Up @@ -46,7 +46,7 @@ def get_pkg_unique_identifier():
return pkg_identifier


def get_cache_dir():
def get_cache_dir() -> str:
"""
Get a cache dir that we have permission to write to

Expand Down
Empty file added tldextract/py.typed
Empty file.
2 changes: 1 addition & 1 deletion tldextract/suffix_list.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import re

import requests
from requests_file import FileAdapter
from requests_file import FileAdapter # type: ignore[import]

LOG = logging.getLogger("tldextract")

Expand Down
59 changes: 32 additions & 27 deletions tldextract/tldextract.py
Original file line number Diff line number Diff line change
Expand Up @@ -49,10 +49,10 @@
'127.0.0.1'
"""

import collections
import logging
import os
from functools import wraps
from typing import List, NamedTuple, Optional, Sequence, Union

import idna

Expand All @@ -71,14 +71,14 @@
)


class ExtractResult(collections.namedtuple("ExtractResult", "subdomain domain suffix")):
class ExtractResult(NamedTuple):
"""namedtuple of a URL's subdomain, domain, and suffix."""

# Necessary for __dict__ member to get populated in Python 3+
__slots__ = ()
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks for deleting! 🙌 This workaround has been obsolete for a long time. See #70 (comment).

subdomain: str
domain: str
suffix: str

@property
def registered_domain(self):
def registered_domain(self) -> str:
"""
Joins the domain and suffix fields with a dot, if they're both set.

Expand All @@ -92,7 +92,7 @@ def registered_domain(self):
return ""

@property
def fqdn(self):
def fqdn(self) -> str:
"""
Returns a Fully Qualified Domain Name, if there is a proper domain/suffix.

Expand All @@ -103,11 +103,12 @@ def fqdn(self):
"""
if self.domain and self.suffix:
# self is the namedtuple (subdomain domain suffix)
return ".".join(i for i in self if i)
# https://github.com/PyCQA/pylint/issues/2568
return ".".join(i for i in self if i) # pylint: disable=not-an-iterable
return ""

@property
def ipv4(self):
def ipv4(self) -> str:
"""
Returns the ipv4 if that is what the presented domain/url is

Expand All @@ -130,13 +131,13 @@ class TLDExtract:
# TODO: Agreed with Pylint: too-many-arguments
def __init__( # pylint: disable=too-many-arguments
self,
cache_dir=get_cache_dir(),
suffix_list_urls=PUBLIC_SUFFIX_LIST_URLS,
fallback_to_snapshot=True,
include_psl_private_domains=False,
extra_suffixes=(),
cache_fetch_timeout=CACHE_TIMEOUT,
):
cache_dir: str = get_cache_dir(),
suffix_list_urls: Sequence[str] = PUBLIC_SUFFIX_LIST_URLS,
fallback_to_snapshot: bool = True,
include_psl_private_domains: bool = False,
extra_suffixes: Sequence[str] = (),
cache_fetch_timeout: Union[str, float, None] = CACHE_TIMEOUT,
) -> None:
"""
Constructs a callable for extracting subdomain, domain, and suffix
components from a URL.
Expand Down Expand Up @@ -193,14 +194,18 @@ def __init__( # pylint: disable=too-many-arguments

self.include_psl_private_domains = include_psl_private_domains
self.extra_suffixes = extra_suffixes
self._extractor = None
self._extractor: Optional[_PublicSuffixListTLDExtractor] = None

self.cache_fetch_timeout = cache_fetch_timeout
self.cache_fetch_timeout = (
float(cache_fetch_timeout)
if isinstance(cache_fetch_timeout, str)
else cache_fetch_timeout
)
self._cache = DiskCache(cache_dir)
if isinstance(self.cache_fetch_timeout, str):
self.cache_fetch_timeout = float(self.cache_fetch_timeout)

def __call__(self, url, include_psl_private_domains=None):
def __call__(
self, url: str, include_psl_private_domains: Optional[bool] = None
) -> ExtractResult:
"""
Takes a string URL and splits it into its subdomain, domain, and
suffix (effective TLD, gTLD, ccTLD, etc.) component.
Expand Down Expand Up @@ -238,23 +243,23 @@ def __call__(self, url, include_psl_private_domains=None):
domain = labels[suffix_index - 1] if suffix_index else ""
return ExtractResult(subdomain, domain, suffix)

def update(self, fetch_now=False):
def update(self, fetch_now: bool = False) -> None:
"""Force fetch the latest suffix list definitions."""
self._extractor = None
self._cache.clear()
if fetch_now:
self._get_tld_extractor()

@property
def tlds(self):
def tlds(self) -> List[str]:
"""
Returns the list of tld's used by default

This will vary based on `include_psl_private_domains` and `extra_suffixes`
"""
return list(self._get_tld_extractor().tlds())

def _get_tld_extractor(self):
def _get_tld_extractor(self) -> "_PublicSuffixListTLDExtractor":
"""Get or compute this object's TLDExtractor. Looks up the TLDExtractor
in roughly the following order, based on the settings passed to
__init__:
Expand Down Expand Up @@ -290,9 +295,9 @@ def _get_tld_extractor(self):


@wraps(TLD_EXTRACTOR.__call__)
def extract(
url, include_psl_private_domains=False
): # pylint: disable=missing-function-docstring
def extract( # pylint: disable=missing-function-docstring
url: str, include_psl_private_domains: Optional[bool] = False
) -> ExtractResult:
return TLD_EXTRACTOR(url, include_psl_private_domains=include_psl_private_domains)


Expand Down
9 changes: 8 additions & 1 deletion tox.ini
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
[tox]
envlist = py{36,37,38,39,py3},codestyle,lint
envlist = py{36,37,38,39,py3},codestyle,lint,typecheck

[testenv]
deps =
Expand All @@ -24,6 +24,13 @@ deps =
responses
commands = pytest --pylint -m pylint {posargs}

[testenv:typecheck]
deps =
mypy
types-requests
types-filelock
commands = mypy tldextract --show-error-codes

[pycodestyle]
# E203 - whitespace before; disagrees with PEP8 https://github.com/psf/black/issues/354#issuecomment-397684838
# E501 - line too long
Expand Down