diff --git a/python/dazl/ledger/config/__init__.py b/python/dazl/ledger/config/__init__.py index 2ab6b1b5..da36e7d3 100644 --- a/python/dazl/ledger/config/__init__.py +++ b/python/dazl/ledger/config/__init__.py @@ -1,40 +1,328 @@ # Copyright (c) 2017-2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +""" +:mod:`dazl.ledger.config` — connection configuration +==================================================== + +This module contains configuration objects for a :class:`Connection`. + +Normally you don't need to construct these objects directly; instead, simply call +:meth:`dazl.connect`, which contains the same options exposed by the :meth:`Config.create` function, +which in turn includes the options exposed by the subobjects of :class:`Config`. + +Generally you should not modify a :class:`Config` object (or subobjects) that have already been +passed to a :class:`Connection`. There are some exceptions to this rule, though; for example, when +using tokens, you can simply assign ``Config.access.token`` to a new value. + +Configuration options are broken up into three subobjects: + +* :ref:`Access configuration `: settings commonly found in Daml JWT tokens + (party settings, ledger ID, application name) +* :ref:`SSL/TLS configuration `: settings for configuring TLS connections +* :ref:`URL Configuration `: settings that determine the location of gRPC Ledger + API or HTTP JSON API implementation + +.. autoclass:: Config + :members: + +.. _access-configuration: + +Access configuration +-------------------- + +The :class:`AccessConfig` protocol specifies how ``dazl`` identifies itself to a ledger. There are +two built-in mechanisms for this: **property-based access**, which is traditionally used with +ledgers that do NOT require authorization/authentication (typical in a local development scenario, +for example coding against a local sandbox), and **token-based access**, which is required for +ledgers that DO require authorization/authentication (typical in a production scenario and/or hosted +ledgers). + +In **property-based access** (:class:`PropertyBasedAccessConfig`), the behavior differs depending on +whether you are connecting over the gRPC Ledger API or the HTTP JSON API: + +* For the *gRPC Ledger API*, ``read_as`` and ``act_as`` are used as-is. ``ledger_id`` is + defaulted to the value requested from the ledger (but only if not initially specified). + The ``admin`` property is ignored and unused. +* For the *HTTP JSON API*, ``read_as``, ``act_as``, ``admin``, ``ledger_id``, and + ``application_name`` are all used to generate an unsigned JWT locally. ``ledger_id`` MUST be + supplied. + +In **token-based access** (:class:`TokenBasedAccessConfig`), the value of the token completely +determines the parties that can be used, the ledger ID to connect to, and the name of the +application. ``AccessConfig.token`` can be overwritten at any time, and that value will be used for +all subsequent calls to the ledger. If your ledger requires authorization/authentication using +tokens, you _must_ use token-based access. + +Although ``dazl`` does not currently refresh tokens automatically, you can update the token yourself +at any time: + +.. code-block:: python + + async def main(): + async with dazl.connect(token=MY_INITIAL_TOKEN) as conn: + task1 = asyncio.create_task(do_ledger_stuff(conn)) + task2 = asyncio.create_task(refresh_token(conn)) + + await task1 + task2.cancel() + + async def do_ledger_stuff(conn): + # use the connection normally to make ledger calls + ... + + async def refresh(conn): + while True: + # sleep for one hour + await asyncio.sleep(3600) + conn.config.access.token = NEW_REFRESHED_TOKEN + +Deeper support for token refreshing may be added in a future release. + +.. autoclass:: AccessConfig() + :members: + +.. autofunction:: create_access + +.. autoclass:: PropertyBasedAccessConfig() + :members: + +.. autoclass:: TokenBasedAccessConfig() + :members: + +.. _ssl-tls-configuration: + +SSL/TLS configuration +--------------------- + +.. autoclass:: SSLConfig + :members: + +.. _url-configuration: + +URL configuration +----------------- + +The :class:`URLConfig` protocol specifies the ledger implementation that ``dazl`` connects to +(either gRPC Ledger API or HTTP JSON API). + +.. autofunction:: create_url + +.. autoclass:: URLConfig() + :members: + +""" +from __future__ import annotations + +import itertools +import logging +from logging import Logger from os import PathLike from typing import Collection, Optional, Union from ...prim import Party, TimeDeltaLike -from .access import AccessConfig, create_access +from .access import AccessConfig, PropertyBasedAccessConfig, TokenBasedAccessConfig, create_access +from .argv import configure_parser from .ssl import SSLConfig -from .url import URLConfig +from .url import URLConfig, create_url + +__all__ = [ + "Config", + "AccessConfig", + "SSLConfig", + "URLConfig", + "TokenBasedAccessConfig", + "PropertyBasedAccessConfig", + "create_access", + "create_url", + "configure_parser", +] -__all__ = ["Config", "AccessConfig", "SSLConfig", "URLConfig"] +# an incrementing counter that helps keep loggers for individual config connections unique +# (see https://mail.python.org/pipermail//python-ideas/2016-August/041871.html); this is safe to +# do even in multithreaded environments because ultimately we're reusing the GIL as our lock +id_generator = itertools.count() +# PyCharm thinks ``from .url import ...`` clashes with variables named ``url`` +# (same with ``ssl`` and ``access``). +# noinspection PyShadowingNames class Config: + """ + Stores configuration for a :class:`Connection`. + """ + @classmethod def create( cls, - url: "Optional[str]" = None, - read_as: "Union[None, Party, Collection[Party]]" = None, - act_as: "Union[None, Party, Collection[Party]]" = None, - admin: "Optional[bool]" = False, - ledger_id: "Optional[str]" = None, - application_name: "Optional[str]" = None, - oauth_token: "Optional[str]" = None, - ca: "Optional[bytes]" = None, - ca_file: "Optional[PathLike]" = None, - cert: "Optional[bytes]" = None, - cert_file: "Optional[PathLike]" = None, - cert_key: "Optional[bytes]" = None, - cert_key_file: "Optional[PathLike]" = None, - connect_timeout: "Optional[TimeDeltaLike]" = None, - enable_http_proxy: "bool" = True, + url: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + scheme: Optional[str] = None, + read_as: Union[None, Party, Collection[Party]] = None, + act_as: Union[None, Party, Collection[Party]] = None, + admin: Optional[bool] = False, + ledger_id: Optional[str] = None, + application_name: Optional[str] = None, + oauth_token: Optional[str] = None, + oauth_token_file: Optional[str] = None, + ca: Optional[bytes] = None, + ca_file: Optional[PathLike] = None, + cert: Optional[bytes] = None, + cert_file: Optional[PathLike] = None, + cert_key: Optional[bytes] = None, + cert_key_file: Optional[PathLike] = None, + connect_timeout: Optional[TimeDeltaLike] = None, + use_http_proxy: bool = True, + logger: Optional[Logger] = None, + logger_name: Optional[str] = None, + log_level: Optional[str] = None, ) -> "Config": - url_config = URLConfig( + """ + Create a :class:`Config` object from the supplied parameters. + + The remote can be configured either by supplying a URL or by supplying a host (and optional + port and scheme). If none of ``url``, ``host``, ``port``, or ``scheme`` are supplied, then + environment variables are consulted; if no environment variables are specified either, then + default values are used. + + The remote can be configured either by supplying a URL or by supplying a host (and optional + port and scheme). If none of ``url``, ``host``, ``port``, or ``scheme`` are supplied, then + environment variables are consulted; if no environment variables are specified either, then + default values are used. + + When the URL scheme is ``http`` or ``https``, dazl will first attempt to connect assuming + the HTTP JSON API; if this fails, gRPC Ledger API is attempted. + + If ``oauth_token`` is supplied and non-empty, then *token-based access* is used to connect + to the ledger. + + If other fields are specified, then *property-based access* is used. At least one of + ``read_as``, ``act_as``, or ``admin`` must be supplied. For the HTTP JSON API, + ``ledger_id`` MUST be supplied. + + If _no_ fields are specified, this is an error unless environment variables supply an + alternate source of configuration. + + +-------------------+----------------------+-----------------------------------------------+ + | function argument | environment variable | default value | + +===================+======================+===============================================+ + | ``url`` | DAML_LEDGER_URL | localhost:6865 | + +-------------------+----------------------+-----------------------------------------------+ + | ``host`` | DAML_LEDGER_HOST | localhost | + +-------------------+----------------------+-----------------------------------------------+ + | ``port`` | DAML_LEDGER_PORT | 6865, unless ``scheme`` is specified: | + | | | * 80 for ``http`` | + | | | * 443 for ``https`` | + | | | * 6865 for ``grpc`` | + +-------------------+----------------------+-----------------------------------------------+ + | ``scheme`` | DAML_LEDGER_SCHEME | | ``https`` for port 443 or 8443 | + | | | | ``http`` for port 80, 7575 or 8080 | + | | | | ``grpc`` for port 6865 | + | | | | ``https`` for all other ports | + +-------------------+----------------------+-----------------------------------------------+ + + HTTP(s) proxies (gRPC Ledger API only) + ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + + When connecting to the gRPC Ledger API, note that + `gRPC environment variables + `_ + are always also respected; you can set ``https_proxy``/`http_proxy`` (note the lowercase + environment variable name). dazl, by default, will _also_ disable usage of a proxy server + for a ``localhost`` host or a ``127.0.0.1`` host; this can be overridden by passing a value + of ``use_http_proxy`` to ``True``. + + :param url: + The URL to connect to. Can be used as an alternative to supplying ``host``, ``port``, + and ``scheme`` as individual values. Can alternatively be supplied via the environment + variable ``DAML_LEDGER_URL``. + :param host: + The host to connect to. Can be used as an alternative to supplying ``url`` as a combined + value. Can alternatively be supplied via the environment variable ``DAML_LEDGER_HOST``. + :param port: + The port to connect to. Can be used as an alternative to supplying ``url`` as a combined + value. Can alternatively be supplied via the environment variable ``DAML_LEDGER_PORT``. + :param scheme: + The scheme to connect to. Can be used as an alternative to supplying ``url`` as a + combined value. Can alternatively be supplied via the environment variable + ``DAML_LEDGER_SCHEME``. + :param connect_timeout: + Length of time to wait before giving up connecting to the remote and declaring an error. + The default value is 30 seconds. + :param use_http_proxy: + ``True`` to use an HTTP(S) proxy server if configured; ``False`` to avoid using any + configured server. If unspecified and the host is ``localhost``, the proxy server is + avoided; otherwise the proxy server is used. + :param ca: + A certificate authority to use to validate the server's certificate. If not supplied, + the operating system's default trust store is used. Cannot be specified with + ``ca_file``. + :param ca_file: + A file containing the certificate authority to use to validate the server's certificate. + If not supplied, the operating system's default trust store is used. Cannot be specified + with ``ca``. + :param cert: + A client-side certificate to be used when connecting to a server that requires mutual + TLS. Cannot be specified with ``cert_file``. + :param cert_file: + A file containing the client-side certificate to be used when connecting to a server + that requires mutual TLS. Cannot be specified with ``cert``. + :param cert_key: + A client-side private key to be used when connecting to a server that requires mutual + TLS. Cannot be specified with ``cert_key_file``. + :param cert_key_file: + A client-side private key to be used when connecting to a server that requires mutual + TLS. Cannot be specified with ``cert_key``. + :param read_as: + A party or set of parties on whose behalf (in addition to all parties listed in + ``act_as``) contracts can be retrieved. Cannot be specified if ``oauth_token`` or + ``oauth_token_file`` is specified. + :param act_as: + A party or set of parties on whose behalf commands should be executed. Parties here are + also implicitly granted ``read_as`` access as well. Cannot be specified if + ``oauth_token`` or ``oauth_token_file`` is specified. + :param admin: + HTTP JSON API only: allow admin endpoints to be used. This flag is ignored when + connecting to gRPC Ledger API implementations. Cannot be specified if ``oauth_token`` or + ``oauth_token_file`` is specified. + :param ledger_id: + The ledger ID to connect to. For the HTTP JSON API, this value is required. For the gRPC + Ledger API, if this value is _not_ supplied, its value will be retrieved from the + server. Cannot be specified if ``oauth_token`` or ``oauth_token_file`` is specified. + :param application_name: + A string that identifies this application. This is used for tracing purposes on the + server-side. Cannot be specified if ``oauth_token`` or ``oauth_token_file`` is + specified. + :param oauth_token: + The OAuth bearer token to be used on all requests. If specified, no other access + parameters can be specified. + :param oauth_token_file: + A file that contains the OAuth bearer token to be used on all requests. If specified, no + other access parameters can be specified. + :param logger: + The logger to use for connections created from this configuration. If not supplied, a + logger will be created. + :param logger_name: + The name of the logger. Only used if ``logger`` is not provided. + :param log_level: + The logging level for the logger. The default is ``warn``. Only used if ``logger`` is + not provided. + """ + if logger is None: + if not logger_name: + logger_name = f"dazl.conn.{next(id_generator)}" + logger = logging.getLogger(logger_name) + if log_level is not None: + logger.setLevel(log_level) + + url_config = create_url( url=url, + host=host, + port=port, + scheme=scheme, connect_timeout=connect_timeout, - enable_http_proxy=enable_http_proxy, + use_http_proxy=use_http_proxy, + logger=logger, ) access_config = create_access( @@ -44,6 +332,8 @@ def create( ledger_id=ledger_id, application_name=application_name, oauth_token=oauth_token, + oauth_token_file=oauth_token_file, + logger=logger, ) ssl_config = SSLConfig( @@ -53,11 +343,16 @@ def create( cert_file=cert_file, cert_key=cert_key, cert_key_file=cert_key_file, + logger=logger, ) - return cls(access_config, ssl_config, url_config) + return cls(access_config, ssl_config, url_config, logger) - def __init__(self, access: "AccessConfig", ssl: "SSLConfig", url: "URLConfig"): + def __init__(self, access: AccessConfig, ssl: SSLConfig, url: URLConfig, logger: Logger): + """ + Initialize an instance of :class:`Config`. + """ self.access = access self.ssl = ssl self.url = url + self.logger = logger diff --git a/python/dazl/ledger/config/access.py b/python/dazl/ledger/config/access.py index b010727e..9578b392 100644 --- a/python/dazl/ledger/config/access.py +++ b/python/dazl/ledger/config/access.py @@ -1,35 +1,101 @@ # Copyright (c) 2017-2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + import base64 from collections.abc import MutableSet as MutableSetBase, Set as SetBase import json -from typing import AbstractSet, Any, Collection, Iterator, Mapping, MutableSet, Optional, Union +import os +from pathlib import Path +from typing import ( + TYPE_CHECKING, + AbstractSet, + Any, + Collection, + Iterator, + Mapping, + MutableSet, + Optional, + Protocol, + Union, + runtime_checkable, +) from ...prim import Party +from .exc import ConfigError + +__all__ = [ + "AccessConfig", + "TokenBasedAccessConfig", + "PropertyBasedAccessConfig", + "PartyRights", + "PartyRightsSet", + "create_access", +] + +if TYPE_CHECKING: + # We refer to the Config class in a docstring and + # without this import, Sphinx can't resolve the reference + # noinspection PyUnresolvedReferences + from . import Config + + +def parties_from_env(*env_vars: str) -> AbstractSet[Party]: + """ + Read the set of parties + """ + return {Party(p) for env_var in env_vars for p in os.getenv(env_var, "").split(",") if p} +# mypy note: typing.overload cannot properly express a more correct signature for this function, +# which is that if oauth_token is supplied, then a TokenBasedAccessConfig class is returned and +# otherwise, a PropertyBasedAccessConfig class is returned. Trying to type this properly with +# overloads results in: +# "Not all union combinations were tried because there are too many unions" +# It's also worth nothing that our only usage of this function also supplies all parameters since +# we're effectively just passing **kwargs around, so it's not clear a more accurate type helps much +# anyway. def create_access( *, - read_as: "Union[None, Party, Collection[Party]]" = None, - act_as: "Union[None, Party, Collection[Party]]" = None, - admin: "Optional[bool]" = None, - ledger_id: "Optional[str]" = None, - application_name: "Optional[str]" = None, - oauth_token: "Optional[str]" = None, -) -> "AccessConfig": - if oauth_token: - # if a token is supplied, none of the other arguments are allowed - if ( - read_as is not None - or act_as is not None - or admin is not None - or ledger_id is not None - or application_name is not None - ): - raise ValueError( - "cannot configure access with both tokens and " - "read_as/act_as/admin/ledger_id/application_name configuration options" - ) + read_as: Union[None, Party, Collection[Party]] = None, + act_as: Union[None, Party, Collection[Party]] = None, + admin: Optional[bool] = None, + ledger_id: Optional[str] = None, + application_name: Optional[str] = None, + oauth_token: Optional[str] = None, + oauth_token_file: Optional[str] = None, +) -> AccessConfig: + """ + Create an appropriate instance of :class:`AccessConfig`. + + See :meth:`Config.create` for a more detailed description of these parameters. + + """ + # admin = None is effectively the same as admin = False in this context + is_property_based = read_as or act_as or admin or ledger_id or application_name + if not is_property_based and not oauth_token and not oauth_token_file: + # none of the access-related parameters were passed in, so try to read some from the + # environment + act_as = parties_from_env("DAML_LEDGER_ACT_AS", "DAML_LEDGER_PARTY") + read_as = parties_from_env("DAML_LEDGER_READ_AS", "DABL_PUBLIC_PARTY") + ledger_id = os.getenv("DAML_LEDGER_ID", "") + application_name = os.getenv("DAML_LEDGER_APPLICATION_NAME") + oauth_token = os.getenv("DAML_LEDGER_OAUTH_TOKEN") + oauth_token_file = os.getenv("DAML_LEDGER_OAUTH_TOKEN_FILE") + + is_property_based = read_as or act_as or admin or ledger_id or application_name + if not is_property_based and not oauth_token and not oauth_token_file: + raise ConfigError("no oauth token access or read_as/act_as/admin was specified") + + # how do they configure thee? let me count the ways... + if sum(map(int, (is_property_based, oauth_token, oauth_token_file))): + raise ConfigError( + "must specify ONE of read_as/act_as/admin, oauth_token, or oauth_token_file" + ) + + if oauth_token_file: + return TokenFileBasedAccessConfig(oauth_token_file) + elif oauth_token: return TokenBasedAccessConfig(oauth_token) else: return PropertyBasedAccessConfig( @@ -41,30 +107,44 @@ def create_access( ) -class AccessConfig: +@runtime_checkable +class AccessConfig(Protocol): """ Configuration parameters for providing access to a ledger. + + To create an instance of this protocol, call :func:`create_access` and provide *either* + ``oauth_token`` *or* the other fields you wish to set (such as ``act_as``). You cannot specify + both an access token and other fields. + + You may implement this protocol using your own custom type if you have very specialized access + needs. """ @property - def read_as(self) -> "AbstractSet[Party]": + def read_as(self) -> AbstractSet[Party]: """ The set of parties that can be used to read data from the ledger. This also includes the set of parties that can be used to write data to the ledger. + + :type: AbstractSet[Party] """ raise NotImplementedError @property - def read_only_as(self) -> "AbstractSet[Party]": + def read_only_as(self) -> AbstractSet[Party]: """ The set of parties that have read-only access to the underlying ledger. + + :type: AbstractSet[Party] """ raise NotImplementedError @property - def act_as(self) -> "AbstractSet[Party]": + def act_as(self) -> AbstractSet[Party]: """ The set of parties that can be used to write data to the ledger. + + :type: AbstractSet[Party] """ raise NotImplementedError @@ -72,13 +152,17 @@ def act_as(self) -> "AbstractSet[Party]": def admin(self) -> bool: """ ``True`` if the token grants "admin" access. + + :type: bool """ raise NotImplementedError @property - def ledger_id(self) -> "Optional[str]": + def ledger_id(self) -> Optional[str]: """ The ledger ID. For non-token based access methods, this can be queried from the ledger. + + :type: Optional[str] """ raise NotImplementedError @@ -86,6 +170,8 @@ def ledger_id(self) -> "Optional[str]": def application_name(self) -> str: """ The application name. + + :type: str """ raise NotImplementedError @@ -93,6 +179,8 @@ def application_name(self) -> str: def token(self) -> str: """ The bearer token that provides authorization and authentication to a ledger. + + :type: str """ raise NotImplementedError @@ -104,12 +192,18 @@ class TokenBasedAccessConfig(AccessConfig): """ def __init__(self, oauth_token: str): + """ + Initialize a token-based access configuration. + + :param oauth_token: The initial value of the bearer token. + """ self.token = oauth_token @property def token(self) -> str: """ - The bearer token that provides authorization and authentication to a ledger. + The bearer token that provides authorization and authentication to a ledger. This value can + be replaced on a live connection in order to support use cases such as token refreshing. """ return self._token @@ -128,25 +222,37 @@ def token(self, value: str) -> None: self._ledger_id = claims.get("ledgerId", None) self._application_name = claims.get("applicationId", None) - def read_as(self) -> "AbstractSet[Party]": + @property + def read_as(self) -> AbstractSet[Party]: return self._read_as - def read_only_as(self) -> "AbstractSet[Party]": + @property + def read_only_as(self) -> AbstractSet[Party]: return self._read_only_as - def act_as(self) -> "AbstractSet[Party]": + @property + def act_as(self) -> AbstractSet[Party]: return self._act_as + @property def admin(self) -> bool: return self._admin - def ledger_id(self) -> "Optional[str]": + @property + def ledger_id(self) -> Optional[str]: return self._ledger_id + @property def application_name(self) -> str: return self._application_name +class TokenFileBasedAccessConfig(TokenBasedAccessConfig): + def __init__(self, oauth_token_file: str): + # TODO: Update the token when the contents of this file change. + super().__init__(Path(oauth_token_file).read_text()) + + class PropertyBasedAccessConfig(AccessConfig): """ Access configuration that is manually specified outside of an authentication/authorization @@ -156,34 +262,34 @@ class PropertyBasedAccessConfig(AccessConfig): def __init__( self, - party: "Optional[Party]" = None, - read_as: "Union[None, Party, Collection[Party]]" = None, - act_as: "Union[None, Party, Collection[Party]]" = None, - admin: "Optional[bool]" = False, - ledger_id: "Optional[str]" = None, - application_name: "Optional[str]" = None, + read_as: Union[None, Party, Collection[Party]] = None, + act_as: Union[None, Party, Collection[Party]] = None, + admin: Optional[bool] = False, + ledger_id: Optional[str] = None, + application_name: Optional[str] = None, ): """ + Initialize a property-based access configuration. - :param party: - The singular party to use for reading and writing. This parameter is a convenience - parameter for the common case where "read as" and "act as" parties are the same, - and there is only one of them. If you specify this parameter, you CANNOT supply - ``read_as`` or ``act_as``, nor can you supply an access token When connecting to the - HTTP JSON API, ``ledger_id`` must _also_ be supplied when using this parameter.. :param read_as: + A party or set of parties on whose behalf (in addition to all parties listed in ``act_as``) + contracts can be retrieved. :param act_as: - A party of set of parties that can be used to submit commands to the ledger. In a - Daml-based ledger, act-as rights imply read-as rights. If you specify this parameter, - you CANNOT supply ``party``, nor can you supply an access token. When connecting to the - HTTP JSON API, ``ledger_id`` must _also_ be supplied when using this parameter. + A party or set of parties on whose behalf commands should be executed. Parties here are also + implicitly granted ``read_as`` access as well. + :param admin: + HTTP JSON API only: allow admin endpoints to be used. This flag is ignored when connecting + to gRPC Ledger API implementations. :param ledger_id: - The + The ledger ID to connect to. For the HTTP JSON API, this value is required. For the gRPC + Ledger API, if this value is _not_ supplied, its value will be retrieved from the server. + :param application_name: + A string that identifies this application. This is used for tracing purposes on the + server-side. """ self._parties = PartyRights() self._parties.maybe_add(read_as, False) self._parties.maybe_add(act_as, True) - self._parties.maybe_add(party, True) self._admin = bool(admin) self._ledger_id = ledger_id self._application_name = application_name or "dazl-client" @@ -199,6 +305,12 @@ def token(self): @property def ledger_id(self) -> "Optional[str]": + """ + The ledger ID. When connecting to the gRPC Ledger API, this can be inferred and does not + need to be supplied. When connecting to the HTTP JSON API, it must be supplied. + + :type: Optional[str] + """ return self._ledger_id @ledger_id.setter @@ -212,32 +324,51 @@ def application_name(self) -> str: @property def read_as(self) -> "AbstractSet[Party]": """ - Return the set of parties for which read rights are granted. + The set of parties for which read rights are granted. This collection is read-only. If you + want to add a party with read-only access, add it to :meth:`read_only_as`; if you want to + add a party with act-as access as well as read-only access, add it to :meth:`act_as`. This set always includes the act_as parties. For the set of parties that can be read as - but NOT acted as, use :property:`read_only_as`. + but NOT acted as, use :meth:`read_only_as`. + + :type: AbstractSet[Party] """ return self._parties @property def read_only_as(self) -> "MutableSet[Party]": - """""" + """ + The set of parties for which read-as rights are granted, but act-as rights are NOT granted. + This collection can be modified. + + :type: MutableSet[Party] + """ return self._parties.read_as @property def act_as(self) -> "MutableSet[Party]": """ - Return the set of parties for which act-as rights are granted. This collection can be - modified. + The set of parties for which act-as rights are granted. This collection can be modified. + Adding a party to this set _removes_ it from :meth:`read_only_as`. + + :type: MutableSet[Party] """ return self._parties.act_as @property def admin(self) -> bool: + """ + Whether or not the token sent to HTTP JSON API contains the ``admin: true`` flag that + signals a token bearer with admin access. + """ return self._admin + @admin.setter + def admin(self, value: bool) -> None: + self._admin = value + -def parties(p: "Union[None, Party, Collection[Party]]") -> "Collection[Party]": +def parties(p: Union[None, Party, Collection[Party]]) -> Collection[Party]: if p is None: return [] elif isinstance(p, str): @@ -249,7 +380,7 @@ def parties(p: "Union[None, Party, Collection[Party]]") -> "Collection[Party]": DamlLedgerApiNamespace = "https://daml.com/ledger-api" -def decode_token(token: str) -> "Mapping[str, Any]": +def decode_token(token: str) -> Mapping[str, Any]: components = token.split(".", 3) if len(components) != 3: raise ValueError("not a JWT") @@ -262,8 +393,8 @@ def decode_token(token: str) -> "Mapping[str, Any]": def encode_unsigned_token( - read_as: "Collection[Party]", - act_as: "Collection[Party]", + read_as: Collection[Party], + act_as: Collection[Party], ledger_id: str, application_id: str, admin: bool = True, diff --git a/python/dazl/ledger/config/ssl.py b/python/dazl/ledger/config/ssl.py index a6ec21ef..cd8c9bfd 100644 --- a/python/dazl/ledger/config/ssl.py +++ b/python/dazl/ledger/config/ssl.py @@ -1,12 +1,20 @@ # Copyright (c) 2017-2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 from os import PathLike, fspath -from typing import Optional +from typing import TYPE_CHECKING, Optional + +if TYPE_CHECKING: + # We refer to the Config class in a docstring and + # without this import, Sphinx can't resolve the reference + # noinspection PyUnresolvedReferences + from . import Config class SSLConfig: """ Configuration parameters that affect SSL connections. + + See :meth:`Config.create` for a more detailed description of these parameters. """ def __init__( diff --git a/python/dazl/ledger/config/url.py b/python/dazl/ledger/config/url.py index d74839e8..9f0aad3f 100644 --- a/python/dazl/ledger/config/url.py +++ b/python/dazl/ledger/config/url.py @@ -1,88 +1,167 @@ # Copyright (c) 2017-2021 Digital Asset (Switzerland) GmbH and/or its affiliates. All rights reserved. # SPDX-License-Identifier: Apache-2.0 +from __future__ import annotations + from datetime import timedelta +import ipaddress +from logging import Logger, getLogger import os -from typing import Optional +from reprlib import repr +from types import MappingProxyType +from typing import TYPE_CHECKING, Optional, Protocol, runtime_checkable from urllib.parse import urlparse +import warnings from ...prim import TimeDeltaLike, to_timedelta +from .exc import ConfigError, ConfigWarning + +__all__ = ["URLConfig", "create_url", "KNOWN_SCHEME_PORTS", "DEFAULT_CONNECT_TIMEOUT"] + +if TYPE_CHECKING: + # We refer to the Config class in a docstring and + # without this import, Sphinx can't resolve the reference + # noinspection PyUnresolvedReferences + from . import Config + +DEFAULT_CONNECT_TIMEOUT = timedelta(seconds=30) + +# The set of schemes we understand, and the known ports that map to those schemes. The first port +# in the list is the default value for the scheme. +KNOWN_SCHEME_PORTS = MappingProxyType( + {"http": (80, 7575, 8080), "https": (443, 8443), "grpc": (6865,), "grpcs": ()} +) + +# some environment variables that we frequently refer to +DAML_LEDGER_URL = "DAML_LEDGER_URL" +DAML_LEDGER_HOST = "DAML_LEDGER_HOST" +DAML_LEDGER_PORT = "DAML_LEDGER_PORT" +DAML_LEDGER_SCHEME = "DAML_LEDGER_SCHEME" + + +def create_url( + *, + url: Optional[str] = None, + host: Optional[str] = None, + port: Optional[int] = None, + scheme: Optional[str] = None, + connect_timeout: Optional[TimeDeltaLike] = None, + use_http_proxy: Optional[bool] = None, + logger: Optional[Logger] = None, +): + """ + Create an instance of :class:`URLConfig`, possibly with values taken from environment variables, + or defaulted if otherwise unspecified. -__all__ = ["URLConfig"] - -DEFAULT_CONNECT_TIMEOUT = timedelta(30) - - -class URLConfig: - def __init__( - self, - url: "Optional[str]" = None, - host: "Optional[str]" = None, - port: "Optional[int]" = None, - scheme: "Optional[str]" = None, - connect_timeout: "Optional[TimeDeltaLike]" = DEFAULT_CONNECT_TIMEOUT, - enable_http_proxy: bool = True, - ): - url = url if url is not None else os.getenv("DAML_LEDGER_URL", "//localhost:6865") - if url: - if host or port or scheme: - raise ValueError("url or host/port/scheme must be specified, but not both") - self._url = sanitize_url(url) - components = urlparse(self._url, allow_fragments=False) - self._host = components.hostname - self._port = components.port - self._scheme = components.scheme - else: - self._scheme = scheme or "" - self._host = host or "localhost" - self._port = port or 6865 - self._url = f"{self._scheme}//{self._host}:{self._port}" - self._connect_timeout = ( - to_timedelta(connect_timeout) - if connect_timeout is not None - else DEFAULT_CONNECT_TIMEOUT + See :meth:`Config.create` for a more detailed description of these parameters. + """ + if logger is None: + logger = getLogger("dazl.conn") + + if not url and not host and not port and not scheme: + # use environment variables to provide default values + url = os.getenv(DAML_LEDGER_URL) + host = os.getenv(DAML_LEDGER_HOST) + port = os.getenv(DAML_LEDGER_PORT) + scheme = os.getenv(DAML_LEDGER_SCHEME) + if host or port or scheme: + if url: + logger.error("Found conflicting environment variables:") + logger.error(" %s=%s", DAML_LEDGER_URL, url) + logger.error(" %s=%s", DAML_LEDGER_HOST, host) + logger.error(" %s=%s", DAML_LEDGER_PORT, port) + logger.error(" %s=%s", DAML_LEDGER_SCHEME, scheme) + logger.error( + f"Specify ONLY either {DAML_LEDGER_URL} OR " + "{DAML_LEDGER_HOST}/{DAML_LEDGER_PORT}/{DAML_LEDGER_SCHEME}" + ) + else: + logger.info("Using URL configuration from the environment:") + logger.info(" %s=%s", DAML_LEDGER_HOST, host) + logger.info(" %s=%s", DAML_LEDGER_PORT, port) + logger.info(" %s=%s", DAML_LEDGER_SCHEME, scheme) + elif url: + logger.info("Using URL configuration from the environment:") + logger.info(" %s=%s", DAML_LEDGER_URL, url) + + if not url and not host and not port and not scheme: + # if no values are supplied _and_ no environment variables are specified either, then + # fall back to default values + logger.info( + 'Configuring a connection to "localhost" because neither url/host/port/scheme nor ' + f"the environment variables {DAML_LEDGER_URL}, {DAML_LEDGER_HOST}, {DAML_LEDGER_PORT}, " + f"or {DAML_LEDGER_SCHEME} are defined" ) - self._enable_http_proxy = enable_http_proxy + url = "localhost" - @property - def url(self) -> str: - """ - The full URL to connect to, including a protocol, host, and port. - """ - return self._url + if url: + if host or port or scheme: + raise ValueError("url or host/port/scheme must be specified, but not both") + url = sanitize_url(url) + else: + url = build_url(host, port, scheme) - @property - def scheme(self) -> "Optional[str]": - return self._scheme + if use_http_proxy is None: + use_http_proxy = not is_localhost(urlparse(url).hostname) - @property - def host(self) -> "Optional[str]": - return self._host + logger.debug("Building a URL configuration:") + logger.debug(" url=%s", url) + logger.debug(" connect_timeout=%s", connect_timeout) + logger.debug(" use_http_proxy=%s", use_http_proxy) + + return SimpleURLConfig( + url=url, + connect_timeout=to_timedelta(connect_timeout) if connect_timeout is not None else None, + use_http_proxy=use_http_proxy, + ) + + +@runtime_checkable +class URLConfig(Protocol): + """ + Configuration parameters for the remote host, including basic connection parameters that do not + belong in :class:`AccessConfig`. + """ @property - def port(self) -> "Optional[int]": - return self._port + def url(self) -> str: + """ + The full URL to connect to, including a protocol (scheme), host, and port. + """ + raise NotImplementedError @property - def enable_http_proxy(self) -> bool: + def use_http_proxy(self) -> bool: """ Whether to allow the use of HTTP proxies. """ - return self._enable_http_proxy + raise NotImplementedError @property - def connect_timeout(self) -> "timedelta": + def connect_timeout(self) -> timedelta: """ How long to wait for a connection before giving up. The default is 30 seconds. """ - return self._connect_timeout + raise NotImplementedError + + +class SimpleURLConfig: + """ + Trivial implementation of the :class:`URLConfig` protocol. + """ + + def __init__(self, url: str, connect_timeout: timedelta, use_http_proxy: bool): + self.url = url + self.connect_timeout = connect_timeout + self.use_http_proxy = use_http_proxy def sanitize_url(url: str) -> str: """ - Perform some basic sanitization on a URL string: - * Convert a URL with no specified protocol to one with a blank protocol + Perform some basic sanitization on a URL string (see :meth:`build_url`): + * Supply a default protocol according to our general rules + * Supply a port, even if one wasn't specified * Strip out any trailing slashes >>> sanitize_url("somewhere:1000") @@ -97,4 +176,76 @@ def sanitize_url(url: str) -> str: first_slash = url.find("/") if first_slash == -1 or first_slash != url.find("//"): url = "//" + url - return url + + original = urlparse(url) + sanitized = urlparse(build_url(original.hostname, original.port, original.scheme)) + return original._replace( + netloc=f"{sanitized.hostname}:{sanitized.port}", scheme=sanitized.scheme + ).geturl() + + +def build_url(host: Optional[str], port: Optional[int], scheme: Optional[str]) -> str: + """ + Build a URL from host/port/scheme components. + + :param host: + The host to connect to. If unspecified, ``"localhost"`` is assumed. + :param port: + The port to connect to. If provided, must be a valid port number + (between 1 and 65535 inclusive). + :param scheme: + The scheme to use (must be ``"http"``, ``"https"``, ``"grpc"``, ``"grpcs"`` or ``None``). + :raises ConfigError: One or more of the passed parameters had invalid values. + """ + + if not host: + host = "localhost" + + if port is not None and port != "": + port = int(port) + if not 1 <= port <= 65535: + raise ConfigError(f"not a valid port number: {repr(port)}") + + if scheme: + # Scheme specified; if a port is _not_ specified, the we'll try to figure out a + # default from the scheme. + scheme = scheme.lower() + known_ports = KNOWN_SCHEME_PORTS.get(scheme) + if known_ports is None: + raise ConfigError(f"not a valid scheme: {repr(scheme)}") + if port: + return f"{scheme}://{host}:{port}" + elif not known_ports: + raise ConfigError( + f"there is no default port for {scheme} URLs; you must specify a port" + ) + else: + return f"{scheme}://{host}:{known_ports[0]}" + + if port: + # No scheme provided, but a port was provided. + # See if we can infer the scheme from the provided port. + for proposed_scheme, ports in KNOWN_SCHEME_PORTS.items(): + if port in ports: + return f"{proposed_scheme}://{host}:{port}" + + warnings.warn( + f"A port ({port}) was specified without supplying a scheme (http/https); " + "will attempt connecting with https. Unless you are using a standard port, " + "you should specify a scheme explicitly.", + ConfigWarning, + ) + + # Neither scheme nor port are provided. If the host is localhost, then assume we're targeting + # a sandbox; otherwise prefer an SSL/TLS connection. + return f"grpc://{host}:6865" if is_localhost(host) else f"https://{host}:443" + + +def is_localhost(host: str) -> bool: + if host == "localhost": + return True + try: + addr = ipaddress.ip_address(host) + return addr.is_loopback + except ValueError: + return False diff --git a/python/mypy.ini b/python/mypy.ini index 34c32b5c..fa25d222 100644 --- a/python/mypy.ini +++ b/python/mypy.ini @@ -1,8 +1,5 @@ [mypy] -[mypy-grpc.*] -ignore_missing_imports = True - [mypy-google.protobuf.pyext.*] ignore_missing_imports = True diff --git a/python/poetry.lock b/python/poetry.lock index bae3bfef..724cc0df 100644 --- a/python/poetry.lock +++ b/python/poetry.lock @@ -187,6 +187,19 @@ version = ">=3.1.4,<5" aiohttp = ["aiohttp (>=3.6.2,<4.0.0dev)"] pyopenssl = ["pyopenssl (>=20.0.0)"] +[[package]] +category = "dev" +description = "Mypy stubs for gRPC" +name = "grpc-stubs" +optional = false +python-versions = ">=3.6" +version = "1.24.5" + +[package.dependencies] +grpcio = "*" +mypy = ">=0.730" +typing-extensions = "*" + [[package]] category = "main" description = "HTTP/2-based RPC framework" @@ -805,7 +818,7 @@ pygments = ["pygments"] server = ["aiohttp"] [metadata] -content-hash = "7e4928447dd57782785c6e05724719190c61f42fc7e869567f38f42d5ddd5522" +content-hash = "f02eb8d3b0606de7a2eebfea612a4e4200a7c8807fd4daa974690caf6832134a" python-versions = "^3.8" [metadata.files] @@ -907,6 +920,10 @@ google-auth = [ {file = "google-auth-1.27.1.tar.gz", hash = "sha256:d8958af6968e4ecd599f82357ebcfeb126f826ed0656126ad68416f810f7531e"}, {file = "google_auth-1.27.1-py2.py3-none-any.whl", hash = "sha256:63a5636d7eacfe6ef5b7e36e112b3149fa1c5b5ad77dd6df54910459bcd6b89f"}, ] +grpc-stubs = [ + {file = "grpc-stubs-1.24.5.tar.gz", hash = "sha256:d875d5827e0a1ee6003aa5848243ea5df650f89cef8cf8abd917a27493e970e7"}, + {file = "grpc_stubs-1.24.5-py3-none-any.whl", hash = "sha256:2ce3a4ce40a7bb755620f012bf7301a63fa42cfa7d2fb715f69854d9123ceaa9"}, +] grpcio = [ {file = "grpcio-1.36.1-cp27-cp27m-macosx_10_10_x86_64.whl", hash = "sha256:e3a83c5db16f95daac1d96cf3c9018d765579b5a29bb336758d793028e729921"}, {file = "grpcio-1.36.1-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:c18739fecb90760b183bfcb4da1cf2c6bf57e38f7baa2c131d5f67d9a4c8365d"}, diff --git a/python/pyproject.toml b/python/pyproject.toml index 92e00d21..082265bb 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -39,6 +39,7 @@ setuptools = "==40.8.0" sphinx = "*" sphinx-markdown-builder = "^0.5.1" watchdog = "*" +grpc-stubs = "^1.24.5" [tool.poetry.extras] oauth = ["google-auth", "oauthlib"]