From ea877227c11968158d91e1ccd601bfceeca8bde2 Mon Sep 17 00:00:00 2001 From: Chris White Date: Thu, 16 Jan 2025 18:05:41 -0800 Subject: [PATCH] Expose more Sqlalchemy settings, part 1 (#16742) Co-authored-by: zzstoatzz --- docs/v3/develop/settings-ref.mdx | 89 ++++-- schemas/settings.schema.json | 97 +++--- setup.cfg | 1 + .../prefect_kubernetes/settings.py | 6 +- .../prefect-redis/prefect_redis/client.py | 4 +- src/prefect/server/database/configurations.py | 20 +- src/prefect/settings/base.py | 5 +- src/prefect/settings/legacy.py | 2 +- src/prefect/settings/models/api.py | 7 +- src/prefect/settings/models/cli.py | 7 +- src/prefect/settings/models/client.py | 11 +- src/prefect/settings/models/cloud.py | 7 +- src/prefect/settings/models/deployments.py | 7 +- src/prefect/settings/models/experiments.py | 7 +- src/prefect/settings/models/flows.py | 7 +- src/prefect/settings/models/internal.py | 7 +- src/prefect/settings/models/logging.py | 10 +- src/prefect/settings/models/results.py | 7 +- src/prefect/settings/models/root.py | 15 +- src/prefect/settings/models/runner.py | 13 +- src/prefect/settings/models/server/api.py | 9 +- .../settings/models/server/database.py | 124 ++++++-- .../settings/models/server/deployments.py | 7 +- .../settings/models/server/ephemeral.py | 9 +- src/prefect/settings/models/server/events.py | 9 +- .../settings/models/server/flow_run_graph.py | 7 +- src/prefect/settings/models/server/root.py | 7 +- .../settings/models/server/services.py | 27 +- src/prefect/settings/models/server/tasks.py | 11 +- src/prefect/settings/models/server/ui.py | 7 +- src/prefect/settings/models/tasks.py | 15 +- src/prefect/settings/models/testing.py | 7 +- src/prefect/settings/models/worker.py | 11 +- tests/conftest.py | 8 +- tests/test_settings.py | 295 ++++++++++++++---- 35 files changed, 619 insertions(+), 263 deletions(-) diff --git a/docs/v3/develop/settings-ref.mdx b/docs/v3/develop/settings-ref.mdx index 000ffb36f8b3..fd5520ff4174 100644 --- a/docs/v3/develop/settings-ref.mdx +++ b/docs/v3/develop/settings-ref.mdx @@ -804,7 +804,7 @@ The log level of the runner's webserver. **Type**: `string` -**Default**: `error` +**Default**: `ERROR` **Constraints**: - Allowed values: 'DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL' @@ -871,6 +871,58 @@ Number of seconds a runner should wait between heartbeats for flow runs. **TOML dotted key path**: `runner.server` +--- +## SQLAlchemySettings +Settings for controlling SQLAlchemy behavior; note that these settings only take effect when +using a PostgreSQL database. +### `pool_size` +Controls connection pool size of database connection pools from the Prefect backend. + +**Type**: `integer` + +**Default**: `5` + +**TOML dotted key path**: `server.database.sqlalchemy.pool_size` + +**Supported environment variables**: +`PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_SIZE`, `PREFECT_SQLALCHEMY_POOL_SIZE` + +### `pool_recycle` +This setting causes the pool to recycle connections after the given number of seconds has passed; set it to -1 to avoid recycling entirely. + +**Type**: `integer` + +**Default**: `3600` + +**TOML dotted key path**: `server.database.sqlalchemy.pool_recycle` + +**Supported environment variables**: +`PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_RECYCLE` + +### `pool_timeout` +Number of seconds to wait before giving up on getting a connection from the pool. Defaults to 30 seconds. + +**Type**: `number | None` + +**Default**: `30.0` + +**TOML dotted key path**: `server.database.sqlalchemy.pool_timeout` + +**Supported environment variables**: +`PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_TIMEOUT` + +### `max_overflow` +Controls maximum overflow of the connection pool. To prevent overflow, set to -1. + +**Type**: `integer` + +**Default**: `10` + +**TOML dotted key path**: `server.database.sqlalchemy.max_overflow` + +**Supported environment variables**: +`PREFECT_SERVER_DATABASE_SQLALCHEMY_MAX_OVERFLOW`, `PREFECT_SQLALCHEMY_MAX_OVERFLOW` + --- ## ServerAPISettings Settings for controlling API server behavior @@ -1039,6 +1091,13 @@ The default limit applied to queries that can return multiple objects, such as ` --- ## ServerDatabaseSettings Settings for controlling server database behavior +### `sqlalchemy` +Settings for controlling SQLAlchemy behavior + +**Type**: [SQLAlchemySettings](#sqlalchemysettings) + +**TOML dotted key path**: `server.database.sqlalchemy` + ### `connection_url` A database connection URL in a SQLAlchemy-compatible @@ -1159,7 +1218,7 @@ If `True`, the database will be migrated on application startup. `PREFECT_SERVER_DATABASE_MIGRATE_ON_START`, `PREFECT_API_DATABASE_MIGRATE_ON_START` ### `timeout` -A statement timeout, in seconds, applied to all database interactions made by the API. Defaults to 10 seconds. +A statement timeout, in seconds, applied to all database interactions made by the Prefect backend. Defaults to 10 seconds. **Type**: `number | None` @@ -1182,20 +1241,8 @@ A connection timeout, in seconds, applied to database connections. Defaults to ` **Supported environment variables**: `PREFECT_SERVER_DATABASE_CONNECTION_TIMEOUT`, `PREFECT_API_DATABASE_CONNECTION_TIMEOUT` -### `sqlalchemy_pool_size` -Controls connection pool size of database connection pools from the Prefect API. If not set, the default SQLAlchemy pool size will be used. - -**Type**: `integer | None` - -**Default**: `None` - -**TOML dotted key path**: `server.database.sqlalchemy_pool_size` - -**Supported environment variables**: -`PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_SIZE`, `PREFECT_SQLALCHEMY_POOL_SIZE` - ### `connection_app_name` -Controls the application_name field for connections opened from the connection pool when using a PostgreSQL database with the Prefect API. +Controls the application_name field for connections opened from the connection pool when using a PostgreSQL database with the Prefect backend. **Type**: `string | None` @@ -1206,18 +1253,6 @@ Controls the application_name field for connections opened from the connection p **Supported environment variables**: `PREFECT_SERVER_DATABASE_CONNECTION_APP_NAME` -### `sqlalchemy_max_overflow` -Controls maximum overflow of the connection pool when using a PostgreSQL database with the Prefect API. If not set, the default SQLAlchemy maximum overflow value will be used. - -**Type**: `integer | None` - -**Default**: `None` - -**TOML dotted key path**: `server.database.sqlalchemy_max_overflow` - -**Supported environment variables**: -`PREFECT_SERVER_DATABASE_SQLALCHEMY_MAX_OVERFLOW`, `PREFECT_SQLALCHEMY_MAX_OVERFLOW` - --- ## ServerDeploymentsSettings ### `concurrency_slot_wait_seconds` diff --git a/schemas/settings.schema.json b/schemas/settings.schema.json index 97d577258e16..e3f5db64419c 100644 --- a/schemas/settings.schema.json +++ b/schemas/settings.schema.json @@ -665,7 +665,7 @@ "type": "integer" }, "log_level": { - "default": "error", + "default": "ERROR", "description": "The log level of the runner's webserver.", "enum": [ "DEBUG", @@ -739,6 +739,58 @@ "title": "RunnerSettings", "type": "object" }, + "SQLAlchemySettings": { + "description": "Settings for controlling SQLAlchemy behavior; note that these settings only take effect when\nusing a PostgreSQL database.", + "properties": { + "pool_size": { + "default": 5, + "description": "Controls connection pool size of database connection pools from the Prefect backend.", + "supported_environment_variables": [ + "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_SIZE", + "PREFECT_SQLALCHEMY_POOL_SIZE" + ], + "title": "Pool Size", + "type": "integer" + }, + "pool_recycle": { + "default": 3600, + "description": "This setting causes the pool to recycle connections after the given number of seconds has passed; set it to -1 to avoid recycling entirely.", + "supported_environment_variables": [ + "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_RECYCLE" + ], + "title": "Pool Recycle", + "type": "integer" + }, + "pool_timeout": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "null" + } + ], + "default": 30.0, + "description": "Number of seconds to wait before giving up on getting a connection from the pool. Defaults to 30 seconds.", + "supported_environment_variables": [ + "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_TIMEOUT" + ], + "title": "Pool Timeout" + }, + "max_overflow": { + "default": 10, + "description": "Controls maximum overflow of the connection pool. To prevent overflow, set to -1.", + "supported_environment_variables": [ + "PREFECT_SERVER_DATABASE_SQLALCHEMY_MAX_OVERFLOW", + "PREFECT_SQLALCHEMY_MAX_OVERFLOW" + ], + "title": "Max Overflow", + "type": "integer" + } + }, + "title": "SQLAlchemySettings", + "type": "object" + }, "ServerAPISettings": { "description": "Settings for controlling API server behavior", "properties": { @@ -855,6 +907,11 @@ "ServerDatabaseSettings": { "description": "Settings for controlling server database behavior", "properties": { + "sqlalchemy": { + "$ref": "#/$defs/SQLAlchemySettings", + "description": "Settings for controlling SQLAlchemy behavior", + "supported_environment_variables": [] + }, "connection_url": { "anyOf": [ { @@ -1012,7 +1069,7 @@ } ], "default": 10.0, - "description": "A statement timeout, in seconds, applied to all database interactions made by the API. Defaults to 10 seconds.", + "description": "A statement timeout, in seconds, applied to all database interactions made by the Prefect backend. Defaults to 10 seconds.", "supported_environment_variables": [ "PREFECT_SERVER_DATABASE_TIMEOUT", "PREFECT_API_DATABASE_TIMEOUT" @@ -1036,23 +1093,6 @@ ], "title": "Connection Timeout" }, - "sqlalchemy_pool_size": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Controls connection pool size of database connection pools from the Prefect API. If not set, the default SQLAlchemy pool size will be used.", - "supported_environment_variables": [ - "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_SIZE", - "PREFECT_SQLALCHEMY_POOL_SIZE" - ], - "title": "Sqlalchemy Pool Size" - }, "connection_app_name": { "anyOf": [ { @@ -1063,28 +1103,11 @@ } ], "default": null, - "description": "Controls the application_name field for connections opened from the connection pool when using a PostgreSQL database with the Prefect API.", + "description": "Controls the application_name field for connections opened from the connection pool when using a PostgreSQL database with the Prefect backend.", "supported_environment_variables": [ "PREFECT_SERVER_DATABASE_CONNECTION_APP_NAME" ], "title": "Connection App Name" - }, - "sqlalchemy_max_overflow": { - "anyOf": [ - { - "type": "integer" - }, - { - "type": "null" - } - ], - "default": null, - "description": "Controls maximum overflow of the connection pool when using a PostgreSQL database with the Prefect API. If not set, the default SQLAlchemy maximum overflow value will be used.", - "supported_environment_variables": [ - "PREFECT_SERVER_DATABASE_SQLALCHEMY_MAX_OVERFLOW", - "PREFECT_SQLALCHEMY_MAX_OVERFLOW" - ], - "title": "Sqlalchemy Max Overflow" } }, "title": "ServerDatabaseSettings", diff --git a/setup.cfg b/setup.cfg index c19bd5f1d01b..f22b738e3aef 100644 --- a/setup.cfg +++ b/setup.cfg @@ -72,6 +72,7 @@ filterwarnings = ignore::pluggy.PluggyTeardownRaisedWarning + [mypy] plugins= pydantic.mypy diff --git a/src/integrations/prefect-kubernetes/prefect_kubernetes/settings.py b/src/integrations/prefect-kubernetes/prefect_kubernetes/settings.py index 4dbada798b57..c0852e1cae4b 100644 --- a/src/integrations/prefect-kubernetes/prefect_kubernetes/settings.py +++ b/src/integrations/prefect-kubernetes/prefect_kubernetes/settings.py @@ -2,11 +2,11 @@ from pydantic import AliasChoices, AliasPath, Field -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class KubernetesWorkerSettings(PrefectBaseSettings): - model_config = _build_settings_config(("integrations", "kubernetes", "worker")) + model_config = build_settings_config(("integrations", "kubernetes", "worker")) api_key_secret_name: Optional[str] = Field( default=None, @@ -35,7 +35,7 @@ class KubernetesWorkerSettings(PrefectBaseSettings): class KubernetesSettings(PrefectBaseSettings): - model_config = _build_settings_config(("integrations", "kubernetes")) + model_config = build_settings_config(("integrations", "kubernetes")) cluster_uid: Optional[str] = Field( default=None, diff --git a/src/integrations/prefect-redis/prefect_redis/client.py b/src/integrations/prefect-redis/prefect_redis/client.py index cd6a707825cd..803613bc7922 100644 --- a/src/integrations/prefect-redis/prefect_redis/client.py +++ b/src/integrations/prefect-redis/prefect_redis/client.py @@ -8,12 +8,12 @@ from prefect.settings.base import ( PrefectBaseSettings, - _build_settings_config, # type: ignore[reportPrivateUsage] + build_settings_config, # type: ignore[reportPrivateUsage] ) class RedisMessagingSettings(PrefectBaseSettings): - model_config = _build_settings_config( + model_config = build_settings_config( ( "redis", "messaging", diff --git a/src/prefect/server/database/configurations.py b/src/prefect/server/database/configurations.py index c09caec7fc28..b2055b92541b 100644 --- a/src/prefect/server/database/configurations.py +++ b/src/prefect/server/database/configurations.py @@ -28,8 +28,8 @@ PREFECT_API_DATABASE_TIMEOUT, PREFECT_SERVER_DATABASE_CONNECTION_APP_NAME, PREFECT_SQLALCHEMY_MAX_OVERFLOW, - PREFECT_SQLALCHEMY_POOL_SIZE, PREFECT_TESTING_UNIT_TEST_MODE, + get_current_settings, ) from prefect.utilities.asyncutils import add_event_loop_shutdown_callback @@ -131,7 +131,8 @@ def __init__( connection_timeout or PREFECT_API_DATABASE_CONNECTION_TIMEOUT.value() ) self.sqlalchemy_pool_size: Optional[int] = ( - sqlalchemy_pool_size or PREFECT_SQLALCHEMY_POOL_SIZE.value() + sqlalchemy_pool_size + or get_current_settings().server.database.sqlalchemy.pool_size ) self.sqlalchemy_max_overflow: Optional[int] = ( sqlalchemy_max_overflow or PREFECT_SQLALCHEMY_MAX_OVERFLOW.value() @@ -205,8 +206,11 @@ async def engine(self) -> AsyncEngine: self.timeout, ) if cache_key not in ENGINES: - # apply database timeout - kwargs: dict[str, Any] = dict() + kwargs: dict[ + str, Any + ] = get_current_settings().server.database.sqlalchemy.model_dump( + mode="json" + ) connect_args: dict[str, Any] = dict() if self.timeout is not None: @@ -337,7 +341,7 @@ async def engine(self) -> AsyncEngine: f"{sqlite3.sqlite_version}" ) - kwargs: dict[str, Any] = {} + kwargs: dict[str, Any] = dict() loop = get_running_loop() @@ -347,12 +351,6 @@ async def engine(self) -> AsyncEngine: if self.timeout is not None: kwargs["connect_args"] = dict(timeout=self.timeout) - if self.sqlalchemy_pool_size is not None: - kwargs["pool_size"] = self.sqlalchemy_pool_size - - if self.sqlalchemy_max_overflow is not None: - kwargs["max_overflow"] = self.sqlalchemy_max_overflow - # use `named` paramstyle for sqlite instead of `qmark` in very rare # circumstances, we've seen aiosqlite pass parameters in the wrong # order; by using named parameters we avoid this issue diff --git a/src/prefect/settings/base.py b/src/prefect/settings/base.py index 47acc1aa358b..6d3ff587e4ca 100644 --- a/src/prefect/settings/base.py +++ b/src/prefect/settings/base.py @@ -196,7 +196,7 @@ def _add_environment_variables( env_vars.append(f"{model.model_config.get('env_prefix')}{property.upper()}") -def _build_settings_config( # pyright: ignore[reportUnusedFunction] This is used elsewhere. TODO: update to be a public function because it is used in integration libraries. +def build_settings_config( path: Tuple[str, ...] = tuple(), frozen: bool = False ) -> PrefectSettingsConfigDict: env_prefix = f"PREFECT_{'_'.join(path).upper()}_" if path else "PREFECT_" @@ -212,6 +212,9 @@ def _build_settings_config( # pyright: ignore[reportUnusedFunction] This is use ) +_build_settings_config = build_settings_config # noqa # TODO: remove once all usage updated + + def _to_environment_variable_value( value: list[object] | set[object] | tuple[object] | Any, ) -> str: diff --git a/src/prefect/settings/legacy.py b/src/prefect/settings/legacy.py index 6026801dc3aa..821d6971dd85 100644 --- a/src/prefect/settings/legacy.py +++ b/src/prefect/settings/legacy.py @@ -123,7 +123,7 @@ def _get_settings_fields( settings: Type[BaseSettings], accessor_prefix: Optional[str] = None ) -> Dict[str, "Setting"]: """Get the settings fields for the settings object""" - settings_fields: Dict[str, Setting] = {} + settings_fields: dict[str, Setting] = {} for field_name, field in settings.model_fields.items(): if inspect.isclass(field.annotation) and issubclass( field.annotation, PrefectBaseSettings diff --git a/src/prefect/settings/models/api.py b/src/prefect/settings/models/api.py index a8589ef00b06..1c71b35b924d 100644 --- a/src/prefect/settings/models/api.py +++ b/src/prefect/settings/models/api.py @@ -1,11 +1,12 @@ import os from typing import ClassVar, Optional -from pydantic import ConfigDict, Field, SecretStr +from pydantic import Field, SecretStr +from pydantic_settings import SettingsConfigDict from prefect.settings.base import ( PrefectBaseSettings, - _build_settings_config, + build_settings_config, ) @@ -14,7 +15,7 @@ class APISettings(PrefectBaseSettings): Settings for interacting with the Prefect API """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("api",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("api",)) url: Optional[str] = Field( default=None, description="The URL of the Prefect API. If not set, the client will attempt to infer it.", diff --git a/src/prefect/settings/models/cli.py b/src/prefect/settings/models/cli.py index 187f93ecf0a0..c7c41a1a4ff0 100644 --- a/src/prefect/settings/models/cli.py +++ b/src/prefect/settings/models/cli.py @@ -1,10 +1,11 @@ from typing import ClassVar, Optional -from pydantic import ConfigDict, Field +from pydantic import Field +from pydantic_settings import SettingsConfigDict from prefect.settings.base import ( PrefectBaseSettings, - _build_settings_config, + build_settings_config, ) @@ -13,7 +14,7 @@ class CLISettings(PrefectBaseSettings): Settings for controlling CLI behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("cli",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("cli",)) colors: bool = Field( default=True, diff --git a/src/prefect/settings/models/client.py b/src/prefect/settings/models/client.py index 2a705d186aa0..60166ed8498e 100644 --- a/src/prefect/settings/models/client.py +++ b/src/prefect/settings/models/client.py @@ -1,10 +1,11 @@ from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict from prefect.settings.base import ( PrefectBaseSettings, - _build_settings_config, + build_settings_config, ) from prefect.types import ClientRetryExtraCodes @@ -14,7 +15,9 @@ class ClientMetricsSettings(PrefectBaseSettings): Settings for controlling metrics reporting from the client """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("client", "metrics")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("client", "metrics") + ) enabled: bool = Field( default=False, @@ -39,7 +42,7 @@ class ClientSettings(PrefectBaseSettings): Settings for controlling API client behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("client",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("client",)) max_retries: int = Field( default=5, diff --git a/src/prefect/settings/models/cloud.py b/src/prefect/settings/models/cloud.py index ab65dda4e5ec..268adf9de560 100644 --- a/src/prefect/settings/models/cloud.py +++ b/src/prefect/settings/models/cloud.py @@ -1,12 +1,13 @@ import re from typing import ClassVar, Optional -from pydantic import ConfigDict, Field, model_validator +from pydantic import Field, model_validator +from pydantic_settings import SettingsConfigDict from typing_extensions import Self from prefect.settings.base import ( PrefectBaseSettings, - _build_settings_config, + build_settings_config, ) @@ -32,7 +33,7 @@ class CloudSettings(PrefectBaseSettings): Settings for interacting with Prefect Cloud """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("cloud",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("cloud",)) api_url: str = Field( default="https://api.prefect.cloud/api", diff --git a/src/prefect/settings/models/deployments.py b/src/prefect/settings/models/deployments.py index 0795fb045bbe..18fb8e0ed9ad 100644 --- a/src/prefect/settings/models/deployments.py +++ b/src/prefect/settings/models/deployments.py @@ -1,8 +1,9 @@ from typing import ClassVar, Optional -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class DeploymentsSettings(PrefectBaseSettings): @@ -10,7 +11,7 @@ class DeploymentsSettings(PrefectBaseSettings): Settings for configuring deployments defaults """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("deployments",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("deployments",)) default_work_pool_name: Optional[str] = Field( default=None, diff --git a/src/prefect/settings/models/experiments.py b/src/prefect/settings/models/experiments.py index 2dae9554671a..df235c8b84bb 100644 --- a/src/prefect/settings/models/experiments.py +++ b/src/prefect/settings/models/experiments.py @@ -1,8 +1,9 @@ from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ExperimentsSettings(PrefectBaseSettings): @@ -10,7 +11,7 @@ class ExperimentsSettings(PrefectBaseSettings): Settings for configuring experimental features """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("experiments",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("experiments",)) warn: bool = Field( default=True, diff --git a/src/prefect/settings/models/flows.py b/src/prefect/settings/models/flows.py index 66736e9fdf25..dfcb917ea2c3 100644 --- a/src/prefect/settings/models/flows.py +++ b/src/prefect/settings/models/flows.py @@ -1,8 +1,9 @@ from typing import ClassVar, Union -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class FlowsSettings(PrefectBaseSettings): @@ -10,7 +11,7 @@ class FlowsSettings(PrefectBaseSettings): Settings for controlling flow behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("flows",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("flows",)) default_retries: int = Field( default=0, diff --git a/src/prefect/settings/models/internal.py b/src/prefect/settings/models/internal.py index ed0ca8c7b096..f51fc9079e0d 100644 --- a/src/prefect/settings/models/internal.py +++ b/src/prefect/settings/models/internal.py @@ -1,13 +1,14 @@ from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config from prefect.types import LogLevel class InternalSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("internal",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("internal",)) logging_level: LogLevel = Field( default="ERROR", diff --git a/src/prefect/settings/models/logging.py b/src/prefect/settings/models/logging.py index 643e927a01ae..165228c54f89 100644 --- a/src/prefect/settings/models/logging.py +++ b/src/prefect/settings/models/logging.py @@ -6,13 +6,13 @@ AliasChoices, AliasPath, BeforeValidator, - ConfigDict, Field, model_validator, ) +from pydantic_settings import SettingsConfigDict from typing_extensions import Self -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config from prefect.types import LogLevel, validate_set_T_from_delim_string @@ -33,7 +33,9 @@ class LoggingToAPISettings(PrefectBaseSettings): Settings for controlling logging to the API """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("logging", "to_api")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("logging", "to_api") + ) enabled: bool = Field( default=True, @@ -86,7 +88,7 @@ class LoggingSettings(PrefectBaseSettings): Settings for controlling logging behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("logging",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("logging",)) level: LogLevel = Field( default="INFO", diff --git a/src/prefect/settings/models/results.py b/src/prefect/settings/models/results.py index aaab632e8fbe..0f6093ee53a0 100644 --- a/src/prefect/settings/models/results.py +++ b/src/prefect/settings/models/results.py @@ -1,9 +1,10 @@ from pathlib import Path from typing import ClassVar, Optional -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ResultsSettings(PrefectBaseSettings): @@ -11,7 +12,7 @@ class ResultsSettings(PrefectBaseSettings): Settings for controlling result storage behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("results",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("results",)) default_serializer: str = Field( default="pickle", diff --git a/src/prefect/settings/models/root.py b/src/prefect/settings/models/root.py index 896977fda788..ef59160259af 100644 --- a/src/prefect/settings/models/root.py +++ b/src/prefect/settings/models/root.py @@ -11,10 +11,11 @@ ) from urllib.parse import urlparse -from pydantic import BeforeValidator, ConfigDict, Field, SecretStr, model_validator +from pydantic import BeforeValidator, Field, SecretStr, model_validator +from pydantic_settings import SettingsConfigDict from typing_extensions import Self -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config from prefect.settings.models.tasks import TasksSettings from prefect.settings.models.testing import TestingSettings from prefect.settings.models.worker import WorkerSettings @@ -44,7 +45,7 @@ class Settings(PrefectBaseSettings): See https://docs.pydantic.dev/latest/concepts/pydantic_settings """ - model_config: ClassVar[ConfigDict] = _build_settings_config() + model_config: ClassVar[SettingsConfigDict] = build_settings_config() home: Annotated[Path, BeforeValidator(lambda x: Path(x).expanduser())] = Field( default=Path("~") / ".prefect", @@ -271,7 +272,7 @@ def copy_with_update( # To restore defaults, we need to resolve the setting path and then # set the default value on the new settings object. When restoring # defaults, all settings sources will be ignored. - restore_defaults_obj = {} + restore_defaults_obj: dict[str, Any] = {} for r in restore_defaults or []: path = r.accessor.split(".") model = self @@ -297,11 +298,11 @@ def copy_with_update( updates = updates or {} set_defaults = set_defaults or {} - set_defaults_obj = {} + set_defaults_obj: dict[str, Any] = {} for setting, value in set_defaults.items(): set_in_dict(set_defaults_obj, setting.accessor, value) - updates_obj = {} + updates_obj: dict[str, Any] = {} for setting, value in updates.items(): set_in_dict(updates_obj, setting.accessor, value) @@ -373,7 +374,7 @@ def _warn_on_misconfigured_api_url(settings: "Settings"): "`PREFECT_API_URL` uses `/workspace/` but should use `/workspaces/`." ), } - warnings_list = [] + warnings_list: list[str] = [] for misconfig, warning in misconfigured_mappings.items(): if misconfig in api_url: diff --git a/src/prefect/settings/models/runner.py b/src/prefect/settings/models/runner.py index bbf8525ac4d3..2596914c1a6c 100644 --- a/src/prefect/settings/models/runner.py +++ b/src/prefect/settings/models/runner.py @@ -1,8 +1,9 @@ from typing import ClassVar, Optional -from pydantic import ConfigDict, Field +from pydantic import Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config from prefect.types import LogLevel @@ -11,7 +12,9 @@ class RunnerServerSettings(PrefectBaseSettings): Settings for controlling runner server behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("runner", "server")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("runner", "server") + ) enable: bool = Field( default=False, @@ -29,7 +32,7 @@ class RunnerServerSettings(PrefectBaseSettings): ) log_level: LogLevel = Field( - default="error", + default="ERROR", description="The log level of the runner's webserver.", ) @@ -44,7 +47,7 @@ class RunnerSettings(PrefectBaseSettings): Settings for controlling runner behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("runner",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("runner",)) process_limit: int = Field( default=5, diff --git a/src/prefect/settings/models/server/api.py b/src/prefect/settings/models/server/api.py index 691d40309681..be173565606c 100644 --- a/src/prefect/settings/models/server/api.py +++ b/src/prefect/settings/models/server/api.py @@ -1,9 +1,10 @@ from datetime import timedelta from typing import ClassVar, Optional -from pydantic import AliasChoices, AliasPath, ConfigDict, Field, SecretStr +from pydantic import AliasChoices, AliasPath, Field, SecretStr +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerAPISettings(PrefectBaseSettings): @@ -11,7 +12,9 @@ class ServerAPISettings(PrefectBaseSettings): Settings for controlling API server behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "api")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "api") + ) auth_string: Optional[SecretStr] = Field( default=None, diff --git a/src/prefect/settings/models/server/database.py b/src/prefect/settings/models/server/database.py index 255bf92389a5..b851a1db7b51 100644 --- a/src/prefect/settings/models/server/database.py +++ b/src/prefect/settings/models/server/database.py @@ -1,18 +1,59 @@ import warnings -from typing import ClassVar, Optional +from typing import Any, ClassVar, Optional from urllib.parse import quote_plus from pydantic import ( AliasChoices, AliasPath, - ConfigDict, Field, SecretStr, model_validator, ) +from pydantic_settings import SettingsConfigDict from typing_extensions import Literal, Self -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config + + +class SQLAlchemySettings(PrefectBaseSettings): + """ + Settings for controlling SQLAlchemy behavior; note that these settings only take effect when + using a PostgreSQL database. + """ + + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "database", "sqlalchemy") + ) + + pool_size: int = Field( + default=5, + description="Controls connection pool size of database connection pools from the Prefect backend.", + validation_alias=AliasChoices( + AliasPath("pool_size"), + "prefect_server_database_sqlalchemy_pool_size", + "prefect_sqlalchemy_pool_size", + ), + ) + + pool_recycle: int = Field( + default=3600, + description="This setting causes the pool to recycle connections after the given number of seconds has passed; set it to -1 to avoid recycling entirely.", + ) + + pool_timeout: Optional[float] = Field( + default=30.0, + description="Number of seconds to wait before giving up on getting a connection from the pool. Defaults to 30 seconds.", + ) + + max_overflow: int = Field( + default=10, + description="Controls maximum overflow of the connection pool. To prevent overflow, set to -1.", + validation_alias=AliasChoices( + AliasPath("max_overflow"), + "prefect_server_database_sqlalchemy_max_overflow", + "prefect_sqlalchemy_max_overflow", + ), + ) class ServerDatabaseSettings(PrefectBaseSettings): @@ -20,7 +61,14 @@ class ServerDatabaseSettings(PrefectBaseSettings): Settings for controlling server database behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "database")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "database") + ) + + sqlalchemy: SQLAlchemySettings = Field( + default_factory=SQLAlchemySettings, + description="Settings for controlling SQLAlchemy behavior", + ) connection_url: Optional[SecretStr] = Field( default=None, @@ -128,7 +176,7 @@ class ServerDatabaseSettings(PrefectBaseSettings): timeout: Optional[float] = Field( default=10.0, - description="A statement timeout, in seconds, applied to all database interactions made by the API. Defaults to 10 seconds.", + description="A statement timeout, in seconds, applied to all database interactions made by the Prefect backend. Defaults to 10 seconds.", validation_alias=AliasChoices( AliasPath("timeout"), "prefect_server_database_timeout", @@ -146,30 +194,54 @@ class ServerDatabaseSettings(PrefectBaseSettings): ), ) - sqlalchemy_pool_size: Optional[int] = Field( - default=None, - description="Controls connection pool size of database connection pools from the Prefect API. If not set, the default SQLAlchemy pool size will be used.", - validation_alias=AliasChoices( - AliasPath("sqlalchemy_pool_size"), - "prefect_server_database_sqlalchemy_pool_size", - "prefect_sqlalchemy_pool_size", - ), - ) - connection_app_name: Optional[str] = Field( default=None, - description="Controls the application_name field for connections opened from the connection pool when using a PostgreSQL database with the Prefect API.", + description="Controls the application_name field for connections opened from the connection pool when using a PostgreSQL database with the Prefect backend.", ) - sqlalchemy_max_overflow: Optional[int] = Field( - default=None, - description="Controls maximum overflow of the connection pool when using a PostgreSQL database with the Prefect API. If not set, the default SQLAlchemy maximum overflow value will be used.", - validation_alias=AliasChoices( - AliasPath("sqlalchemy_max_overflow"), - "prefect_server_database_sqlalchemy_max_overflow", - "prefect_sqlalchemy_max_overflow", - ), - ) + # handle deprecated fields + + def __getattribute__(self, name: str) -> Any: + if name in ["sqlalchemy_pool_size", "sqlalchemy_max_overflow"]: + warnings.warn( + f"Setting {name} has been moved to the `sqlalchemy` settings group.", + DeprecationWarning, + ) + field_name = name.replace("sqlalchemy_", "") + return getattr(super().__getattribute__("sqlalchemy"), field_name) + return super().__getattribute__(name) + + # validators + + @model_validator(mode="before") + @classmethod + def set_deprecated_sqlalchemy_settings_on_child_model_and_warn( + cls, values: dict[str, Any] + ) -> dict[str, Any]: + """ + Set deprecated settings on the child model. + """ + # Initialize sqlalchemy settings if not present + if "sqlalchemy" not in values: + values["sqlalchemy"] = SQLAlchemySettings() + + if "sqlalchemy_pool_size" in values: + warnings.warn( + "`sqlalchemy_pool_size` has been moved to the `sqlalchemy` settings group as `pool_size`.", + DeprecationWarning, + ) + if "pool_size" not in values["sqlalchemy"].model_fields_set: + values["sqlalchemy"].pool_size = values["sqlalchemy_pool_size"] + + if "sqlalchemy_max_overflow" in values: + warnings.warn( + "`sqlalchemy_max_overflow` has been moved to the `sqlalchemy` settings group as `max_overflow`.", + DeprecationWarning, + ) + if "max_overflow" not in values["sqlalchemy"].model_fields_set: + values["sqlalchemy"].max_overflow = values["sqlalchemy_max_overflow"] + + return values @model_validator(mode="after") def emit_warnings(self) -> Self: # noqa: F821 @@ -208,4 +280,4 @@ def warn_on_database_password_value_without_usage( "PREFECT_SERVER_DATABASE_CONNECTION_URL. " "The provided password will be ignored." ) - return settings + return None diff --git a/src/prefect/settings/models/server/deployments.py b/src/prefect/settings/models/server/deployments.py index dec42a4b898e..ce5eec9a2402 100644 --- a/src/prefect/settings/models/server/deployments.py +++ b/src/prefect/settings/models/server/deployments.py @@ -1,12 +1,13 @@ from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerDeploymentsSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "deployments") ) diff --git a/src/prefect/settings/models/server/ephemeral.py b/src/prefect/settings/models/server/ephemeral.py index e8ca6201f4d9..c8dd97da2438 100644 --- a/src/prefect/settings/models/server/ephemeral.py +++ b/src/prefect/settings/models/server/ephemeral.py @@ -1,8 +1,9 @@ from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerEphemeralSettings(PrefectBaseSettings): @@ -10,7 +11,9 @@ class ServerEphemeralSettings(PrefectBaseSettings): Settings for controlling ephemeral server behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "ephemeral")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "ephemeral") + ) enabled: bool = Field( default=False, diff --git a/src/prefect/settings/models/server/events.py b/src/prefect/settings/models/server/events.py index f91dd414d391..f50f10dd2f20 100644 --- a/src/prefect/settings/models/server/events.py +++ b/src/prefect/settings/models/server/events.py @@ -1,9 +1,10 @@ from datetime import timedelta from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerEventsSettings(PrefectBaseSettings): @@ -11,7 +12,9 @@ class ServerEventsSettings(PrefectBaseSettings): Settings for controlling behavior of the events subsystem """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "events")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "events") + ) ########################################################################### # Events settings diff --git a/src/prefect/settings/models/server/flow_run_graph.py b/src/prefect/settings/models/server/flow_run_graph.py index 879f17d86f84..e7c76a654228 100644 --- a/src/prefect/settings/models/server/flow_run_graph.py +++ b/src/prefect/settings/models/server/flow_run_graph.py @@ -1,8 +1,9 @@ from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerFlowRunGraphSettings(PrefectBaseSettings): @@ -10,7 +11,7 @@ class ServerFlowRunGraphSettings(PrefectBaseSettings): Settings for controlling behavior of the flow run graph """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "flow_run_graph") ) diff --git a/src/prefect/settings/models/server/root.py b/src/prefect/settings/models/server/root.py index 9d7c3d33f6dc..fcc4f9a416d5 100644 --- a/src/prefect/settings/models/server/root.py +++ b/src/prefect/settings/models/server/root.py @@ -1,9 +1,10 @@ from pathlib import Path from typing import ClassVar, Optional -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config from prefect.types import LogLevel from .api import ServerAPISettings @@ -22,7 +23,7 @@ class ServerSettings(PrefectBaseSettings): Settings for controlling server behavior """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("server",)) logging_level: LogLevel = Field( default="WARNING", diff --git a/src/prefect/settings/models/server/services.py b/src/prefect/settings/models/server/services.py index 1e7face210c1..5c0173de8cfd 100644 --- a/src/prefect/settings/models/server/services.py +++ b/src/prefect/settings/models/server/services.py @@ -1,9 +1,10 @@ from datetime import timedelta from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerServicesCancellationCleanupSettings(PrefectBaseSettings): @@ -11,7 +12,7 @@ class ServerServicesCancellationCleanupSettings(PrefectBaseSettings): Settings for controlling the cancellation cleanup service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "cancellation_cleanup") ) @@ -41,7 +42,7 @@ class ServerServicesEventPersisterSettings(PrefectBaseSettings): Settings for controlling the event persister service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "event_persister") ) @@ -83,7 +84,7 @@ class ServerServicesFlowRunNotificationsSettings(PrefectBaseSettings): Settings for controlling the flow run notifications service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "flow_run_notifications") ) @@ -103,7 +104,7 @@ class ServerServicesForemanSettings(PrefectBaseSettings): Settings for controlling the foreman service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "foreman") ) @@ -184,7 +185,7 @@ class ServerServicesLateRunsSettings(PrefectBaseSettings): Settings for controlling the late runs service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "late_runs") ) @@ -228,7 +229,7 @@ class ServerServicesSchedulerSettings(PrefectBaseSettings): Settings for controlling the scheduler service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "scheduler") ) @@ -353,7 +354,7 @@ class ServerServicesPauseExpirationsSettings(PrefectBaseSettings): Settings for controlling the pause expiration service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "pause_expirations") ) @@ -389,7 +390,7 @@ class ServerServicesTaskRunRecorderSettings(PrefectBaseSettings): Settings for controlling the task run recorder service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "task_run_recorder") ) @@ -409,7 +410,7 @@ class ServerServicesTriggersSettings(PrefectBaseSettings): Settings for controlling the triggers service """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "services", "triggers") ) @@ -429,7 +430,9 @@ class ServerServicesSettings(PrefectBaseSettings): Settings for controlling server services """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "services")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "services") + ) cancellation_cleanup: ServerServicesCancellationCleanupSettings = Field( default_factory=ServerServicesCancellationCleanupSettings, diff --git a/src/prefect/settings/models/server/tasks.py b/src/prefect/settings/models/server/tasks.py index f838b1853406..c0ec5cae1e61 100644 --- a/src/prefect/settings/models/server/tasks.py +++ b/src/prefect/settings/models/server/tasks.py @@ -1,9 +1,10 @@ from datetime import timedelta from typing import ClassVar -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerTasksSchedulingSettings(PrefectBaseSettings): @@ -11,7 +12,7 @@ class ServerTasksSchedulingSettings(PrefectBaseSettings): Settings for controlling server-side behavior related to task scheduling """ - model_config: ClassVar[ConfigDict] = _build_settings_config( + model_config: ClassVar[SettingsConfigDict] = build_settings_config( ("server", "tasks", "scheduling") ) @@ -51,7 +52,9 @@ class ServerTasksSettings(PrefectBaseSettings): Settings for controlling server-side behavior related to tasks """ - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "tasks")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("server", "tasks") + ) tag_concurrency_slot_wait_seconds: float = Field( default=30, diff --git a/src/prefect/settings/models/server/ui.py b/src/prefect/settings/models/server/ui.py index acdc63de63e6..f76bdb3bb020 100644 --- a/src/prefect/settings/models/server/ui.py +++ b/src/prefect/settings/models/server/ui.py @@ -1,12 +1,13 @@ from typing import ClassVar, Optional -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class ServerUISettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("server", "ui")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("server", "ui")) enabled: bool = Field( default=True, diff --git a/src/prefect/settings/models/tasks.py b/src/prefect/settings/models/tasks.py index d9010f7267af..0aad05ac2459 100644 --- a/src/prefect/settings/models/tasks.py +++ b/src/prefect/settings/models/tasks.py @@ -1,12 +1,15 @@ from typing import ClassVar, Optional, Union -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class TasksRunnerSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("tasks", "runner")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("tasks", "runner") + ) thread_pool_max_workers: Optional[int] = Field( default=None, @@ -21,7 +24,9 @@ class TasksRunnerSettings(PrefectBaseSettings): class TasksSchedulingSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("tasks", "scheduling")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("tasks", "scheduling") + ) default_storage_block: Optional[str] = Field( default=None, @@ -45,7 +50,7 @@ class TasksSchedulingSettings(PrefectBaseSettings): class TasksSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("tasks",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("tasks",)) refresh_cache: bool = Field( default=False, diff --git a/src/prefect/settings/models/testing.py b/src/prefect/settings/models/testing.py index 03cf61b5eb9c..7e54dacad4d8 100644 --- a/src/prefect/settings/models/testing.py +++ b/src/prefect/settings/models/testing.py @@ -1,12 +1,13 @@ from typing import Any, ClassVar, Optional -from pydantic import AliasChoices, AliasPath, ConfigDict, Field +from pydantic import AliasChoices, AliasPath, Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class TestingSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("testing",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("testing",)) test_mode: bool = Field( default=False, diff --git a/src/prefect/settings/models/worker.py b/src/prefect/settings/models/worker.py index 03f1396aaa2b..61741a3cdc98 100644 --- a/src/prefect/settings/models/worker.py +++ b/src/prefect/settings/models/worker.py @@ -1,12 +1,15 @@ from typing import ClassVar -from pydantic import ConfigDict, Field +from pydantic import Field +from pydantic_settings import SettingsConfigDict -from prefect.settings.base import PrefectBaseSettings, _build_settings_config +from prefect.settings.base import PrefectBaseSettings, build_settings_config class WorkerWebserverSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("worker", "webserver")) + model_config: ClassVar[SettingsConfigDict] = build_settings_config( + ("worker", "webserver") + ) host: str = Field( default="0.0.0.0", @@ -20,7 +23,7 @@ class WorkerWebserverSettings(PrefectBaseSettings): class WorkerSettings(PrefectBaseSettings): - model_config: ClassVar[ConfigDict] = _build_settings_config(("worker",)) + model_config: ClassVar[SettingsConfigDict] = build_settings_config(("worker",)) heartbeat_seconds: float = Field( default=30, diff --git a/tests/conftest.py b/tests/conftest.py index 425456095275..38e175e856a2 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -477,7 +477,9 @@ async def generate_test_database_connection_url( @pytest.fixture(scope="session", autouse=True) -def test_database_connection_url(generate_test_database_connection_url): +def test_database_connection_url( + generate_test_database_connection_url: Optional[str], +) -> Generator[Optional[str], None, None]: """ Update the setting for the database connection url to the generated value from `generate_test_database_connection_url` @@ -489,7 +491,9 @@ def test_database_connection_url(generate_test_database_connection_url): if url is None: yield None else: - with temporary_settings({PREFECT_SERVER_DATABASE_CONNECTION_URL: url}): + with temporary_settings( + updates={PREFECT_SERVER_DATABASE_CONNECTION_URL: url}, + ): yield url diff --git a/tests/test_settings.py b/tests/test_settings.py index ba925186bc2d..9e23cf67e8b8 100644 --- a/tests/test_settings.py +++ b/tests/test_settings.py @@ -1,9 +1,12 @@ +from __future__ import annotations + import copy import os import textwrap import warnings from datetime import timedelta from pathlib import Path +from typing import Any, Callable, Generator import pydantic import pytest @@ -54,6 +57,7 @@ from prefect.settings.base import _to_environment_variable_value from prefect.settings.constants import DEFAULT_PROFILES_PATH from prefect.settings.legacy import ( + Setting, _env_var_to_accessor, _get_settings_fields, _get_valid_setting_names, @@ -63,6 +67,10 @@ from prefect.settings.models.logging import LoggingSettings from prefect.settings.models.server import ServerSettings from prefect.settings.models.server.api import ServerAPISettings +from prefect.settings.models.server.database import ( + ServerDatabaseSettings, + SQLAlchemySettings, +) from prefect.utilities.collections import get_from_dict, set_in_dict from prefect.utilities.filesystem import tmpchdir @@ -308,7 +316,9 @@ "PREFECT_SERVER_DATABASE_PASSWORD": {"test_value": "password"}, "PREFECT_SERVER_DATABASE_PORT": {"test_value": 5432}, "PREFECT_SERVER_DATABASE_SQLALCHEMY_MAX_OVERFLOW": {"test_value": 10}, + "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_RECYCLE": {"test_value": 10}, "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_SIZE": {"test_value": 10}, + "PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_TIMEOUT": {"test_value": 10.0}, "PREFECT_SERVER_DATABASE_TIMEOUT": {"test_value": 10.0}, "PREFECT_SERVER_DATABASE_USER": {"test_value": "user"}, "PREFECT_SERVER_DEPLOYMENTS_CONCURRENCY_SLOT_WAIT_SECONDS": {"test_value": 10.0}, @@ -450,11 +460,11 @@ @pytest.fixture -def temporary_env_file(tmp_path): - with tmpchdir(tmp_path): +def temporary_env_file(tmp_path: Path) -> Generator[Callable[[str], None], None, None]: + with tmpchdir(str(tmp_path)): env_file = Path(".env") - def _create_temp_env(content): + def _create_temp_env(content: str) -> None: env_file.write_text(content) yield _create_temp_env @@ -464,14 +474,14 @@ def _create_temp_env(content): @pytest.fixture -def temporary_toml_file(tmp_path): - with tmpchdir(tmp_path): +def temporary_toml_file(tmp_path: Path) -> Generator[Callable[[str], None], None, None]: + with tmpchdir(str(tmp_path)): toml_file = Path("prefect.toml") - def _create_temp_toml(content, path=toml_file): + def _create_temp_toml(content: str, path: Path = toml_file) -> None: nonlocal toml_file with path.open("w") as f: - toml.dump(content, f) + toml.dump(content, f) # type: ignore toml_file = path # update toml_file in case path was changed yield _create_temp_toml @@ -543,7 +553,9 @@ def test_settings_copy_with_update(self): ), "Changed, existing value was default" assert new_settings.client.retry_extra_codes == {400, 500} - def test_settings_copy_with_update_restore_defaults(self, monkeypatch): + def test_settings_copy_with_update_restore_defaults( + self, monkeypatch: pytest.MonkeyPatch + ): monkeypatch.setenv("PREFECT_TESTING_TEST_SETTING", "Not the default") settings = Settings() assert settings.testing.test_setting == "Not the default" @@ -552,16 +564,17 @@ def test_settings_copy_with_update_restore_defaults(self, monkeypatch): ) assert new_settings.testing.test_setting == "FOO" - def test_settings_loads_environment_variables_at_instantiation(self, monkeypatch): + def test_settings_loads_environment_variables_at_instantiation( + self, monkeypatch: pytest.MonkeyPatch + ): assert PREFECT_TEST_MODE.value() is True monkeypatch.setenv("PREFECT_TESTING_TEST_MODE", "0") new_settings = Settings() assert PREFECT_TEST_MODE.value_from(new_settings) is False - def test_settings_to_environment_includes_all_settings_with_non_null_values( - self, disable_hosted_api_server - ): + @pytest.mark.usefixtures("disable_hosted_api_server") + def test_settings_to_environment_includes_all_settings_with_non_null_values(self): settings = Settings() expected_names = { s.name @@ -622,7 +635,11 @@ def test_settings_to_environment_casts_to_strings(self): ) @pytest.mark.parametrize("exclude_unset", [True, False]) - def test_settings_to_environment_roundtrip(self, exclude_unset, monkeypatch): + def test_settings_to_environment_roundtrip( + self, + exclude_unset: bool, + monkeypatch: pytest.MonkeyPatch, + ): settings = Settings() variables = settings.to_environment_variables(exclude_unset=exclude_unset) for key, value in variables.items(): @@ -646,12 +663,16 @@ def test_settings_hash_key(self): PREFECT_LOGGING_SERVER_LEVEL, ], ) - def test_settings_validates_log_levels(self, log_level_setting, monkeypatch): + def test_settings_validates_log_levels( + self, + log_level_setting: Setting, + monkeypatch: pytest.MonkeyPatch, + ): with pytest.raises( pydantic.ValidationError, match="should be 'DEBUG', 'INFO', 'WARNING', 'ERROR' or 'CRITICAL'", ): - kwargs = {} + kwargs: dict[str, Any] = {} set_in_dict(kwargs, log_level_setting.accessor, "FOOBAR") Settings(**kwargs) @@ -662,7 +683,10 @@ def test_settings_validates_log_levels(self, log_level_setting, monkeypatch): PREFECT_SERVER_LOGGING_LEVEL, ], ) - def test_settings_uppercases_log_levels(self, log_level_setting): + def test_settings_uppercases_log_levels( + self, + log_level_setting: Setting, + ): with temporary_settings({log_level_setting: "debug"}): assert log_level_setting.value() == "DEBUG" @@ -695,7 +719,9 @@ def test_include_secrets(self): == "test" ) - def test_loads_when_profile_path_does_not_exist(self, monkeypatch): + def test_loads_when_profile_path_does_not_exist( + self, monkeypatch: pytest.MonkeyPatch + ): monkeypatch.setenv("PREFECT_PROFILES_PATH", str(Path.home() / "nonexistent")) monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) @@ -703,7 +729,9 @@ def test_loads_when_profile_path_does_not_exist(self, monkeypatch): # Should default to ephemeral profile assert Settings().server.ephemeral.enabled - def test_loads_when_profile_path_is_not_a_toml_file(self, monkeypatch, tmp_path): + def test_loads_when_profile_path_is_not_a_toml_file( + self, monkeypatch: pytest.MonkeyPatch, tmp_path: Path + ): monkeypatch.setenv("PREFECT_PROFILES_PATH", str(tmp_path / "profiles.toml")) monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) @@ -779,7 +807,7 @@ def test_settings_in_truthy_statements_use_value(self): "python_list", ], ) - def test_extra_loggers(self, value, expected): + def test_extra_loggers(self, value: str | list[str], expected: list[str]): settings = Settings(logging=LoggingSettings(extra_loggers=value)) assert set(PREFECT_LOGGING_EXTRA_LOGGERS.value_from(settings)) == set(expected) @@ -799,7 +827,7 @@ def test_prefect_home_expands_tilde_in_path(self): ("https://api.foo.bar", "https://api.foo.bar"), ], ) - def test_ui_url_inferred_from_api_url(self, api_url, ui_url): + def test_ui_url_inferred_from_api_url(self, api_url: str, ui_url: str): with temporary_settings({PREFECT_API_URL: api_url}): assert PREFECT_UI_URL.value() == ui_url @@ -839,7 +867,7 @@ def test_ui_url_set_directly(self): ("https://api.foo.bar", "https://api.foo.bar"), ], ) - def test_cloud_ui_url_inferred_from_cloud_api_url(self, api_url, ui_url): + def test_cloud_ui_url_inferred_from_cloud_api_url(self, api_url: str, ui_url: str): with temporary_settings({PREFECT_CLOUD_API_URL: api_url}): assert PREFECT_CLOUD_UI_URL.value() == ui_url @@ -872,7 +900,7 @@ def test_ui_api_url_default(self): ("400, 401, 402", {400, 401, 402}), ], ) - def test_client_retry_extra_codes(self, extra_codes, expected): + def test_client_retry_extra_codes(self, extra_codes: str, expected: set[int]): with temporary_settings({PREFECT_CLIENT_RETRY_EXTRA_CODES: extra_codes}): assert PREFECT_CLIENT_RETRY_EXTRA_CODES.value() == expected @@ -887,7 +915,7 @@ def test_client_retry_extra_codes(self, extra_codes, expected): "400,500,foo", ], ) - def test_client_retry_extra_codes_invalid(self, extra_codes): + def test_client_retry_extra_codes_invalid(self, extra_codes: str): with pytest.raises(ValueError): with temporary_settings({PREFECT_CLIENT_RETRY_EXTRA_CODES: extra_codes}): PREFECT_CLIENT_RETRY_EXTRA_CODES.value() @@ -909,7 +937,7 @@ def test_deprecated_ENV_VAR_attribute_access(self): assert value == settings.testing.test_mode - def test_settings_with_serialization_alias(self, monkeypatch): + def test_settings_with_serialization_alias(self, monkeypatch: pytest.MonkeyPatch): assert not Settings().client.metrics.enabled # Use old value monkeypatch.setenv("PREFECT_CLIENT_ENABLE_METRICS", "True") @@ -924,10 +952,13 @@ def test_settings_with_serialization_alias(self, monkeypatch): # Check both can be imported from prefect.settings import ( - PREFECT_CLIENT_ENABLE_METRICS, # noqa - PREFECT_CLIENT_METRICS_ENABLED, # noqa + PREFECT_CLIENT_ENABLE_METRICS, + PREFECT_CLIENT_METRICS_ENABLED, ) + assert isinstance(PREFECT_CLIENT_ENABLE_METRICS, Setting) + assert isinstance(PREFECT_CLIENT_METRICS_ENABLED, Setting) + class TestDatabaseSettings: def test_database_connection_url_templates_password(self): @@ -1110,6 +1141,81 @@ def test_connection_string_with_dollar_sign(self): assert url.database == "the-database" assert url.password == "the-$password" + def test_sqlalchemy_settings_migration(self): + """Test that SQLAlchemy settings work with both old and new structures.""" + + with pytest.warns( + DeprecationWarning, match="moved to the `sqlalchemy` settings group." + ): + settings_with_old_keys = Settings( + server=ServerSettings( + database=ServerDatabaseSettings( + sqlalchemy_pool_size=42, + sqlalchemy_max_overflow=37, + ) + ) + ) + assert settings_with_old_keys.server.database.sqlalchemy_pool_size == 42 + assert settings_with_old_keys.server.database.sqlalchemy_max_overflow == 37 + + assert settings_with_old_keys.server.database.sqlalchemy.pool_size == 42 + assert settings_with_old_keys.server.database.sqlalchemy.max_overflow == 37 + + settings_with_new_keys = Settings( + server=ServerSettings( + database=ServerDatabaseSettings( + sqlalchemy=SQLAlchemySettings( + pool_size=42, + max_overflow=37, + ) + ) + ) + ) + + with pytest.warns( + DeprecationWarning, match="moved to the `sqlalchemy` settings group." + ): + assert settings_with_new_keys.server.database.sqlalchemy_pool_size == 42 + assert settings_with_new_keys.server.database.sqlalchemy_max_overflow == 37 + + # new keys are updated by setting new keys + assert settings_with_new_keys.server.database.sqlalchemy.pool_size == 42 + assert settings_with_new_keys.server.database.sqlalchemy.max_overflow == 37 + + def test_sqlalchemy_settings_migration_via_env_var( + self, monkeypatch: pytest.MonkeyPatch + ): + monkeypatch.setenv("PREFECT_SERVER_DATABASE_SQLALCHEMY_POOL_SIZE", "128") + monkeypatch.setenv("PREFECT_SERVER_DATABASE_SQLALCHEMY_MAX_OVERFLOW", "9001") + assert Settings().server.database.sqlalchemy.pool_size == 128 + assert Settings().server.database.sqlalchemy.max_overflow == 9001 + + def test_sqlalchemy_settings_migration_via_toml( + self, temporary_toml_file: Callable[..., None] + ): + """Test that SQLAlchemy settings can be configured via TOML files.""" + toml_data: dict[str, Any] = { + "server": { + "database": { + "sqlalchemy": { + "pool_size": 42, + "max_overflow": 37, + } + } + } + } + temporary_toml_file(toml_data) + + settings = Settings() + assert settings.server.database.sqlalchemy.pool_size == 42 + assert settings.server.database.sqlalchemy.max_overflow == 37 + + with pytest.warns( + DeprecationWarning, match="moved to the `sqlalchemy` settings group." + ): + assert settings.server.database.sqlalchemy_pool_size == 42 + assert settings.server.database.sqlalchemy_max_overflow == 37 + class TestTemporarySettings: def test_temporary_settings(self): @@ -1153,7 +1259,7 @@ def test_temporary_settings_restores_on_error(self): class TestSettingsSources: - def test_env_source(self, temporary_env_file): + def test_env_source(self, temporary_env_file: Callable[[str], None]): temporary_env_file("PREFECT_CLIENT_RETRY_EXTRA_CODES=420,500") assert Settings().client.retry_extra_codes == {420, 500} @@ -1162,7 +1268,12 @@ def test_env_source(self, temporary_env_file): assert Settings().client.retry_extra_codes == set() - def test_resolution_order(self, temporary_env_file, monkeypatch, tmp_path): + def test_resolution_order( + self, + temporary_env_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ): profiles_path = tmp_path / "profiles.toml" monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) @@ -1218,7 +1329,11 @@ def test_resolution_order(self, temporary_env_file, monkeypatch, tmp_path): assert Settings().client.retry_extra_codes == set() - def test_read_legacy_setting_from_profile(self, monkeypatch, tmp_path): + def test_read_legacy_setting_from_profile( + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, + ): Settings().client.metrics.enabled = False profiles_path = tmp_path / "profiles.toml" @@ -1240,7 +1355,10 @@ def test_read_legacy_setting_from_profile(self, monkeypatch, tmp_path): assert Settings().client.metrics.enabled is True def test_resolution_order_with_nested_settings( - self, temporary_env_file, monkeypatch, tmp_path + self, + temporary_env_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): profiles_path = tmp_path / "profiles.toml" @@ -1270,7 +1388,10 @@ def test_resolution_order_with_nested_settings( assert Settings().api.url == "http://example.com:4200" def test_profiles_path_from_env_source( - self, temporary_env_file, monkeypatch, tmp_path + self, + temporary_env_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): profiles_path = tmp_path / "custom_profiles.toml" @@ -1301,7 +1422,10 @@ def test_profiles_path_from_env_source( assert Settings().client.retry_extra_codes == set() def test_profiles_path_from_toml_source( - self, temporary_toml_file, monkeypatch, tmp_path + self, + temporary_toml_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): profiles_path = tmp_path / "custom_profiles.toml" @@ -1332,7 +1456,10 @@ def test_profiles_path_from_toml_source( assert Settings().client.retry_extra_codes == set() def test_profiles_path_from_pyproject_source( - self, temporary_toml_file, monkeypatch, tmp_path + self, + temporary_toml_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) @@ -1365,7 +1492,10 @@ def test_profiles_path_from_pyproject_source( assert Settings().client.retry_extra_codes == set() def test_profiles_path_resolution_order_from_sources( - self, temporary_env_file, monkeypatch, tmp_path + self, + temporary_env_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): monkeypatch.delenv("PREFECT_TESTING_TEST_MODE", raising=False) monkeypatch.delenv("PREFECT_TESTING_UNIT_TEST_MODE", raising=False) @@ -1443,7 +1573,10 @@ def test_profiles_path_resolution_order_from_sources( assert Settings().client.retry_extra_codes == set() - def test_dot_env_filters_as_expected(self, temporary_env_file): + def test_dot_env_filters_as_expected( + self, + temporary_env_file: Callable[[str], None], + ): expected_home = Settings().home expected_db_name = Settings().server.database.name temporary_env_file("HOME=foo\nNAME=bar") @@ -1453,7 +1586,9 @@ def test_dot_env_filters_as_expected(self, temporary_env_file): assert Settings().server.database.name != "bar" def test_environment_variables_take_precedence_over_toml_settings( - self, monkeypatch, temporary_toml_file + self, + monkeypatch: pytest.MonkeyPatch, + temporary_toml_file: Callable[[str], None], ): """ Test to ensure that fields with multiple validation aliases respect the @@ -1473,7 +1608,9 @@ def test_environment_variables_take_precedence_over_toml_settings( assert not PREFECT_SERVER_ALLOW_EPHEMERAL_MODE.value() def test_handle_profile_settings_without_active_profile( - self, monkeypatch, tmp_path + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): profiles_path = tmp_path / "profiles.toml" @@ -1492,7 +1629,9 @@ def test_handle_profile_settings_without_active_profile( assert Settings().server.ephemeral.enabled def test_handle_profile_settings_with_invalid_active_profile( - self, monkeypatch, tmp_path + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): profiles_path = tmp_path / "profiles.toml" @@ -1516,7 +1655,9 @@ def test_handle_profile_settings_with_invalid_active_profile( assert Settings().logging.level != "DEBUG" def test_handle_profile_settings_with_missing_profile_data( - self, monkeypatch, tmp_path + self, + monkeypatch: pytest.MonkeyPatch, + tmp_path: Path, ): profiles_path = tmp_path / "profiles.toml" @@ -1538,7 +1679,7 @@ def test_handle_profile_settings_with_missing_profile_data( class TestLoadProfiles: @pytest.fixture(autouse=True) - def temporary_profiles_path(self, tmp_path): + def temporary_profiles_path(self, tmp_path: Path): path = tmp_path / "profiles.toml" with temporary_settings(updates={PREFECT_PROFILES_PATH: path}): yield path @@ -1546,7 +1687,10 @@ def temporary_profiles_path(self, tmp_path): def test_load_profiles_no_profiles_file(self): assert load_profiles() - def test_env_variables_respected_when_no_profiles_file(self, monkeypatch): + def test_env_variables_respected_when_no_profiles_file( + self, + monkeypatch: pytest.MonkeyPatch, + ): """ Regression test for https://github.com/PrefectHQ/prefect/issues/15981 """ @@ -1560,7 +1704,10 @@ def test_env_variables_respected_when_no_profiles_file(self, monkeypatch): assert not Settings().server.ephemeral.enabled assert not PREFECT_SERVER_ALLOW_EPHEMERAL_MODE.value() - def test_load_profiles_missing_ephemeral(self, temporary_profiles_path): + def test_load_profiles_missing_ephemeral( + self, + temporary_profiles_path: Path, + ): temporary_profiles_path.write_text( textwrap.dedent( """ @@ -1572,7 +1719,7 @@ def test_load_profiles_missing_ephemeral(self, temporary_profiles_path): assert load_profiles()["foo"].settings == {PREFECT_API_KEY: "bar"} assert isinstance(load_profiles()["ephemeral"].settings, dict) - def test_load_profiles_only_active_key(self, temporary_profiles_path): + def test_load_profiles_only_active_key(self, temporary_profiles_path: Path): temporary_profiles_path.write_text( textwrap.dedent( """ @@ -1583,12 +1730,12 @@ def test_load_profiles_only_active_key(self, temporary_profiles_path): assert load_profiles().active_name == "ephemeral" assert isinstance(load_profiles()["ephemeral"].settings, dict) - def test_load_profiles_empty_file(self, temporary_profiles_path): + def test_load_profiles_empty_file(self, temporary_profiles_path: Path): temporary_profiles_path.touch() assert load_profiles().active_name == "ephemeral" assert isinstance(load_profiles()["ephemeral"].settings, dict) - def test_load_profiles_with_ephemeral(self, temporary_profiles_path): + def test_load_profiles_with_ephemeral(self, temporary_profiles_path: Path): temporary_profiles_path.write_text( """ [profiles.ephemeral] @@ -1621,7 +1768,7 @@ def test_load_profile_missing(self): with pytest.raises(ValueError, match="Profile 'foo' not found."): load_profile("foo") - def test_load_profile(self, temporary_profiles_path): + def test_load_profile(self, temporary_profiles_path: Path): temporary_profiles_path.write_text( textwrap.dedent( """ @@ -1640,7 +1787,9 @@ def test_load_profile(self, temporary_profiles_path): source=temporary_profiles_path, ) - def test_load_profile_does_not_allow_nested_data(self, temporary_profiles_path): + def test_load_profile_does_not_allow_nested_data( + self, temporary_profiles_path: Path + ): temporary_profiles_path.write_text( textwrap.dedent( """ @@ -1669,12 +1818,14 @@ def test_load_profile_with_invalid_key(self, temporary_profiles_path): class TestSaveProfiles: @pytest.fixture(autouse=True) - def temporary_profiles_path(self, tmp_path): + def temporary_profiles_path(self, tmp_path: Path): path = tmp_path / "profiles.toml" with temporary_settings(updates={PREFECT_PROFILES_PATH: path}): yield path - def test_save_profiles_does_not_include_default(self, temporary_profiles_path): + def test_save_profiles_does_not_include_default( + self, temporary_profiles_path: Path + ): """ Including the default has a tendency to bake in settings the user may not want, and can prevent them from gaining new defaults. @@ -1682,7 +1833,7 @@ def test_save_profiles_does_not_include_default(self, temporary_profiles_path): save_profiles(ProfilesCollection(active=None, profiles=[])) assert "profiles.default" not in temporary_profiles_path.read_text() - def test_save_profiles_additional_profiles(self, temporary_profiles_path): + def test_save_profiles_additional_profiles(self, temporary_profiles_path: Path): save_profiles( ProfilesCollection( profiles=[ @@ -1726,7 +1877,9 @@ def test_validate_settings(self): ): profile.validate_settings() - def test_validate_settings_ignores_environment_variables(self, monkeypatch): + def test_validate_settings_ignores_environment_variables( + self, monkeypatch: pytest.MonkeyPatch + ): """ If using `context.use_profile` to validate settings, environment variables may override the setting and hide validation errors @@ -1987,23 +2140,29 @@ def test_equality(self): class TestSettingValues: @pytest.fixture(autouse=True) - def clear_env_vars(self, monkeypatch): + def clear_env_vars(self, monkeypatch: pytest.MonkeyPatch): for env_var in os.environ: if env_var.startswith("PREFECT_"): monkeypatch.delenv(env_var, raising=False) @pytest.fixture(scope="function", params=list(SUPPORTED_SETTINGS.keys())) - def setting_and_value(self, request): + def setting_and_value(self, request: pytest.FixtureRequest) -> tuple[str, Any]: setting = request.param return setting, SUPPORTED_SETTINGS[setting]["test_value"] @pytest.fixture(autouse=True) - def temporary_profiles_path(self, tmp_path, monkeypatch): + def temporary_profiles_path( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> Generator[Path, None, None]: path = tmp_path / "profiles.toml" monkeypatch.setenv("PREFECT_PROFILES_PATH", str(path)) yield path - def check_setting_value(self, setting, value): + def check_setting_value( + self, + setting: str, + value: Any, + ): # create new root context to pick up the env var changes warnings.filterwarnings("ignore", category=UserWarning) with prefect.context.root_settings_context(): @@ -2041,7 +2200,11 @@ def check_setting_value(self, setting, value): to_jsonable_python(value) ) - def test_set_via_env_var(self, setting_and_value, monkeypatch): + def test_set_via_env_var( + self, + setting_and_value: tuple[str, Any], + monkeypatch: pytest.MonkeyPatch, + ): setting, value = setting_and_value if ( @@ -2056,7 +2219,10 @@ def test_set_via_env_var(self, setting_and_value, monkeypatch): self.check_setting_value(setting, value) def test_set_via_profile( - self, temporary_profiles_path, setting_and_value, monkeypatch + self, + temporary_profiles_path: Path, + setting_and_value: tuple[str, Any], + monkeypatch: pytest.MonkeyPatch, ): setting, value = setting_and_value if setting == "PREFECT_PROFILES_PATH": @@ -2081,7 +2247,10 @@ def test_set_via_profile( self.check_setting_value(setting, value) def test_set_via_dot_env_file( - self, setting_and_value, temporary_env_file, monkeypatch + self, + setting_and_value: tuple[str, Any], + temporary_env_file: Callable[[str], None], + monkeypatch: pytest.MonkeyPatch, ): setting, value = setting_and_value if setting == "PREFECT_PROFILES_PATH": @@ -2097,7 +2266,10 @@ def test_set_via_dot_env_file( self.check_setting_value(setting, value) def test_set_via_prefect_toml_file( - self, setting_and_value, temporary_toml_file, monkeypatch + self, + setting_and_value: tuple[str, Any], + temporary_toml_file: Callable[[dict[str, Any], Path], None], + monkeypatch: pytest.MonkeyPatch, ): setting, value = setting_and_value if setting == "PREFECT_PROFILES_PATH": @@ -2118,7 +2290,10 @@ def test_set_via_prefect_toml_file( self.check_setting_value(setting, value) def test_set_via_pyproject_toml_file( - self, setting_and_value, temporary_toml_file, monkeypatch + self, + setting_and_value: tuple[str, Any], + temporary_toml_file: Callable[[dict[str, Any], Path], None], + monkeypatch: pytest.MonkeyPatch, ): setting, value = setting_and_value if setting == "PREFECT_PROFILES_PATH":