diff --git a/.gitleaks.toml b/.gitleaks.toml new file mode 100644 index 00000000000..0622b57c118 --- /dev/null +++ b/.gitleaks.toml @@ -0,0 +1,15 @@ +# Title for the gitleaks configuration file. +title = "Gitleaks" + +[extend] +# useDefault will extend the base configuration with the default gitleaks config: +# https://github.com/zricethezav/gitleaks/blob/master/config/gitleaks.toml +useDefault = true + +[allowlist] +description = "Allow list false positive" + +# Allow list paths to ignore due to false positives. +paths = [ + '''tests/unit/parser/test_kinesis_firehose\.py''', +] diff --git a/aws_lambda_powertools/metrics/__init__.py b/aws_lambda_powertools/metrics/__init__.py index 5f30f14102d..b8c94478816 100644 --- a/aws_lambda_powertools/metrics/__init__.py +++ b/aws_lambda_powertools/metrics/__init__.py @@ -1,23 +1,22 @@ """CloudWatch Embedded Metric Format utility """ -from .base import MetricResolution, MetricUnit -from .exceptions import ( +from aws_lambda_powertools.metrics.base import MetricResolution, MetricUnit, single_metric +from aws_lambda_powertools.metrics.exceptions import ( MetricResolutionError, MetricUnitError, MetricValueError, SchemaValidationError, ) -from .metric import single_metric -from .metrics import EphemeralMetrics, Metrics +from aws_lambda_powertools.metrics.metrics import EphemeralMetrics, Metrics __all__ = [ - "Metrics", - "EphemeralMetrics", "single_metric", - "MetricUnit", "MetricUnitError", - "MetricResolution", "MetricResolutionError", "SchemaValidationError", "MetricValueError", + "Metrics", + "EphemeralMetrics", + "MetricResolution", + "MetricUnit", ] diff --git a/aws_lambda_powertools/metrics/base.py b/aws_lambda_powertools/metrics/base.py index 6a5e7282392..b32421431cd 100644 --- a/aws_lambda_powertools/metrics/base.py +++ b/aws_lambda_powertools/metrics/base.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import datetime import functools import json @@ -7,59 +9,28 @@ import warnings from collections import defaultdict from contextlib import contextmanager -from enum import Enum from typing import Any, Callable, Dict, Generator, List, Optional, Union -from ..shared import constants -from ..shared.functions import resolve_env_var_choice -from .exceptions import ( +from aws_lambda_powertools.metrics.exceptions import ( MetricResolutionError, MetricUnitError, MetricValueError, SchemaValidationError, ) -from .types import MetricNameUnitResolution +from aws_lambda_powertools.metrics.provider.cloudwatch_emf import cold_start +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.cold_start import ( + reset_cold_start_flag, # noqa: F401 # backwards compatibility +) +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.constants import MAX_DIMENSIONS, MAX_METRICS +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.metric_properties import MetricResolution, MetricUnit +from aws_lambda_powertools.metrics.types import MetricNameUnitResolution +from aws_lambda_powertools.shared import constants +from aws_lambda_powertools.shared.functions import resolve_env_var_choice logger = logging.getLogger(__name__) -MAX_METRICS = 100 -MAX_DIMENSIONS = 29 - -is_cold_start = True - - -class MetricResolution(Enum): - Standard = 60 - High = 1 - - -class MetricUnit(Enum): - Seconds = "Seconds" - Microseconds = "Microseconds" - Milliseconds = "Milliseconds" - Bytes = "Bytes" - Kilobytes = "Kilobytes" - Megabytes = "Megabytes" - Gigabytes = "Gigabytes" - Terabytes = "Terabytes" - Bits = "Bits" - Kilobits = "Kilobits" - Megabits = "Megabits" - Gigabits = "Gigabits" - Terabits = "Terabits" - Percent = "Percent" - Count = "Count" - BytesPerSecond = "Bytes/Second" - KilobytesPerSecond = "Kilobytes/Second" - MegabytesPerSecond = "Megabytes/Second" - GigabytesPerSecond = "Gigabytes/Second" - TerabytesPerSecond = "Terabytes/Second" - BitsPerSecond = "Bits/Second" - KilobitsPerSecond = "Kilobits/Second" - MegabitsPerSecond = "Megabits/Second" - GigabitsPerSecond = "Gigabits/Second" - TerabitsPerSecond = "Terabits/Second" - CountPerSecond = "Count/Second" +# Maintenance: alias due to Hyrum's law +is_cold_start = cold_start.is_cold_start class MetricManager: @@ -94,11 +65,11 @@ class MetricManager: def __init__( self, - metric_set: Optional[Dict[str, Any]] = None, - dimension_set: Optional[Dict] = None, - namespace: Optional[str] = None, - metadata_set: Optional[Dict[str, Any]] = None, - service: Optional[str] = None, + metric_set: Dict[str, Any] | None = None, + dimension_set: Dict | None = None, + namespace: str | None = None, + metadata_set: Dict[str, Any] | None = None, + service: str | None = None, ): self.metric_set = metric_set if metric_set is not None else {} self.dimension_set = dimension_set if dimension_set is not None else {} @@ -112,9 +83,9 @@ def __init__( def add_metric( self, name: str, - unit: Union[MetricUnit, str], + unit: MetricUnit | str, value: float, - resolution: Union[MetricResolution, int] = 60, + resolution: MetricResolution | int = 60, ) -> None: """Adds given metric @@ -173,9 +144,9 @@ def add_metric( def serialize_metric_set( self, - metrics: Optional[Dict] = None, - dimensions: Optional[Dict] = None, - metadata: Optional[Dict] = None, + metrics: Dict | None = None, + dimensions: Dict | None = None, + metadata: Dict | None = None, ) -> Dict: """Serializes metric and dimensions set @@ -355,10 +326,10 @@ def flush_metrics(self, raise_on_empty_metrics: bool = False) -> None: def log_metrics( self, - lambda_handler: Union[Callable[[Dict, Any], Any], Optional[Callable[[Dict, Any, Optional[Dict]], Any]]] = None, + lambda_handler: Callable[[Dict, Any], Any] | Optional[Callable[[Dict, Any, Optional[Dict]], Any]] = None, capture_cold_start_metric: bool = False, raise_on_empty_metrics: bool = False, - default_dimensions: Optional[Dict[str, str]] = None, + default_dimensions: Dict[str, str] | None = None, ): """Decorator to serialize and publish metrics at the end of a function execution. @@ -537,9 +508,9 @@ class SingleMetric(MetricManager): def add_metric( self, name: str, - unit: Union[MetricUnit, str], + unit: MetricUnit | str, value: float, - resolution: Union[MetricResolution, int] = 60, + resolution: MetricResolution | int = 60, ) -> None: """Method to prevent more than one metric being created @@ -565,9 +536,9 @@ def single_metric( name: str, unit: MetricUnit, value: float, - resolution: Union[MetricResolution, int] = 60, - namespace: Optional[str] = None, - default_dimensions: Optional[Dict[str, str]] = None, + resolution: MetricResolution | int = 60, + namespace: str | None = None, + default_dimensions: Dict[str, str] | None = None, ) -> Generator[SingleMetric, None, None]: """Context manager to simplify creation of a single metric @@ -622,7 +593,7 @@ def single_metric( SchemaValidationError When metric object fails EMF schema validation """ # noqa: E501 - metric_set: Optional[Dict] = None + metric_set: Dict | None = None try: metric: SingleMetric = SingleMetric(namespace=namespace) metric.add_metric(name=name, unit=unit, value=value, resolution=resolution) @@ -635,9 +606,3 @@ def single_metric( metric_set = metric.serialize_metric_set() finally: print(json.dumps(metric_set, separators=(",", ":"))) - - -def reset_cold_start_flag(): - global is_cold_start - if not is_cold_start: - is_cold_start = True diff --git a/aws_lambda_powertools/metrics/exceptions.py b/aws_lambda_powertools/metrics/exceptions.py index 94f492d14d7..30a4996d67e 100644 --- a/aws_lambda_powertools/metrics/exceptions.py +++ b/aws_lambda_powertools/metrics/exceptions.py @@ -1,13 +1,4 @@ -class MetricUnitError(Exception): - """When metric unit is not supported by CloudWatch""" - - pass - - -class MetricResolutionError(Exception): - """When metric resolution is not supported by CloudWatch""" - - pass +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.exceptions import MetricResolutionError, MetricUnitError class SchemaValidationError(Exception): @@ -20,3 +11,6 @@ class MetricValueError(Exception): """When metric value isn't a valid number""" pass + + +__all__ = ["MetricUnitError", "MetricResolutionError", "SchemaValidationError", "MetricValueError"] diff --git a/aws_lambda_powertools/metrics/metric.py b/aws_lambda_powertools/metrics/metric.py index 5465889f1f0..e2ac49df489 100644 --- a/aws_lambda_powertools/metrics/metric.py +++ b/aws_lambda_powertools/metrics/metric.py @@ -1,4 +1,4 @@ # NOTE: prevents circular inheritance import -from .base import SingleMetric, single_metric +from aws_lambda_powertools.metrics.base import SingleMetric, single_metric __all__ = ["SingleMetric", "single_metric"] diff --git a/aws_lambda_powertools/metrics/metrics.py b/aws_lambda_powertools/metrics/metrics.py index 487f2ab9b2f..d87b7bdd401 100644 --- a/aws_lambda_powertools/metrics/metrics.py +++ b/aws_lambda_powertools/metrics/metrics.py @@ -1,9 +1,13 @@ -from typing import Any, Dict, Optional +# NOTE: keeps for compatibility +from __future__ import annotations -from .base import MetricManager +from typing import Any, Callable, Dict, Optional, Union +from aws_lambda_powertools.metrics.base import MetricResolution, MetricUnit +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.cloudwatch import AmazonCloudWatchEMFProvider -class Metrics(MetricManager): + +class Metrics: """Metrics create an EMF object with up to 100 metrics Use Metrics when you need to create multiple metrics that have @@ -69,76 +73,82 @@ def lambda_handler(): _metadata: Dict[str, Any] = {} _default_dimensions: Dict[str, Any] = {} - def __init__(self, service: Optional[str] = None, namespace: Optional[str] = None): + def __init__( + self, + service: str | None = None, + namespace: str | None = None, + provider: AmazonCloudWatchEMFProvider | None = None, + ): self.metric_set = self._metrics self.metadata_set = self._metadata self.default_dimensions = self._default_dimensions self.dimension_set = self._dimensions self.dimension_set.update(**self._default_dimensions) - super().__init__( - namespace=namespace, - service=service, - metric_set=self.metric_set, - dimension_set=self.dimension_set, - metadata_set=self.metadata_set, - ) - - def set_default_dimensions(self, **dimensions) -> None: - """Persist dimensions across Lambda invocations - - Parameters - ---------- - dimensions : Dict[str, Any], optional - metric dimensions as key=value - - Example - ------- - **Sets some default dimensions that will always be present across metrics and invocations** - from aws_lambda_powertools import Metrics - - metrics = Metrics(namespace="ServerlessAirline", service="payment") - metrics.set_default_dimensions(environment="demo", another="one") - - @metrics.log_metrics() - def lambda_handler(): - return True - """ - for name, value in dimensions.items(): - self.add_dimension(name, value) - - self.default_dimensions.update(**dimensions) - - def clear_default_dimensions(self) -> None: - self.default_dimensions.clear() - - def clear_metrics(self) -> None: - super().clear_metrics() - # re-add default dimensions - self.set_default_dimensions(**self.default_dimensions) - - -class EphemeralMetrics(MetricManager): - """Non-singleton version of Metrics to not persist metrics across instances - - NOTE: This is useful when you want to: - - - Create metrics for distinct namespaces - - Create the same metrics with different dimensions more than once - """ + if provider is None: + self.provider = AmazonCloudWatchEMFProvider( + namespace=namespace, + service=service, + metric_set=self.metric_set, + dimension_set=self.dimension_set, + metadata_set=self.metadata_set, + default_dimensions=self._default_dimensions, + ) + else: + self.provider = provider + + def add_metric( + self, + name: str, + unit: MetricUnit | str, + value: float, + resolution: MetricResolution | int = 60, + ) -> None: + self.provider.add_metric(name=name, unit=unit, value=value, resolution=resolution) + + def add_dimension(self, name: str, value: str) -> None: + self.provider.add_dimension(name=name, value=value) + + def serialize_metric_set( + self, + metrics: Dict | None = None, + dimensions: Dict | None = None, + metadata: Dict | None = None, + ) -> Dict: + return self.provider.serialize_metric_set(metrics=metrics, dimensions=dimensions, metadata=metadata) + + def add_metadata(self, key: str, value: Any) -> None: + self.provider.add_metadata(key=key, value=value) + + def flush_metrics(self, raise_on_empty_metrics: bool = False) -> None: + self.provider.flush_metrics(raise_on_empty_metrics=raise_on_empty_metrics) + + def log_metrics( + self, + lambda_handler: Callable[[Dict, Any], Any] | Optional[Callable[[Dict, Any, Optional[Dict]], Any]] = None, + capture_cold_start_metric: bool = False, + raise_on_empty_metrics: bool = False, + default_dimensions: Dict[str, str] | None = None, + ): + return self.provider.log_metrics( + lambda_handler=lambda_handler, + capture_cold_start_metric=capture_cold_start_metric, + raise_on_empty_metrics=raise_on_empty_metrics, + default_dimensions=default_dimensions, + ) - _dimensions: Dict[str, str] = {} - _default_dimensions: Dict[str, Any] = {} + def _extract_metric_resolution_value(self, resolution: Union[int, MetricResolution]) -> int: + return self.provider._extract_metric_resolution_value(resolution=resolution) - def __init__(self, service: Optional[str] = None, namespace: Optional[str] = None): - self.default_dimensions = self._default_dimensions - self.dimension_set = self._dimensions + def _extract_metric_unit_value(self, unit: Union[str, MetricUnit]) -> str: + return self.provider._extract_metric_unit_value(unit=unit) - self.dimension_set.update(**self._default_dimensions) - super().__init__(namespace=namespace, service=service) + def _add_cold_start_metric(self, context: Any) -> None: + self.provider._add_cold_start_metric(context=context) def set_default_dimensions(self, **dimensions) -> None: + self.provider.set_default_dimensions(**dimensions) """Persist dimensions across Lambda invocations Parameters @@ -165,9 +175,133 @@ def lambda_handler(): self.default_dimensions.update(**dimensions) def clear_default_dimensions(self) -> None: + self.provider.default_dimensions.clear() self.default_dimensions.clear() def clear_metrics(self) -> None: - super().clear_metrics() - # re-add default dimensions - self.set_default_dimensions(**self.default_dimensions) + self.provider.clear_metrics() + + +# Maintenance: until v3, we can't afford to break customers. +# AmazonCloudWatchEMFProvider has the exact same functionality (non-singleton) +# so we simply alias. If a customer subclassed `EphemeralMetrics` and somehow relied on __name__ +# we can quickly revert and duplicate code while using self.provider + +EphemeralMetrics = AmazonCloudWatchEMFProvider + +# noqa: ERA001 +# class EphemeralMetrics(MetricManager): +# """Non-singleton version of Metrics to not persist metrics across instances +# +# NOTE: This is useful when you want to: +# +# - Create metrics for distinct namespaces +# - Create the same metrics with different dimensions more than once +# """ +# +# # _dimensions: Dict[str, str] = {} +# _default_dimensions: Dict[str, Any] = {} +# +# def __init__( +# self, +# service: str | None = None, +# namespace: str | None = None, +# provider: AmazonCloudWatchEMFProvider | None = None, +# ): +# super().__init__(namespace=namespace, service=service) +# +# self.default_dimensions = self._default_dimensions +# # # self.dimension_set = self._dimensions +# # self.dimension_set.update(**self._default_dimensions) +# +# self.provider = provider or AmazonCloudWatchEMFProvider( +# namespace=namespace, +# service=service, +# metric_set=self.metric_set, +# metadata_set=self.metadata_set, +# dimension_set=self.dimension_set, +# default_dimensions=self._default_dimensions, +# ) +# +# def add_metric( +# self, +# name: str, +# unit: MetricUnit | str, +# value: float, +# resolution: MetricResolution | int = 60, +# ) -> None: +# return self.provider.add_metric(name=name, unit=unit, value=value, resolution=resolution) +# +# def add_dimension(self, name: str, value: str) -> None: +# return self.provider.add_dimension(name=name, value=value) +# +# def serialize_metric_set( +# self, +# metrics: Dict | None = None, +# dimensions: Dict | None = None, +# metadata: Dict | None = None, +# ) -> Dict: +# return self.provider.serialize_metric_set(metrics=metrics, dimensions=dimensions, metadata=metadata) +# +# def add_metadata(self, key: str, value: Any) -> None: +# self.provider.add_metadata(key=key, value=value) +# +# def flush_metrics(self, raise_on_empty_metrics: bool = False) -> None: +# self.provider.flush_metrics(raise_on_empty_metrics=raise_on_empty_metrics) +# +# def log_metrics( +# self, +# lambda_handler: Callable[[Dict, Any], Any] | Optional[Callable[[Dict, Any, Optional[Dict]], Any]] = None, +# capture_cold_start_metric: bool = False, +# raise_on_empty_metrics: bool = False, +# default_dimensions: Dict[str, str] | None = None, +# ): +# return self.provider.log_metrics( +# lambda_handler=lambda_handler, +# capture_cold_start_metric=capture_cold_start_metric, +# raise_on_empty_metrics=raise_on_empty_metrics, +# default_dimensions=default_dimensions, +# ) +# +# def _extract_metric_resolution_value(self, resolution: Union[int, MetricResolution]) -> int: +# return self.provider._extract_metric_resolution_value(resolution=resolution) +# +# def _extract_metric_unit_value(self, unit: Union[str, MetricUnit]) -> str: +# return self.provider._extract_metric_unit_value(unit=unit) +# +# def _add_cold_start_metric(self, context: Any) -> None: +# return self.provider._add_cold_start_metric(context=context) +# +# def set_default_dimensions(self, **dimensions) -> None: +# """Persist dimensions across Lambda invocations +# +# Parameters +# ---------- +# dimensions : Dict[str, Any], optional +# metric dimensions as key=value +# +# Example +# ------- +# **Sets some default dimensions that will always be present across metrics and invocations** +# +# from aws_lambda_powertools import Metrics +# +# metrics = Metrics(namespace="ServerlessAirline", service="payment") +# metrics.set_default_dimensions(environment="demo", another="one") +# +# @metrics.log_metrics() +# def lambda_handler(): +# return True +# """ +# return self.provider.set_default_dimensions(**dimensions) +# +# def clear_default_dimensions(self) -> None: +# self.default_dimensions.clear() +# +# def clear_metrics(self) -> None: +# self.provider.clear_metrics() +# # re-add default dimensions +# self.set_default_dimensions(**self.default_dimensions) +# + +# __all__ = [] diff --git a/aws_lambda_powertools/metrics/provider/__init__.py b/aws_lambda_powertools/metrics/provider/__init__.py new file mode 100644 index 00000000000..814812c135b --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/__init__.py @@ -0,0 +1,6 @@ +from aws_lambda_powertools.metrics.provider.base import MetricsBase, MetricsProviderBase + +__all__ = [ + "MetricsBase", + "MetricsProviderBase", +] diff --git a/aws_lambda_powertools/metrics/provider/base.py b/aws_lambda_powertools/metrics/provider/base.py new file mode 100644 index 00000000000..7617193033e --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/base.py @@ -0,0 +1,244 @@ +from __future__ import annotations + +import functools +import logging +from typing import Any, Callable, Dict, Optional + +from typing_extensions import Protocol + +logger = logging.getLogger(__name__) + +is_cold_start = True + + +class MetricsProviderBase(Protocol): + """ + Class for metric provider interface. + + This class serves as an interface for creating your own metric provider. Inherit from this class + and implement the required methods to define your specific metric provider. + + Usage: + 1. Inherit from this class. + 2. Implement the required methods specific to your metric provider. + 3. Customize the behavior and functionality of the metric provider in your subclass. + """ + + def add_metric(self, *args: Any, **kwargs: Any) -> Any: + """ + Abstract method for adding a metric. + + This method must be implemented in subclasses to add a metric and return a combined metrics dictionary. + + Parameters + ---------- + *args: + Positional arguments. + *kwargs: + Keyword arguments. + + Returns + ---------- + Dict + A combined metrics dictionary. + + Raises + ---------- + NotImplementedError + This method must be implemented in subclasses. + """ + raise NotImplementedError + + def serialize_metric_set(self, *args: Any, **kwargs: Any) -> Any: + """ + Abstract method for serialize a metric. + + This method must be implemented in subclasses to add a metric and return a combined metrics dictionary. + + Parameters + ---------- + *args: + Positional arguments. + *kwargs: + Keyword arguments. + + Returns + ---------- + Dict + Serialized metrics + + Raises + ---------- + NotImplementedError + This method must be implemented in subclasses. + """ + raise NotImplementedError + + # flush serialized data to output, or send to API directly + def flush_metrics(self, *args: Any, **kwargs) -> Any: + """ + Abstract method for flushing a metric. + + This method must be implemented in subclasses to add a metric and return a combined metrics dictionary. + + Parameters + ---------- + *args: + Positional arguments. + *kwargs: + Keyword arguments. + + Raises + ---------- + NotImplementedError + This method must be implemented in subclasses. + """ + raise NotImplementedError + + +class MetricsBase(Protocol): + """ + Class for metric template. + + This class serves as a template for creating your own metric class. Inherit from this class + and implement the necessary methods to define your specific metric. + + NOTE: need to improve this docstring + """ + + def add_metric(self, *args, **kwargs): + """ + Abstract method for adding a metric. + + This method must be implemented in subclasses to add a metric and return a combined metrics dictionary. + + Parameters + ---------- + *args: + Positional arguments. + *kwargs: + Keyword arguments. + + Returns + ---------- + Dict + A combined metrics dictionary. + + Raises + ---------- + NotImplementedError + This method must be implemented in subclasses. + """ + raise NotImplementedError + + def flush_metrics(self, raise_on_empty_metrics: bool = False) -> None: + """Manually flushes the metrics. This is normally not necessary, + unless you're running on other runtimes besides Lambda, where the @log_metrics + decorator already handles things for you. + + Parameters + ---------- + raise_on_empty_metrics : bool, optional + raise exception if no metrics are emitted, by default False + """ + raise NotImplementedError + + def add_cold_start_metric(self, metric_name: str, function_name: str) -> None: + """ + Add a cold start metric for a specific function. + + Parameters + ---------- + metric_name: str + The name of the cold start metric to add. + function_name: str + The name of the function associated with the cold start metric. + """ + raise NotImplementedError + + def log_metrics( + self, + lambda_handler: Callable[[Dict, Any], Any] | Optional[Callable[[Dict, Any, Optional[Dict]], Any]] = None, + capture_cold_start_metric: bool = False, + raise_on_empty_metrics: bool = False, + ): + """Decorator to serialize and publish metrics at the end of a function execution. + + Be aware that the log_metrics **does call* the decorated function (e.g. lambda_handler). + + Example + ------- + **Lambda function using tracer and metrics decorators** + + from aws_lambda_powertools import Metrics, Tracer + + metrics = Metrics(service="payment") + tracer = Tracer(service="payment") + + @tracer.capture_lambda_handler + @metrics.log_metrics + def handler(event, context): + ... + + Parameters + ---------- + lambda_handler : Callable[[Any, Any], Any], optional + lambda function handler, by default None + capture_cold_start_metric : bool, optional + captures cold start metric, by default False + raise_on_empty_metrics : bool, optional + raise exception if no metrics are emitted, by default False + default_dimensions: Dict[str, str], optional + metric dimensions as key=value that will always be present + + Raises + ------ + e + Propagate error received + """ + + # If handler is None we've been called with parameters + # Return a partial function with args filled + if lambda_handler is None: + logger.debug("Decorator called with parameters") + return functools.partial( + self.log_metrics, + capture_cold_start_metric=capture_cold_start_metric, + raise_on_empty_metrics=raise_on_empty_metrics, + ) + + @functools.wraps(lambda_handler) + def decorate(event, context): + try: + response = lambda_handler(event, context) + if capture_cold_start_metric: + self._add_cold_start_metric(context=context) + finally: + self.flush_metrics(raise_on_empty_metrics=raise_on_empty_metrics) + + return response + + return decorate + + def _add_cold_start_metric(self, context: Any) -> None: + """Add cold start metric and function_name dimension + + Parameters + ---------- + context : Any + Lambda context + """ + global is_cold_start + if not is_cold_start: + return + + logger.debug("Adding cold start metric and function_name dimension") + self.add_cold_start_metric(metric_name="ColdStart", function_name=context.function_name) + + is_cold_start = False + + +def reset_cold_start_flag_provider(): + global is_cold_start + if not is_cold_start: + is_cold_start = True diff --git a/aws_lambda_powertools/metrics/provider/cloudwatch_emf/__init__.py b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/aws_lambda_powertools/metrics/provider/cloudwatch_emf/cloudwatch.py b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/cloudwatch.py new file mode 100644 index 00000000000..921fcee6045 --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/cloudwatch.py @@ -0,0 +1,496 @@ +from __future__ import annotations + +import datetime +import functools +import json +import logging +import numbers +import os +import warnings +from collections import defaultdict +from typing import Any, Callable, Dict, List, Optional, Union + +from aws_lambda_powertools.metrics.base import single_metric +from aws_lambda_powertools.metrics.exceptions import MetricValueError, SchemaValidationError +from aws_lambda_powertools.metrics.provider import MetricsProviderBase +from aws_lambda_powertools.metrics.provider.cloudwatch_emf import cold_start +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.constants import MAX_DIMENSIONS, MAX_METRICS +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.exceptions import ( + MetricResolutionError, + MetricUnitError, +) +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.metric_properties import MetricResolution, MetricUnit +from aws_lambda_powertools.metrics.types import MetricNameUnitResolution +from aws_lambda_powertools.shared import constants +from aws_lambda_powertools.shared.functions import resolve_env_var_choice + +logger = logging.getLogger(__name__) + + +class AmazonCloudWatchEMFProvider(MetricsProviderBase): + """Base class for metric functionality (namespace, metric, dimension, serialization) + + MetricManager creates metrics asynchronously thanks to CloudWatch Embedded Metric Format (EMF). + CloudWatch EMF can create up to 100 metrics per EMF object + and metrics, dimensions, and namespace created via MetricManager + will adhere to the schema, will be serialized and validated against EMF Schema. + + **Use `aws_lambda_powertools.metrics.metrics.Metrics` or + `aws_lambda_powertools.metrics.metric.single_metric` to create EMF metrics.** + + Environment variables + --------------------- + POWERTOOLS_METRICS_NAMESPACE : str + metric namespace to be set for all metrics + POWERTOOLS_SERVICE_NAME : str + service name used for default dimension + + Raises + ------ + MetricUnitError + When metric unit isn't supported by CloudWatch + MetricResolutionError + When metric resolution isn't supported by CloudWatch + MetricValueError + When metric value isn't a number + SchemaValidationError + When metric object fails EMF schema validation + """ + + def __init__( + self, + metric_set: Dict[str, Any] | None = None, + dimension_set: Dict | None = None, + namespace: str | None = None, + metadata_set: Dict[str, Any] | None = None, + service: str | None = None, + default_dimensions: Dict[str, Any] | None = None, + ): + self.metric_set = metric_set if metric_set is not None else {} + self.dimension_set = dimension_set if dimension_set is not None else {} + self.default_dimensions = default_dimensions or {} + self.namespace = resolve_env_var_choice(choice=namespace, env=os.getenv(constants.METRICS_NAMESPACE_ENV)) + self.service = resolve_env_var_choice(choice=service, env=os.getenv(constants.SERVICE_NAME_ENV)) + self.metadata_set = metadata_set if metadata_set is not None else {} + + self._metric_units = [unit.value for unit in MetricUnit] + self._metric_unit_valid_options = list(MetricUnit.__members__) + self._metric_resolutions = [resolution.value for resolution in MetricResolution] + + self.dimension_set.update(**self.default_dimensions) + + def add_metric( + self, + name: str, + unit: MetricUnit | str, + value: float, + resolution: MetricResolution | int = 60, + ) -> None: + """Adds given metric + + Example + ------- + **Add given metric using MetricUnit enum** + + metric.add_metric(name="BookingConfirmation", unit=MetricUnit.Count, value=1) + + **Add given metric using plain string as value unit** + + metric.add_metric(name="BookingConfirmation", unit="Count", value=1) + + **Add given metric with MetricResolution non default value** + + metric.add_metric(name="BookingConfirmation", unit="Count", value=1, resolution=MetricResolution.High) + + Parameters + ---------- + name : str + Metric name + unit : Union[MetricUnit, str] + `aws_lambda_powertools.helper.models.MetricUnit` + value : float + Metric value + resolution : Union[MetricResolution, int] + `aws_lambda_powertools.helper.models.MetricResolution` + + Raises + ------ + MetricUnitError + When metric unit is not supported by CloudWatch + MetricResolutionError + When metric resolution is not supported by CloudWatch + """ + if not isinstance(value, numbers.Number): + raise MetricValueError(f"{value} is not a valid number") + + unit = self._extract_metric_unit_value(unit=unit) + resolution = self._extract_metric_resolution_value(resolution=resolution) + metric: Dict = self.metric_set.get(name, defaultdict(list)) + metric["Unit"] = unit + metric["StorageResolution"] = resolution + metric["Value"].append(float(value)) + logger.debug(f"Adding metric: {name} with {metric}") + self.metric_set[name] = metric + + if len(self.metric_set) == MAX_METRICS or len(metric["Value"]) == MAX_METRICS: + logger.debug(f"Exceeded maximum of {MAX_METRICS} metrics - Publishing existing metric set") + metrics = self.serialize_metric_set() + print(json.dumps(metrics)) + + # clear metric set only as opposed to metrics and dimensions set + # since we could have more than 100 metrics + self.metric_set.clear() + + def serialize_metric_set( + self, + metrics: Dict | None = None, + dimensions: Dict | None = None, + metadata: Dict | None = None, + ) -> Dict: + """Serializes metric and dimensions set + + Parameters + ---------- + metrics : Dict, optional + Dictionary of metrics to serialize, by default None + dimensions : Dict, optional + Dictionary of dimensions to serialize, by default None + metadata: Dict, optional + Dictionary of metadata to serialize, by default None + + Example + ------- + **Serialize metrics into EMF format** + + metrics = MetricManager() + # ...add metrics, dimensions, namespace + ret = metrics.serialize_metric_set() + + Returns + ------- + Dict + Serialized metrics following EMF specification + + Raises + ------ + SchemaValidationError + Raised when serialization fail schema validation + """ + if metrics is None: # pragma: no cover + metrics = self.metric_set + + if dimensions is None: # pragma: no cover + dimensions = self.dimension_set + + if metadata is None: # pragma: no cover + metadata = self.metadata_set + + if self.service and not self.dimension_set.get("service"): + # self.service won't be a float + self.add_dimension(name="service", value=self.service) + + if len(metrics) == 0: + raise SchemaValidationError("Must contain at least one metric.") + + if self.namespace is None: + raise SchemaValidationError("Must contain a metric namespace.") + + logger.debug({"details": "Serializing metrics", "metrics": metrics, "dimensions": dimensions}) + + # For standard resolution metrics, don't add StorageResolution field to avoid unnecessary ingestion of data into cloudwatch # noqa E501 + # Example: [ { "Name": "metric_name", "Unit": "Count"} ] # noqa ERA001 + # + # In case using high-resolution metrics, add StorageResolution field + # Example: [ { "Name": "metric_name", "Unit": "Count", "StorageResolution": 1 } ] # noqa ERA001 + metric_definition: List[MetricNameUnitResolution] = [] + metric_names_and_values: Dict[str, float] = {} # { "metric_name": 1.0 } + + for metric_name in metrics: + metric: dict = metrics[metric_name] + metric_value: int = metric.get("Value", 0) + metric_unit: str = metric.get("Unit", "") + metric_resolution: int = metric.get("StorageResolution", 60) + + metric_definition_data: MetricNameUnitResolution = {"Name": metric_name, "Unit": metric_unit} + + # high-resolution metrics + if metric_resolution == 1: + metric_definition_data["StorageResolution"] = metric_resolution + + metric_definition.append(metric_definition_data) + + metric_names_and_values.update({metric_name: metric_value}) + + return { + "_aws": { + "Timestamp": int(datetime.datetime.now().timestamp() * 1000), # epoch + "CloudWatchMetrics": [ + { + "Namespace": self.namespace, # "test_namespace" + "Dimensions": [list(dimensions.keys())], # [ "service" ] + "Metrics": metric_definition, + }, + ], + }, + **dimensions, # "service": "test_service" + **metadata, # "username": "test" + **metric_names_and_values, # "single_metric": 1.0 + } + + def add_dimension(self, name: str, value: str) -> None: + """Adds given dimension to all metrics + + Example + ------- + **Add a metric dimensions** + + metric.add_dimension(name="operation", value="confirm_booking") + + Parameters + ---------- + name : str + Dimension name + value : str + Dimension value + """ + logger.debug(f"Adding dimension: {name}:{value}") + if len(self.dimension_set) == MAX_DIMENSIONS: + raise SchemaValidationError( + f"Maximum number of dimensions exceeded ({MAX_DIMENSIONS}): Unable to add dimension {name}.", + ) + # Cast value to str according to EMF spec + # Majority of values are expected to be string already, so + # checking before casting improves performance in most cases + self.dimension_set[name] = value if isinstance(value, str) else str(value) + + def add_metadata(self, key: str, value: Any) -> None: + """Adds high cardinal metadata for metrics object + + This will not be available during metrics visualization. + Instead, this will be searchable through logs. + + If you're looking to add metadata to filter metrics, then + use add_dimensions method. + + Example + ------- + **Add metrics metadata** + + metric.add_metadata(key="booking_id", value="booking_id") + + Parameters + ---------- + key : str + Metadata key + value : any + Metadata value + """ + logger.debug(f"Adding metadata: {key}:{value}") + + # Cast key to str according to EMF spec + # Majority of keys are expected to be string already, so + # checking before casting improves performance in most cases + if isinstance(key, str): + self.metadata_set[key] = value + else: + self.metadata_set[str(key)] = value + + def clear_metrics(self) -> None: + logger.debug("Clearing out existing metric set from memory") + self.metric_set.clear() + self.dimension_set.clear() + self.metadata_set.clear() + self.set_default_dimensions(**self.default_dimensions) + + def flush_metrics(self, raise_on_empty_metrics: bool = False) -> None: + """Manually flushes the metrics. This is normally not necessary, + unless you're running on other runtimes besides Lambda, where the @log_metrics + decorator already handles things for you. + + Parameters + ---------- + raise_on_empty_metrics : bool, optional + raise exception if no metrics are emitted, by default False + """ + if not raise_on_empty_metrics and not self.metric_set: + warnings.warn( + "No application metrics to publish. The cold-start metric may be published if enabled. " + "If application metrics should never be empty, consider using 'raise_on_empty_metrics'", + stacklevel=2, + ) + else: + logger.debug("Flushing existing metrics") + metrics = self.serialize_metric_set() + print(json.dumps(metrics, separators=(",", ":"))) + self.clear_metrics() + + def log_metrics( + self, + lambda_handler: Callable[[Dict, Any], Any] | Optional[Callable[[Dict, Any, Optional[Dict]], Any]] = None, + capture_cold_start_metric: bool = False, + raise_on_empty_metrics: bool = False, + default_dimensions: Dict[str, str] | None = None, + ): + """Decorator to serialize and publish metrics at the end of a function execution. + + Be aware that the log_metrics **does call* the decorated function (e.g. lambda_handler). + + Example + ------- + **Lambda function using tracer and metrics decorators** + + from aws_lambda_powertools import Metrics, Tracer + + metrics = Metrics(service="payment") + tracer = Tracer(service="payment") + + @tracer.capture_lambda_handler + @metrics.log_metrics + def handler(event, context): + ... + + Parameters + ---------- + lambda_handler : Callable[[Any, Any], Any], optional + lambda function handler, by default None + capture_cold_start_metric : bool, optional + captures cold start metric, by default False + raise_on_empty_metrics : bool, optional + raise exception if no metrics are emitted, by default False + default_dimensions: Dict[str, str], optional + metric dimensions as key=value that will always be present + + Raises + ------ + e + Propagate error received + """ + + # If handler is None we've been called with parameters + # Return a partial function with args filled + if lambda_handler is None: + logger.debug("Decorator called with parameters") + return functools.partial( + self.log_metrics, + capture_cold_start_metric=capture_cold_start_metric, + raise_on_empty_metrics=raise_on_empty_metrics, + default_dimensions=default_dimensions, + ) + + @functools.wraps(lambda_handler) + def decorate(event, context): + try: + if default_dimensions: + self.set_default_dimensions(**default_dimensions) + response = lambda_handler(event, context) + if capture_cold_start_metric: + self._add_cold_start_metric(context=context) + finally: + self.flush_metrics(raise_on_empty_metrics=raise_on_empty_metrics) + + return response + + return decorate + + def _extract_metric_resolution_value(self, resolution: Union[int, MetricResolution]) -> int: + """Return metric value from metric unit whether that's str or MetricResolution enum + + Parameters + ---------- + unit : Union[int, MetricResolution] + Metric resolution + + Returns + ------- + int + Metric resolution value must be 1 or 60 + + Raises + ------ + MetricResolutionError + When metric resolution is not supported by CloudWatch + """ + if isinstance(resolution, MetricResolution): + return resolution.value + + if isinstance(resolution, int) and resolution in self._metric_resolutions: + return resolution + + raise MetricResolutionError( + f"Invalid metric resolution '{resolution}', expected either option: {self._metric_resolutions}", # noqa: E501 + ) + + def _extract_metric_unit_value(self, unit: Union[str, MetricUnit]) -> str: + """Return metric value from metric unit whether that's str or MetricUnit enum + + Parameters + ---------- + unit : Union[str, MetricUnit] + Metric unit + + Returns + ------- + str + Metric unit value (e.g. "Seconds", "Count/Second") + + Raises + ------ + MetricUnitError + When metric unit is not supported by CloudWatch + """ + + if isinstance(unit, str): + if unit in self._metric_unit_valid_options: + unit = MetricUnit[unit].value + + if unit not in self._metric_units: + raise MetricUnitError( + f"Invalid metric unit '{unit}', expected either option: {self._metric_unit_valid_options}", + ) + + if isinstance(unit, MetricUnit): + unit = unit.value + + return unit + + def _add_cold_start_metric(self, context: Any) -> None: + """Add cold start metric and function_name dimension + + Parameters + ---------- + context : Any + Lambda context + """ + if cold_start.is_cold_start: + logger.debug("Adding cold start metric and function_name dimension") + with single_metric(name="ColdStart", unit=MetricUnit.Count, value=1, namespace=self.namespace) as metric: + metric.add_dimension(name="function_name", value=context.function_name) + if self.service: + metric.add_dimension(name="service", value=str(self.service)) + cold_start.is_cold_start = False + + def set_default_dimensions(self, **dimensions) -> None: + """Persist dimensions across Lambda invocations + + Parameters + ---------- + dimensions : Dict[str, Any], optional + metric dimensions as key=value + + Example + ------- + **Sets some default dimensions that will always be present across metrics and invocations** + + from aws_lambda_powertools import Metrics + + metrics = Metrics(namespace="ServerlessAirline", service="payment") + metrics.set_default_dimensions(environment="demo", another="one") + + @metrics.log_metrics() + def lambda_handler(): + return True + """ + for name, value in dimensions.items(): + self.add_dimension(name, value) + + self.default_dimensions.update(**dimensions) diff --git a/aws_lambda_powertools/metrics/provider/cloudwatch_emf/cold_start.py b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/cold_start.py new file mode 100644 index 00000000000..c6ef67bd787 --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/cold_start.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +is_cold_start = True + + +def reset_cold_start_flag(): + global is_cold_start + if not is_cold_start: + is_cold_start = True diff --git a/aws_lambda_powertools/metrics/provider/cloudwatch_emf/constants.py b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/constants.py new file mode 100644 index 00000000000..d8f5da0cec8 --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/constants.py @@ -0,0 +1,2 @@ +MAX_DIMENSIONS = 29 +MAX_METRICS = 100 diff --git a/aws_lambda_powertools/metrics/provider/cloudwatch_emf/exceptions.py b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/exceptions.py new file mode 100644 index 00000000000..6ac2d932ea7 --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/exceptions.py @@ -0,0 +1,10 @@ +class MetricUnitError(Exception): + """When metric unit is not supported by CloudWatch""" + + pass + + +class MetricResolutionError(Exception): + """When metric resolution is not supported by CloudWatch""" + + pass diff --git a/aws_lambda_powertools/metrics/provider/cloudwatch_emf/metric_properties.py b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/metric_properties.py new file mode 100644 index 00000000000..ea11bb997bb --- /dev/null +++ b/aws_lambda_powertools/metrics/provider/cloudwatch_emf/metric_properties.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from enum import Enum + + +class MetricUnit(Enum): + Seconds = "Seconds" + Microseconds = "Microseconds" + Milliseconds = "Milliseconds" + Bytes = "Bytes" + Kilobytes = "Kilobytes" + Megabytes = "Megabytes" + Gigabytes = "Gigabytes" + Terabytes = "Terabytes" + Bits = "Bits" + Kilobits = "Kilobits" + Megabits = "Megabits" + Gigabits = "Gigabits" + Terabits = "Terabits" + Percent = "Percent" + Count = "Count" + BytesPerSecond = "Bytes/Second" + KilobytesPerSecond = "Kilobytes/Second" + MegabytesPerSecond = "Megabytes/Second" + GigabytesPerSecond = "Gigabytes/Second" + TerabytesPerSecond = "Terabytes/Second" + BitsPerSecond = "Bits/Second" + KilobitsPerSecond = "Kilobits/Second" + MegabitsPerSecond = "Megabits/Second" + GigabitsPerSecond = "Gigabits/Second" + TerabitsPerSecond = "Terabits/Second" + CountPerSecond = "Count/Second" + + +class MetricResolution(Enum): + Standard = 60 + High = 1 diff --git a/docs/maintainers.md b/docs/maintainers.md index 455d33f6d8a..8f3a1980141 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -288,6 +288,7 @@ Ensure the repo highlights features that should be elevated to the project roadm Add integration checks that validate pull requests and pushes to ease the burden on Pull Request reviewers. Continuously revisit areas of improvement to reduce operational burden in all parties involved. ### Negative Impact on the Project + Actions that negatively impact the project will be handled by the admins, in coordination with other maintainers, in balance with the urgency of the issue. Examples would be [Code of Conduct](https://github.com/aws-powertools/powertools-lambda-python/blob/develop/CODE_OF_CONDUCT.md){target="_blank"} violations, deliberate harmful or malicious actions, spam, monopolization, and security risks. diff --git a/poetry.lock b/poetry.lock index 3fcd22c4a4f..1f0da79064c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -93,17 +93,17 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-alpha" -version = "2.88.0a0" +version = "2.89.0a0" description = "The CDK Construct Library for AWS::APIGatewayv2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-alpha-2.88.0a0.tar.gz", hash = "sha256:87f746336687ba519263ced055912eff8bef643629d5f1eb17f3cdef2408d441"}, - {file = "aws_cdk.aws_apigatewayv2_alpha-2.88.0a0-py3-none-any.whl", hash = "sha256:be0da36994bedbbc4b82db245fa5f3fae59f95665c9d60527f7ece5e9b4df6a4"}, + {file = "aws-cdk.aws-apigatewayv2-alpha-2.89.0a0.tar.gz", hash = "sha256:8300431d4ef9d869066ad5dba955a6b9eca4825eb4ffcdb03d9ce34f82509d6a"}, + {file = "aws_cdk.aws_apigatewayv2_alpha-2.89.0a0-py3-none-any.whl", hash = "sha256:64a84542822bd085b03ac40e39f15c3fee1aaf649a0df34ecf0f288f7bc84c78"}, ] [package.dependencies] -aws-cdk-lib = "2.88.0" +aws-cdk-lib = "2.89.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.85.0,<2.0.0" publication = ">=0.0.3" @@ -111,18 +111,18 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-authorizers-alpha" -version = "2.88.0a0" +version = "2.89.0a0" description = "Authorizers for AWS APIGateway V2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.88.0a0.tar.gz", hash = "sha256:60f3a0ac560b6f3ff729b50110841f134eecd842bf69d02602e750589a35cbff"}, - {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.88.0a0-py3-none-any.whl", hash = "sha256:85e57c5a7a86829a594634f82448c95443d4d29c30baf361257e57fd4df24efc"}, + {file = "aws-cdk.aws-apigatewayv2-authorizers-alpha-2.89.0a0.tar.gz", hash = "sha256:efa23f021efdca83f037569d41d7e96023c3750417fc976023688397f7f57715"}, + {file = "aws_cdk.aws_apigatewayv2_authorizers_alpha-2.89.0a0-py3-none-any.whl", hash = "sha256:7b56ea2889e8a340bfd4feb67f0798827bf58090d368763a59cd0223fe2dd916"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.88.0.a0" -aws-cdk-lib = "2.88.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.89.0.a0" +aws-cdk-lib = "2.89.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.85.0,<2.0.0" publication = ">=0.0.3" @@ -130,18 +130,18 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-aws-apigatewayv2-integrations-alpha" -version = "2.88.0a0" +version = "2.89.0a0" description = "Integrations for AWS APIGateway V2" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.88.0a0.tar.gz", hash = "sha256:a604acb1dde9840ccc24c23aba542b42764c826c8100b787e16198113d6b6e89"}, - {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.88.0a0-py3-none-any.whl", hash = "sha256:ff06fc8192bece85f82f7b008c93e5ada8af1466612b0b76287edce8c5415c47"}, + {file = "aws-cdk.aws-apigatewayv2-integrations-alpha-2.89.0a0.tar.gz", hash = "sha256:81469d688a611d9ab10d528923692eba685cbb04a5d3401c02a4530b001a6a77"}, + {file = "aws_cdk.aws_apigatewayv2_integrations_alpha-2.89.0a0-py3-none-any.whl", hash = "sha256:3367cf5fa8e4bb1939fcd60e919af00ecc6d97a1d046938af25b9c5bef26b4c1"}, ] [package.dependencies] -"aws-cdk.aws-apigatewayv2-alpha" = "2.88.0.a0" -aws-cdk-lib = "2.88.0" +"aws-cdk.aws-apigatewayv2-alpha" = "2.89.0.a0" +aws-cdk-lib = "2.89.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.85.0,<2.0.0" publication = ">=0.0.3" @@ -149,19 +149,19 @@ typeguard = ">=2.13.3,<2.14.0" [[package]] name = "aws-cdk-lib" -version = "2.88.0" +version = "2.89.0" description = "Version 2 of the AWS Cloud Development Kit library" optional = false python-versions = "~=3.7" files = [ - {file = "aws-cdk-lib-2.88.0.tar.gz", hash = "sha256:6514217e6485133b30848f11b6c78ec955d41bed4e39e5ba4bea49c379830e56"}, - {file = "aws_cdk_lib-2.88.0-py3-none-any.whl", hash = "sha256:db4716689cb94e249b8c672139221f1d7866b7d6caca314b52552fa3bacab22c"}, + {file = "aws-cdk-lib-2.89.0.tar.gz", hash = "sha256:8fbd1d4ee0ffeb67bcc845bef5a10575dbc678ad07f74cdb3cb4243afc433db7"}, + {file = "aws_cdk_lib-2.89.0-py3-none-any.whl", hash = "sha256:92eeebd77fe17b36029fae20f46eb601710485ea7c808c3d33fe1c71fee125bd"}, ] [package.dependencies] "aws-cdk.asset-awscli-v1" = ">=2.2.200,<3.0.0" "aws-cdk.asset-kubectl-v20" = ">=2.1.2,<3.0.0" -"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.165,<3.0.0" +"aws-cdk.asset-node-proxy-agent-v5" = ">=2.0.166,<3.0.0" constructs = ">=10.0.0,<11.0.0" jsii = ">=1.85.0,<2.0.0" publication = ">=0.0.3" @@ -183,13 +183,13 @@ requests = ">=0.14.0" [[package]] name = "aws-sam-translator" -version = "1.71.0" +version = "1.72.0" description = "AWS SAM Translator is a library that transform SAM templates into AWS CloudFormation templates" optional = false python-versions = ">=3.7, <=4.0, !=4.0" files = [ - {file = "aws-sam-translator-1.71.0.tar.gz", hash = "sha256:a3ea80aeb116d7978b26ac916d2a5a24d012b742bf28262b17769c4b886e8fba"}, - {file = "aws_sam_translator-1.71.0-py3-none-any.whl", hash = "sha256:17fb87c8137d8d49e7a978396b2b3b279211819dee44618415aab1e99c2cb659"}, + {file = "aws-sam-translator-1.72.0.tar.gz", hash = "sha256:e688aac30943bfe0352147b792d8bbe7c1b5ed648747cd7ef6280875b249e2d8"}, + {file = "aws_sam_translator-1.72.0-py3-none-any.whl", hash = "sha256:69fe3914d61ae6690034c3fc1055743e5415d83c59c35ec5ec9ceb26cc65c8a1"}, ] [package.dependencies] @@ -291,17 +291,17 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "boto3" -version = "1.28.9" +version = "1.28.16" description = "The AWS SDK for Python" optional = false python-versions = ">= 3.7" files = [ - {file = "boto3-1.28.9-py3-none-any.whl", hash = "sha256:01f078047eb4d238c6b9c6cc623f2af33b4ae67980c5326691e35cb5493ff6c7"}, - {file = "boto3-1.28.9.tar.gz", hash = "sha256:4cc0c6005be910e52077227e670930ab55a41ba86cdb6d1c052571d08cd4d32c"}, + {file = "boto3-1.28.16-py3-none-any.whl", hash = "sha256:d8e31f69fb919025a5961f8fbeb51fe92e2f753beb37fc1853138667a231cdaa"}, + {file = "boto3-1.28.16.tar.gz", hash = "sha256:aea48aedf3e8676e598e3202e732295064a4fcad5f2d2d2a699368b8c3ab492c"}, ] [package.dependencies] -botocore = ">=1.31.9,<1.32.0" +botocore = ">=1.31.16,<1.32.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.6.0,<0.7.0" @@ -310,13 +310,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.31.9" +version = "1.31.16" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">= 3.7" files = [ - {file = "botocore-1.31.9-py3-none-any.whl", hash = "sha256:e56ccd3536a90094ea5b176b5dd33bfe4f049efdf71af468ea1661bd424c787d"}, - {file = "botocore-1.31.9.tar.gz", hash = "sha256:bd849d3ac95f1781385ed831d753a04a3ec870a59d6598175aaedd71dc2baf5f"}, + {file = "botocore-1.31.16-py3-none-any.whl", hash = "sha256:92b240e2cb7b3afae5361651d2f48ee582f45d2dab53aef76eef7eec1d3ce582"}, + {file = "botocore-1.31.16.tar.gz", hash = "sha256:563e15979e763b93d78de58d0fc065f8615be12f41bab42f5ad9f412b6a224b3"}, ] [package.dependencies] @@ -1143,19 +1143,20 @@ restructuredtext = ["rst2ansi"] [[package]] name = "markdown" -version = "3.3.7" -description = "Python implementation of Markdown." +version = "3.4.4" +description = "Python implementation of John Gruber's Markdown." optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" files = [ - {file = "Markdown-3.3.7-py3-none-any.whl", hash = "sha256:f5da449a6e1c989a4cea2631aa8ee67caa5a2ef855d551c88f9e309f4634c621"}, - {file = "Markdown-3.3.7.tar.gz", hash = "sha256:cbb516f16218e643d8e0a95b309f77eb118cb138d39a4f27851e6a63581db874"}, + {file = "Markdown-3.4.4-py3-none-any.whl", hash = "sha256:a4c1b65c0957b4bd9e7d86ddc7b3c9868fb9670660f6f99f6d1bca8954d5a941"}, + {file = "Markdown-3.4.4.tar.gz", hash = "sha256:225c6123522495d4119a90b3a3ba31a1e87a70369e03f14799ea9c0d7183a3d6"}, ] [package.dependencies] importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} [package.extras] +docs = ["mdx-gh-links (>=0.2)", "mkdocs (>=1.0)", "mkdocs-nature (>=0.4)"] testing = ["coverage", "pyyaml"] [[package]] @@ -1625,13 +1626,13 @@ files = [ [[package]] name = "pathspec" -version = "0.11.1" +version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.7" files = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, + {file = "pathspec-0.11.2-py3-none-any.whl", hash = "sha256:1d6ed233af05e679efb96b1851550ea95bbb64b7c490b0f5aa52996c11e92a20"}, + {file = "pathspec-0.11.2.tar.gz", hash = "sha256:e0d8d0ac2f12da61956eb2306b69f9469b42f4deb0f3cb6ed47b9cce9996ced3"}, ] [[package]] @@ -1673,21 +1674,21 @@ files = [ [[package]] name = "platformdirs" -version = "3.9.1" +version = "3.10.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." optional = false python-versions = ">=3.7" files = [ - {file = "platformdirs-3.9.1-py3-none-any.whl", hash = "sha256:ad8291ae0ae5072f66c16945166cb11c63394c7a3ad1b1bc9828ca3162da8c2f"}, - {file = "platformdirs-3.9.1.tar.gz", hash = "sha256:1b42b450ad933e981d56e59f1b97495428c9bd60698baab9f3eb3d00d5822421"}, + {file = "platformdirs-3.10.0-py3-none-any.whl", hash = "sha256:d7c24979f292f916dc9cbf8648319032f551ea8c49a4c9bf2fb556a02070ec1d"}, + {file = "platformdirs-3.10.0.tar.gz", hash = "sha256:b45696dab2d7cc691a3226759c0d3b00c47c8b6e293d96f6436f733303f77f6d"}, ] [package.dependencies] -typing-extensions = {version = ">=4.6.3", markers = "python_version < \"3.8\""} +typing-extensions = {version = ">=4.7.1", markers = "python_version < \"3.8\""} [package.extras] -docs = ["furo (>=2023.5.20)", "proselint (>=0.13)", "sphinx (>=7.0.1)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4.1)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.1)", "sphinx-autodoc-typehints (>=1.24)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-cov (>=4.1)", "pytest-mock (>=3.11.1)"] [[package]] name = "pluggy" @@ -2199,13 +2200,13 @@ decorator = ">=3.4.2" [[package]] name = "rich" -version = "13.4.2" +version = "13.5.2" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.7.0" files = [ - {file = "rich-13.4.2-py3-none-any.whl", hash = "sha256:8f87bc7ee54675732fa66a05ebfe489e27264caeeff3728c945d25971b6485ec"}, - {file = "rich-13.4.2.tar.gz", hash = "sha256:d653d6bccede5844304c605d5aac802c7cf9621efd700b46c7ec2b51ea914898"}, + {file = "rich-13.5.2-py3-none-any.whl", hash = "sha256:146a90b3b6b47cac4a73c12866a499e9817426423f57c5a66949c086191a8808"}, + {file = "rich-13.5.2.tar.gz", hash = "sha256:fb9d6c0a0f643c99eed3875b5377a184132ba9be4d61516a55273d3554d75a39"}, ] [package.dependencies] @@ -2276,13 +2277,13 @@ pbr = "*" [[package]] name = "sentry-sdk" -version = "1.29.0" +version = "1.29.2" description = "Python client for Sentry (https://sentry.io)" optional = false python-versions = "*" files = [ - {file = "sentry-sdk-1.29.0.tar.gz", hash = "sha256:5053aa7647533b207417cea5f3ee2d5e13aea7001dbb711f46348d7a72597d60"}, - {file = "sentry_sdk-1.29.0-py2.py3-none-any.whl", hash = "sha256:78c764cbec1967c41e24967e9a4a5d1e67a7bc960d125d905f94c068a0300ac9"}, + {file = "sentry-sdk-1.29.2.tar.gz", hash = "sha256:a99ee105384788c3f228726a88baf515fe7b5f1d2d0f215a03d194369f158df7"}, + {file = "sentry_sdk-1.29.2-py2.py3-none-any.whl", hash = "sha256:3e17215d8006612e2df02b0e73115eb8376c37e3f586d8436fa41644e605074d"}, ] [package.dependencies] diff --git a/ruff.toml b/ruff.toml index 424040ede1f..be67606bbe5 100644 --- a/ruff.toml +++ b/ruff.toml @@ -69,3 +69,5 @@ split-on-trailing-comma = true "tests/e2e/utils/data_fetcher/__init__.py" = ["F401"] "aws_lambda_powertools/utilities/data_classes/s3_event.py" = ["A003"] "aws_lambda_powertools/utilities/parser/models/__init__.py" = ["E402"] +# Maintenance: we're keeping EphemeralMetrics code in case of Hyrum's law so we can quickly revert it +"aws_lambda_powertools/metrics/metrics.py" = ["ERA001"] diff --git a/tests/functional/test_metrics.py b/tests/functional/test_metrics.py index 5a6222f248d..1eed6c82294 100644 --- a/tests/functional/test_metrics.py +++ b/tests/functional/test_metrics.py @@ -5,21 +5,25 @@ import pytest -from aws_lambda_powertools import Metrics, single_metric from aws_lambda_powertools.metrics import ( EphemeralMetrics, MetricResolution, MetricResolutionError, + Metrics, MetricUnit, MetricUnitError, MetricValueError, SchemaValidationError, + single_metric, ) -from aws_lambda_powertools.metrics.base import ( - MAX_DIMENSIONS, - MetricManager, - reset_cold_start_flag, +from aws_lambda_powertools.metrics.provider import ( + MetricsBase, + MetricsProviderBase, ) +from aws_lambda_powertools.metrics.provider.base import reset_cold_start_flag_provider +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.cloudwatch import AmazonCloudWatchEMFProvider +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.cold_start import reset_cold_start_flag +from aws_lambda_powertools.metrics.provider.cloudwatch_emf.constants import MAX_DIMENSIONS @pytest.fixture(scope="function", autouse=True) @@ -110,7 +114,7 @@ def serialize_metrics( metadatas: List[Dict] = None, ) -> Dict: """Helper function to build EMF object from a list of metrics, dimensions""" - my_metrics = MetricManager(namespace=namespace) + my_metrics = AmazonCloudWatchEMFProvider(namespace=namespace) for dimension in dimensions: my_metrics.add_dimension(**dimension) @@ -127,7 +131,7 @@ def serialize_metrics( def serialize_single_metric(metric: Dict, dimension: Dict, namespace: str, metadata: Dict = None) -> Dict: """Helper function to build EMF object from a given metric, dimension and namespace""" - my_metrics = MetricManager(namespace=namespace) + my_metrics = AmazonCloudWatchEMFProvider(namespace=namespace) my_metrics.add_metric(**metric) my_metrics.add_dimension(**dimension) @@ -228,6 +232,27 @@ def test_single_metric_default_dimensions_inherit(capsys, metric, dimension, nam assert expected == output +def test_log_metrics_preconfigured_provider(capsys, metrics, dimensions, namespace): + # GIVEN Metrics is initialized + provider = AmazonCloudWatchEMFProvider(namespace=namespace) + my_metrics = Metrics(provider=provider) + for metric in metrics: + my_metrics.add_metric(**metric) + for dimension in dimensions: + my_metrics.add_dimension(**dimension) + + # WHEN we manually the metrics + my_metrics.flush_metrics() + + output = capture_metrics_output(capsys) + expected = serialize_metrics(metrics=metrics, dimensions=dimensions, namespace=namespace) + + # THEN we should have no exceptions + # and a valid EMF object should be flushed correctly + remove_timestamp(metrics=[output, expected]) + assert expected == output + + def test_log_metrics(capsys, metrics, dimensions, namespace): # GIVEN Metrics is initialized my_metrics = Metrics(namespace=namespace) @@ -1010,7 +1035,7 @@ def test_metric_manage_metadata_set(): expected_dict = {"setting": "On"} try: - metric = MetricManager(metadata_set=expected_dict) + metric = AmazonCloudWatchEMFProvider(metadata_set=expected_dict) assert metric.metadata_set == expected_dict except AttributeError: pytest.fail("AttributeError should not be raised") @@ -1052,6 +1077,20 @@ def test_clear_default_dimensions(namespace): assert not my_metrics.default_dimensions +def test_clear_default_dimensions_with_provider(namespace): + # GIVEN Metrics is initialized with provider and we persist a set of default dimensions + my_provider = AmazonCloudWatchEMFProvider(namespace=namespace) + my_metrics = Metrics(provider=my_provider) + my_metrics.set_default_dimensions(environment="test", log_group="/lambda/test") + + # WHEN they are removed via clear_default_dimensions method + my_metrics.clear_default_dimensions() + + # THEN there should be no default dimensions in provider and metrics + assert not my_metrics.default_dimensions + assert not my_provider.default_dimensions + + def test_default_dimensions_across_instances(namespace): # GIVEN Metrics is initialized and we persist a set of default dimensions my_metrics = Metrics(namespace=namespace) @@ -1143,6 +1182,7 @@ def test_ephemeral_metrics_isolated_data_set_with_default_dimension(metric, dime # GIVEN two EphemeralMetrics instances are initialized # One with default dimension and another without my_metrics = EphemeralMetrics(namespace=namespace) + my_metrics.set_default_dimensions(dev="powertools") isolated_metrics = EphemeralMetrics(namespace=namespace) @@ -1211,3 +1251,159 @@ def lambda_handler(evt, ctx): output = capture_metrics_output_multiple_emf_objects(capsys) assert len(output) == 2 + + +@pytest.fixture +def metrics_provider() -> MetricsProviderBase: + class MetricsProvider: + def __init__(self): + self.metric_store: List = [] + self.result: str + super().__init__() + + def add_metric(self, name: str, value: float, tag: List = None, *args, **kwargs): + self.metric_store.append({"name": name, "value": value, "tag": tag}) + + def serialize(self, raise_on_empty_metrics: bool = False, *args, **kwargs): + if raise_on_empty_metrics and len(self.metric_store) == 0: + raise SchemaValidationError("Must contain at least one metric.") + + self.result = json.dumps(self.metric_store) + + def flush(self, *args, **kwargs): + print(self.result) + + def clear(self): + self.result = "" + self.metric_store = [] + + return MetricsProvider + + +@pytest.fixture +def metrics_class() -> MetricsBase: + class MetricsClass(MetricsBase): + def __init__(self, provider): + self.provider = provider + super().__init__() + + def add_metric(self, name: str, value: float, tag: List = None, *args, **kwargs): + self.provider.add_metric(name=name, value=value, tag=tag) + + def flush_metrics(self, raise_on_empty_metrics: bool = False) -> None: + self.provider.serialize(raise_on_empty_metrics=raise_on_empty_metrics) + self.provider.flush() + self.provider.clear() + + def add_cold_start_metric(self, metric_name: str, function_name: str) -> None: + self.provider.add_metric(name=metric_name, value=1, function_name=function_name) + + return MetricsClass + + +def test_metrics_provider_basic(capsys, metrics_provider, metric): + provider = metrics_provider() + provider.add_metric(**metric) + provider.serialize() + provider.flush() + output = capture_metrics_output(capsys) + assert output[0]["name"] == metric["name"] + assert output[0]["value"] == metric["value"] + + +def test_metrics_provider_class_basic(capsys, metrics_provider, metrics_class, metric): + metrics = metrics_class(provider=metrics_provider()) + metrics.add_metric(**metric) + metrics.flush_metrics() + output = capture_metrics_output(capsys) + assert output[0]["name"] == metric["name"] + assert output[0]["value"] == metric["value"] + + +def test_metrics_provider_class_decorate(metrics_class, metrics_provider): + # GIVEN Metrics is initialized + my_metrics = metrics_class(provider=metrics_provider()) + + # WHEN log_metrics is used to serialize metrics + @my_metrics.log_metrics + def lambda_handler(evt, context): + return True + + # THEN log_metrics should invoke the function it decorates + # and return no error if we have a namespace and dimension + assert lambda_handler({}, {}) is True + + +def test_metrics_provider_class_coldstart(capsys, metrics_provider, metrics_class): + my_metrics = metrics_class(provider=metrics_provider()) + + # WHEN log_metrics is used with capture_cold_start_metric + @my_metrics.log_metrics(capture_cold_start_metric=True) + def lambda_handler(evt, context): + pass + + LambdaContext = namedtuple("LambdaContext", "function_name") + lambda_handler({}, LambdaContext("example_fn")) + + output = capture_metrics_output(capsys) + + # THEN ColdStart metric and function_name and service dimension should be logged + assert output[0]["name"] == "ColdStart" + + +def test_metrics_provider_class_no_coldstart(capsys, metrics_provider, metrics_class): + reset_cold_start_flag_provider() + my_metrics = metrics_class(provider=metrics_provider()) + + # WHEN log_metrics is used with capture_cold_start_metric + @my_metrics.log_metrics(capture_cold_start_metric=True) + def lambda_handler(evt, context): + pass + + LambdaContext = namedtuple("LambdaContext", "function_name") + lambda_handler({}, LambdaContext("example_fn")) + _ = capture_metrics_output(capsys) + # drop first one + + lambda_handler({}, LambdaContext("example_fn")) + output = capture_metrics_output(capsys) + + # no coldstart is here + assert "ColdStart" not in json.dumps(output) + + +def test_metric_provider_raise_on_empty_metrics(metrics_provider, metrics_class): + # GIVEN Metrics is initialized + my_metrics = metrics_class(provider=metrics_provider()) + + # WHEN log_metrics is used with raise_on_empty_metrics param and has no metrics + @my_metrics.log_metrics(raise_on_empty_metrics=True) + def lambda_handler(evt, context): + pass + + # THEN the raised exception should be SchemaValidationError + # and specifically about the lack of Metrics + with pytest.raises(SchemaValidationError, match="Must contain at least one metric."): + lambda_handler({}, {}) + + +def test_log_metrics_capture_cold_start_metric_once_with_provider_and_ephemeral(capsys, namespace, service): + # GIVEN Metrics is initialized + my_metrics = Metrics(service=service, namespace=namespace) + my_isolated_metrics = EphemeralMetrics(service=service, namespace=namespace) + + # WHEN log_metrics is used with capture_cold_start_metric + @my_metrics.log_metrics(capture_cold_start_metric=True) + @my_isolated_metrics.log_metrics(capture_cold_start_metric=True) + def lambda_handler(evt, context): + pass + + LambdaContext = namedtuple("LambdaContext", "function_name") + lambda_handler({}, LambdaContext("example_fn")) + + output = capture_metrics_output(capsys) + + # THEN ColdStart metric and function_name and service dimension should be logged + assert output["ColdStart"] == [1.0] + assert output["function_name"] == "example_fn" + assert output["service"] == service