Merge branch 'develop' into boto

* develop: (33 commits) fix(docs): remove Slack link (aws-powertools#1210) docs(roadmap): add new roadmap section (aws-powertools#1204) fix(batch): missing space in BatchProcessingError message (aws-powertools#1201) fix(batch): docstring fix for success_handler() record parameter (aws-powertools#1202) chore(deps): bump email-validator from 1.1.3 to 1.2.1 (aws-powertools#1199) docs(layer): upgrade to 1.25.10 chore: include regression in changelog chore: bump to 1.25.10 feat(parameters): add clear_cache method for providers (aws-powertools#1194) chore(ci): changelog pre-generation to fetch tags from origin revert(parser): Add missing fields for SESEvent (aws-powertools#1027) (aws-powertools#1190) fix(parser): Add missing fields for SESEvent (aws-powertools#1027) fix(data-classes): Add missing SES fields and (aws-powertools#1045) fix(event_handler): exception_handler to handle ServiceError exceptions (aws-powertools#1160) chore(deps-dev): bump flake8-bugbear from 22.1.11 to 22.4.25 (aws-powertools#1156) chore(deps-dev): bump mypy from 0.942 to 0.950 (aws-powertools#1162) chore(deps-dev): bump flake8-eradicate from 1.2.0 to 1.2.1 (aws-powertools#1158) chore(deps): bump github/codeql-action from 1 to 2 (aws-powertools#1154) chore(ci): post release on tagged issues too fix(event_handler): Allow for event_source support (aws-powertools#1159) ...
ran-isenberg · May 18, 2022 · 2a77841 · 2a77841
2 parents ccc378b + 0777858
commit 2a77841
Show file tree

Hide file tree

Showing 25 changed files with 574 additions and 135 deletions.
diff --git a/.github/dependabot.yml b/.github/dependabot.yml
@@ -17,6 +17,10 @@ updates:
     commit-message:
       prefix: chore
       include: scope
+    ignore:
+      # 2022-04-23: Ignoring boto3 changes until we need to care about them.
+      - dependency-name: "boto3"
+
 
 #   - package-ecosystem: "pip"
 #     directory: "/"

diff --git a/.github/mergify.yml b/.github/mergify.yml
@@ -1,29 +1,31 @@
-queue_rules:
-  - name: default
-    conditions:
-      # Conditions to get out of the queue (= merged)
-      - check-success=Semantic Pull Request
-      - "#approved-reviews-by>=1"
-      - -title~=(WIP|wip)
-      - -label~="do-not-merge"
-      - "#changes-requested-reviews-by=0"
+## Temporarily disabled after Mergify breaking changes
+## might move to custom GitHub Actions altogether
+# queue_rules:
+#   - name: default
+#     conditions:
+#       # Conditions to get out of the queue (= merged)
+#       - check-success=Semantic Pull Request
+#       - "#approved-reviews-by>=1"
+#       - -title~=(WIP|wip)
+#       - -label~="do-not-merge"
+#       - "#changes-requested-reviews-by=0"
 
-pull_request_rules:
-  - name: automatic merge for Dependabot pull requests
-    conditions:
-      - author~=^dependabot(|-preview)\[bot\]$
-    actions:
-      queue:
-        name: default
-        method: squash
-        commit_message: title+body
+# pull_request_rules:
+#   - name: automatic merge for Dependabot pull requests
+#     conditions:
+#       - author~=^dependabot(|-preview)\[bot\]$
+#     actions:
+#       queue:
+#         name: default
+#         method: squash
+#         commit_message: title+body
 
-  - name: Automatic merge ⬇️ on approval ✔
-    conditions:
-      - base!=master
-      - "#approved-reviews-by>=2"
-    actions:
-      queue:
-        name: default
-        method: squash
-        commit_message: title+body
+#   - name: Automatic merge ⬇️ on approval ✔
+#     conditions:
+#       - base!=master
+#       - "#approved-reviews-by>=2"
+#     actions:
+#       queue:
+#         name: default
+#         method: squash
+#         commit_message: title+body
diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml
@@ -24,7 +24,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v1
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -33,4 +33,4 @@ jobs:
         # queries: ./path/to/local/query, your-org/your-repo/queries@main
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v1
+      uses: github/codeql-action/analyze@v2
diff --git a/.github/workflows/post_release.js b/.github/workflows/post_release.js
@@ -15,7 +15,7 @@ const fetchIssues = async ({
 	gh_client,
 	org,
 	repository,
-	state = "open",
+	state = "all",
 	label = STAGED_LABEL,
 }) => {
 

diff --git a/.github/workflows/python_build.yml b/.github/workflows/python_build.yml
@@ -39,7 +39,7 @@ jobs:
       - name: Complexity baseline
         run: make complexity-baseline
       - name: Upload coverage to Codecov
-        uses: codecov/codecov-action@v3.0.0
+        uses: codecov/codecov-action@v3.1.0
         with:
           file: ./coverage.xml
           # flags: unittests

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,69 @@ All notable changes to this project will be documented in this file.
 
 This project follows [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format for changes and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 1.25.10 - 2022-04-29
+
+### Bug Fixes
+
+* **data-classes:** Add missing SES fields and ([#1045](https://github.com/awslabs/aws-lambda-powertools-python/issues/1045))
+* **deps:** Ignore boto3 changes until needed ([#1151](https://github.com/awslabs/aws-lambda-powertools-python/issues/1151))
+* **deps-dev:** remove jmespath due to dev deps conflict  ([#1148](https://github.com/awslabs/aws-lambda-powertools-python/issues/1148))
+* **event_handler:** exception_handler to handle ServiceError exceptions ([#1160](https://github.com/awslabs/aws-lambda-powertools-python/issues/1160))
+* **event_handler:** Allow for event_source support ([#1159](https://github.com/awslabs/aws-lambda-powertools-python/issues/1159))
+* **parser:** Add missing fields for SESEvent ([#1027](https://github.com/awslabs/aws-lambda-powertools-python/issues/1027))
+
+### Documentation
+
+* **layer:** upgrade to 1.25.9
+
+### Features
+
+* **parameters:** add clear_cache method for providers ([#1194](https://github.com/awslabs/aws-lambda-powertools-python/issues/1194))
+
+### Maintenance
+
+* **ci:** changelog pre-generation to fetch tags from origin
+* **ci:** post release on tagged issues too
+* **ci:** disable mergify configuration after breaking changes ([#1188](https://github.com/awslabs/aws-lambda-powertools-python/issues/1188))
+* **deps:** bump codecov/codecov-action from 3.0.0 to 3.1.0 ([#1143](https://github.com/awslabs/aws-lambda-powertools-python/issues/1143))
+* **deps:** bump github/codeql-action from 1 to 2 ([#1154](https://github.com/awslabs/aws-lambda-powertools-python/issues/1154))
+* **deps-dev:** bump flake8-bugbear from 22.1.11 to 22.4.25 ([#1156](https://github.com/awslabs/aws-lambda-powertools-python/issues/1156))
+* **deps-dev:** bump flake8-eradicate from 1.2.0 to 1.2.1 ([#1158](https://github.com/awslabs/aws-lambda-powertools-python/issues/1158))
+* **deps-dev:** bump mkdocs-git-revision-date-plugin ([#1146](https://github.com/awslabs/aws-lambda-powertools-python/issues/1146))
+* **deps-dev:** bump mypy from 0.942 to 0.950 ([#1162](https://github.com/awslabs/aws-lambda-powertools-python/issues/1162))
+* **deps-dev:** bump xenon from 0.8.0 to 0.9.0 ([#1145](https://github.com/awslabs/aws-lambda-powertools-python/issues/1145))
+* **deps-dev:** bump mypy from 0.931 to 0.942 ([#1133](https://github.com/awslabs/aws-lambda-powertools-python/issues/1133))
+
+### Regression
+
+* **parser:** Add missing fields for SESEvent ([#1027](https://github.com/awslabs/aws-lambda-powertools-python/issues/1027)) ([#1190](https://github.com/awslabs/aws-lambda-powertools-python/issues/1190))
+
+## 1.25.9 - 2022-04-21
+
+### Bug Fixes
+
+* **deps**: correct py36 marker for jmespath
+## 1.25.8 - 2022-04-21
+
+### Bug Fixes
+
+* **governance:** removed ambiguous quotes from labels
+* **deps:** update jmespath marker to support 1.0 and py3.6 ([#1139](https://github.com/awslabs/aws-lambda-powertools-python/issues/1139))
+* **governance:** update label in names in issues
+
+### Documentation
+
+* **install:** instructions to reduce pydantic package size ([#1077](https://github.com/awslabs/aws-lambda-powertools-python/issues/1077))
+* **layer:** remove link from clipboard button ([#1135](https://github.com/awslabs/aws-lambda-powertools-python/issues/1135))
+* **layer:** update to 1.25.7
+
+### Maintenance
+
+* **deps:** bump codecov/codecov-action from 2.1.0 to 3.0.0 ([#1102](https://github.com/awslabs/aws-lambda-powertools-python/issues/1102))
+* **deps:** bump actions/upload-artifact from 2 to 3 ([#1103](https://github.com/awslabs/aws-lambda-powertools-python/issues/1103))
+* **deps-dev:** bump mkdocs-material from 8.2.4 to 8.2.7 ([#1131](https://github.com/awslabs/aws-lambda-powertools-python/issues/1131))
+* **deps-dev:** bump pytest from 6.2.5 to 7.0.1 ([#1063](https://github.com/awslabs/aws-lambda-powertools-python/issues/1063))
+
 ## 1.25.7 - 2022-04-08
 
 ### Bug Fixes

diff --git a/Makefile b/Makefile
@@ -84,7 +84,7 @@ release: pr
 	$(MAKE) release-prod
 
 changelog:
-	git fetch --tags
+	git fetch --tags origin
 	@echo "[+] Pre-generating CHANGELOG for tag: $$(git describe --abbrev=0 --tag)"
 	docker run -v "${PWD}":/workdir quay.io/git-chglog/git-chglog $$(git describe --abbrev=0 --tag).. > TMP_CHANGELOG.md
 

diff --git a/README.md b/README.md
@@ -1,12 +1,12 @@
-# AWS Lambda Powertools (Python)
+# AWS Lambda Powertools for Python
 
 ![Build](https://github.com/awslabs/aws-lambda-powertools/workflows/Powertools%20Python/badge.svg?branch=master)
 [![codecov.io](https://codecov.io/github/awslabs/aws-lambda-powertools-python/branch/develop/graphs/badge.svg)](https://app.codecov.io/gh/awslabs/aws-lambda-powertools-python)
 ![PythonSupport](https://img.shields.io/static/v1?label=python&message=3.6%20|%203.7|%203.8|%203.9&color=blue?style=flat-square&logo=python) ![PyPI version](https://badge.fury.io/py/aws-lambda-powertools.svg) ![PyPi monthly downloads](https://img.shields.io/pypi/dm/aws-lambda-powertools)
 
 A suite of Python utilities for AWS Lambda functions to ease adopting best practices such as tracing, structured logging, custom metrics, and more. (AWS Lambda Powertools [Java](https://github.com/awslabs/aws-lambda-powertools-java) and [Typescript](https://github.com/awslabs/aws-lambda-powertools-typescript) is also available).
 
-**[📜Documentation](https://awslabs.github.io/aws-lambda-powertools-python/)** | **[🐍PyPi](https://pypi.org/project/aws-lambda-powertools/)** | **[Roadmap](https://github.com/awslabs/aws-lambda-powertools-roadmap/projects/1)** | **[Detailed blog post](https://aws.amazon.com/blogs/opensource/simplifying-serverless-best-practices-with-lambda-powertools/)**
+**[📜Documentation](https://awslabs.github.io/aws-lambda-powertools-python/)** | **[🐍PyPi](https://pypi.org/project/aws-lambda-powertools/)** | **[Roadmap](https://awslabs.github.io/aws-lambda-powertools-python/latest/roadmap/)** | **[Detailed blog post](https://aws.amazon.com/blogs/opensource/simplifying-serverless-best-practices-with-lambda-powertools/)**
 
 > **An AWS Developer Acceleration (DevAx) initiative by Specialist Solution Architects | aws-devax-open-source@amazon.com**
 
@@ -47,9 +47,11 @@ With [pip](https://pip.pypa.io/en/latest/index.html) installed, run: ``pip insta
 * Powertools idea [DAZN Powertools](https://github.com/getndazn/dazn-lambda-powertools/)
 
 ## Connect
+**Email**: aws-lambda-powertools-feedback@amazon.com
 
-* **AWS Developers Slack**: `#lambda-powertools` - **[Invite, if you don't have an account](https://join.slack.com/t/awsdevelopers/shared_invite/zt-yryddays-C9fkWrmguDv0h2EEDzCqvw)**
-* **Email**: aws-lambda-powertools-feedback@amazon.com
+## Security disclosures
+
+If you think you’ve found a potential security issue, please do not post it in the Issues.  Instead, please follow the instructions [here](https://aws.amazon.com/security/vulnerability-reporting/) or [email AWS security directly](mailto:aws-security@amazon.com).
 
 ## License
 

diff --git a/aws_lambda_powertools/event_handler/api_gateway.py b/aws_lambda_powertools/event_handler/api_gateway.py
@@ -491,6 +491,11 @@ def resolve(self, event, context) -> Dict[str, Any]:
         dict
             Returns the dict response
         """
+        if isinstance(event, BaseProxyEvent):
+            warnings.warn(
+                "You don't need to serialize event to Event Source Data Class when using Event Handler; see issue #1152"
+            )
+            event = event.raw_event
         if self._debug:
             print(self._json_dump(event), end="")
         BaseRouter.current_event = self._to_proxy_event(event)
@@ -651,7 +656,10 @@ def _lookup_exception_handler(self, exp_type: Type) -> Optional[Callable]:
     def _call_exception_handler(self, exp: Exception, route: Route) -> Optional[ResponseBuilder]:
         handler = self._lookup_exception_handler(type(exp))
         if handler:
-            return ResponseBuilder(handler(exp), route)
+            try:
+                return ResponseBuilder(handler(exp), route)
+            except ServiceError as service_error:
+                exp = service_error
 
         if isinstance(exp, ServiceError):
             return ResponseBuilder(

diff --git a/aws_lambda_powertools/utilities/batch/base.py b/aws_lambda_powertools/utilities/batch/base.py
@@ -116,7 +116,7 @@ def success_handler(self, record, result: Any) -> SuccessResponse:
         Parameters
         ----------
         record: Any
-            record that failed processing
+            record that succeeded processing
         result: Any
             result from record handler
 
@@ -380,7 +380,7 @@ def _clean(self):
 
         if self._entire_batch_failed():
             raise BatchProcessingError(
-                msg=f"All records failed processing. {len(self.exceptions)} individual errors logged"
+                msg=f"All records failed processing. {len(self.exceptions)} individual errors logged "
                 f"separately below.",
                 child_exceptions=self.exceptions,
             )

diff --git a/aws_lambda_powertools/utilities/batch/sqs.py b/aws_lambda_powertools/utilities/batch/sqs.py
@@ -135,6 +135,10 @@ def _clean(self) -> Optional[List]:
             return None
 
         queue_url = self._get_queue_url()
+        if queue_url is None:
+            logger.debug("No queue url found")
+            return None
+
         entries_to_remove = self._get_entries_to_clean()
         # Batch delete up to 10 messages at a time (SQS limit)
         max_workers = math.ceil(len(entries_to_remove) / self.max_message_batch)

diff --git a/aws_lambda_powertools/utilities/data_classes/ses_event.py b/aws_lambda_powertools/utilities/data_classes/ses_event.py
@@ -1,4 +1,4 @@
-from typing import Iterator, List
+from typing import Iterator, List, Optional
 
 from aws_lambda_powertools.utilities.data_classes.common import DictWrapper
 
@@ -26,7 +26,7 @@ def get_from(self) -> List[str]:
         return self["from"]
 
     @property
-    def date(self) -> List[str]:
+    def date(self) -> str:
         """The date and time when Amazon SES received the message."""
         return self["date"]
 
@@ -45,6 +45,26 @@ def subject(self) -> str:
         """The value of the Subject header for the email."""
         return str(self["subject"])
 
+    @property
+    def cc(self) -> Optional[List[str]]:
+        """The values in the CC header of the email."""
+        return self.get("cc")
+
+    @property
+    def bcc(self) -> Optional[List[str]]:
+        """The values in the BCC header of the email."""
+        return self.get("bcc")
+
+    @property
+    def sender(self) -> Optional[List[str]]:
+        """The values in the Sender header of the email."""
+        return self.get("sender")
+
+    @property
+    def reply_to(self) -> Optional[List[str]]:
+        """The values in the replyTo header of the email."""
+        return self.get("replyTo")
+
 
 class SESMail(DictWrapper):
     @property
@@ -94,6 +114,9 @@ def common_headers(self) -> SESMailCommonHeaders:
 class SESReceiptStatus(DictWrapper):
     @property
     def status(self) -> str:
+        """Receipt status
+        Possible values: 'PASS', 'FAIL', 'GRAY', 'PROCESSING_FAILED', 'DISABLED'
+        """
         return str(self["status"])
 
 
@@ -107,6 +130,12 @@ def get_type(self) -> str:
         # Note: this name conflicts with existing python builtins
         return self["type"]
 
+    @property
+    def topic_arn(self) -> Optional[str]:
+        """String that contains the Amazon Resource Name (ARN) of the Amazon SNS topic to which the
+        notification was published."""
+        return self.get("topicArn")
+
     @property
     def function_arn(self) -> str:
         """String that contains the ARN of the Lambda function that was triggered.
@@ -154,12 +183,24 @@ def spf_verdict(self) -> SESReceiptStatus:
         """Object that indicates whether the Sender Policy Framework (SPF) check passed."""
         return SESReceiptStatus(self["spfVerdict"])
 
+    @property
+    def dkim_verdict(self) -> SESReceiptStatus:
+        """Object that indicates whether the DomainKeys Identified Mail (DKIM) check passed"""
+        return SESReceiptStatus(self["dkimVerdict"])
+
     @property
     def dmarc_verdict(self) -> SESReceiptStatus:
         """Object that indicates whether the Domain-based Message Authentication,
         Reporting & Conformance (DMARC) check passed."""
         return SESReceiptStatus(self["dmarcVerdict"])
 
+    @property
+    def dmarc_policy(self) -> Optional[str]:
+        """Indicates the Domain-based Message Authentication, Reporting & Conformance (DMARC) settings for
+        the sending domain. This field only appears if the message fails DMARC authentication.
+        Possible values for this field are: none, quarantine, reject"""
+        return self.get("dmarcPolicy")
+
     @property
     def action(self) -> SESReceiptAction:
         """Object that encapsulates information about the action that was executed."""

diff --git a/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py b/aws_lambda_powertools/utilities/idempotency/persistence/dynamodb.py
@@ -22,7 +22,7 @@ def __init__(
         self,
         table_name: str,
         key_attr: str = "id",
-        static_pk_value: str = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}",
+        static_pk_value: Optional[str] = None,
         sort_key_attr: Optional[str] = None,
         expiry_attr: str = "expiration",
         status_attr: str = "status",
@@ -76,6 +76,9 @@ def __init__(
         if sort_key_attr == key_attr:
             raise ValueError(f"key_attr [{key_attr}] and sort_key_attr [{sort_key_attr}] cannot be the same!")
 
+        if static_pk_value is None:
+            static_pk_value = f"idempotency#{os.getenv(constants.LAMBDA_FUNCTION_NAME_ENV, '')}"
+
         self._table = None
         self.table_name = table_name
         self.key_attr = key_attr

diff --git a/aws_lambda_powertools/utilities/parameters/__init__.py b/aws_lambda_powertools/utilities/parameters/__init__.py
@@ -5,7 +5,7 @@
 """
 
 from .appconfig import AppConfigProvider, get_app_config
-from .base import BaseProvider
+from .base import BaseProvider, clear_caches
 from .dynamodb import DynamoDBProvider
 from .exceptions import GetParameterError, TransformParameterError
 from .secrets import SecretsProvider, get_secret
@@ -23,4 +23,5 @@
     "get_parameter",
     "get_parameters",
     "get_secret",
+    "clear_caches",
 ]