ci: (experimental) Enable more ruff rules

Makefile

@@ -45,6 +45,9 @@ lint:
 	# cfn-lint to make sure generated CloudFormation makes sense
 	bin/run_cfn_lint.sh
 
+lint-fix:
+	ruff --fix samtranslator bin schema_source
+
 prepare-companion-stack:
 	pytest -v --no-cov integration/setup -m setup
 

bin/add_transform_test.py

@@ -12,15 +12,16 @@
     --disable-update-partition      Disable updating the partition of arn to aws-cn/aws-us-gov
 """
 import json
-import subprocess
 import os
 import shutil
+import subprocess
 import sys
 import tempfile
-from docopt import docopt  # type: ignore
 from pathlib import Path
 from typing import Any, Dict
 
+from docopt import docopt  # type: ignore
+
 SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
 TRANSFORM_TEST_DIR = os.path.join(SCRIPT_DIR, "..", "tests", "translator")
 CLI_OPTIONS = docopt(__doc__)

bin/parse_docs.py

@@ -14,7 +14,7 @@
 import json
 import re
 from pathlib import Path
-from typing import Iterator, Tuple, Dict
+from typing import Dict, Iterator, Tuple
 
 
 def parse(s: str, cfn_docs: bool) -> Iterator[Tuple[str, str]]:

bin/sam-translate.py

@@ -24,19 +24,18 @@
 import platform
 import subprocess
 import sys
+from functools import reduce
 
 import boto3
-
 from docopt import docopt  # type: ignore[import]
-from functools import reduce
 
 my_path = os.path.dirname(os.path.abspath(__file__))
 sys.path.insert(0, my_path + "/..")
 
+from samtranslator.model.exceptions import InvalidDocumentException
 from samtranslator.public.translator import ManagedPolicyLoader
 from samtranslator.translator.transform import transform
 from samtranslator.yaml_helper import yaml_parse
-from samtranslator.model.exceptions import InvalidDocumentException
 
 LOG = logging.getLogger(__name__)
 cli_options = docopt(__doc__)

ruff.toml

@@ -1,6 +1,12 @@
 # black formatter takes care of the line length
 line-length = 999
 
+# Enable Pyflakes `E`, `F` codes, pylint `PL`, isort `I` and pyupgrade (UP).
+select = ["E", "F", "PL", "I"]
+
+# Mininal python version we support is 3.7
+target-version = "py37"
+
 [per-file-ignores]
 # python scripts in bin/ needs some python path configurations before import
 "bin/*.py" = ["E402"]  # E402: module-import-not-at-top-of-file

samtranslator/feature_toggle/feature_toggle.py

@@ -1,15 +1,15 @@
 import json
+import logging
 from abc import ABC, abstractmethod
 from typing import Any, Dict, cast
 
 import boto3
-import logging
-
 from botocore.config import Config
+
 from samtranslator.feature_toggle.dialup import (
     DisabledDialup,
-    ToggleDialup,
     SimpleAccountPercentileDialup,
+    ToggleDialup,
 )
 from samtranslator.metrics.method_decorator import cw_timer
 from samtranslator.utils.constants import BOTO3_CONNECT_TIMEOUT

samtranslator/intrinsics/actions.py

@@ -1,8 +1,8 @@
 import re
 from abc import ABC
-from typing import Any, Dict, Optional, Tuple, Callable, List
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
-from samtranslator.model.exceptions import InvalidTemplateException, InvalidDocumentException
+from samtranslator.model.exceptions import InvalidDocumentException, InvalidTemplateException
 
 
 class Action(ABC):
@@ -68,10 +68,15 @@ def _parse_resource_reference(cls, ref_value: Any) -> Tuple[Optional[str], Optio
         splits = ref_value.split(cls._resource_ref_separator, 1)
 
         # Either there is no 'dot' (or) one of the values is empty string (Ex: when you split "LogicalId.")
-        if len(splits) != 2 or not all(splits):
+        try:
+            logical_id, property_name = splits
+        except ValueError:
             return no_result
 
-        return splits[0], splits[1]
+        if not logical_id or not property_name:
+            return no_result
+
+        return logical_id, property_name
 
 
 class RefAction(Action):
@@ -233,11 +238,11 @@ def do_replacement(full_ref: str, ref_value: str) -> str:
             splits = ref_value.split(self._resource_ref_separator)
 
             # If we don't find at least two parts, there is nothing to resolve
-            if len(splits) < 2:
+            try:
+                logical_id, property_name = splits[:2]
+            except ValueError:
                 return full_ref
 
-            logical_id = splits[0]
-            property_name = splits[1]
             resolved_value = supported_resource_refs.get(logical_id, property_name)
             if not resolved_value:
                 # This ID/property combination is not in the supported references
@@ -400,6 +405,8 @@ def handler_method(full_ref, ref_value):
 class GetAttAction(Action):
     intrinsic_name = "Fn::GetAtt"
 
+    _MIN_NUM_ARGUMENTS = 2
+
     def resolve_parameter_refs(self, input_dict: Optional[Any], parameters: Dict[str, Any]) -> Optional[Any]:
         # Parameters can never be referenced within GetAtt value
         return input_dict
@@ -503,9 +510,9 @@ def resolve_resource_id_refs(
         return self._get_resolved_dictionary(input_dict, key, resolved_value, remaining)
 
     def _check_input_value(self, value: Any) -> bool:
-        # Value must be an array with *at least* two elements. If not, this is invalid GetAtt syntax. We just pass along
+        # Value must be an array with enough elements. If not, this is invalid GetAtt syntax. We just pass along
         # the input to CFN for it to do the "official" validation.
-        if not isinstance(value, list) or len(value) < 2:
+        if not isinstance(value, list) or len(value) < self._MIN_NUM_ARGUMENTS:
             return False
 
         # If items in value array is not a string, then following join line will fail. So if any element is not a string
@@ -542,6 +549,8 @@ class FindInMapAction(Action):
 
     intrinsic_name = "Fn::FindInMap"
 
+    _NUM_ARGUMENTS = 3
+
     def resolve_parameter_refs(self, input_dict: Optional[Any], parameters: Dict[str, Any]) -> Optional[Any]:
         """
         Recursively resolves "Fn::FindInMap"references that are present in the mappings and returns the value.
@@ -558,11 +567,11 @@ def resolve_parameter_refs(self, input_dict: Optional[Any], parameters: Dict[str
         value = input_dict[self.intrinsic_name]
 
         # FindInMap expects an array with 3 values
-        if not isinstance(value, list) or len(value) != 3:
+        if not isinstance(value, list) or len(value) != self._NUM_ARGUMENTS:
             raise InvalidDocumentException(
                 [
                     InvalidTemplateException(
-                        f"Invalid FindInMap value {value}. FindInMap expects an array with 3 values."
+                        f"Invalid FindInMap value {value}. FindInMap expects an array with {self._NUM_ARGUMENTS} values."
                     )
                 ]
             )

samtranslator/intrinsics/resolver.py

@@ -1,9 +1,9 @@
 # Help resolve intrinsic functions
-from typing import Any, Dict, Optional, Callable, List, Union
+from typing import Any, Callable, Dict, List, Optional, Union
 
-from samtranslator.intrinsics.actions import Action, SubAction, RefAction, GetAttAction
-from samtranslator.model.exceptions import InvalidTemplateException, InvalidDocumentException
+from samtranslator.intrinsics.actions import Action, GetAttAction, RefAction, SubAction
 from samtranslator.intrinsics.resource_refs import SupportedResourceReferences
+from samtranslator.model.exceptions import InvalidDocumentException, InvalidTemplateException
 
 # All intrinsics are supported by default
 DEFAULT_SUPPORTED_INTRINSICS = {action.intrinsic_name: action() for action in [RefAction, SubAction, GetAttAction]}

samtranslator/metrics/method_decorator.py

@@ -2,11 +2,12 @@
 Method decorator for execution latency collection
 """
 import functools
+import logging
 from datetime import datetime
+from typing import Any, Callable, Optional, Union
+
 from samtranslator.metrics.metrics import DummyMetricsPublisher, Metrics
 from samtranslator.model import Resource
-import logging
-from typing import Any, Callable, Optional, Union
 
 LOG = logging.getLogger(__name__)
 

samtranslator/metrics/metrics.py

@@ -2,9 +2,9 @@
 Helper classes to publish metrics
 """
 import logging
+from abc import ABC, abstractmethod
 from datetime import datetime
 from typing import Any, Dict
-from abc import ABC, abstractmethod
 
 LOG = logging.getLogger(__name__)
 

samtranslator/model/__init__.py

@@ -1,14 +1,14 @@
 """ CloudFormation Resource serialization, deserialization, and validation """
-import re
 import inspect
-from abc import ABC
+import re
+from abc import ABC, ABCMeta
 from typing import Any, Callable, Dict, List, Optional, Tuple, Union
-from abc import ABCMeta
+
 from samtranslator.intrinsics.resolver import IntrinsicsResolver
 from samtranslator.model.exceptions import ExpectedType, InvalidResourceException, InvalidResourcePropertyTypeException
+from samtranslator.model.tags.resource_tagging import get_tag_list
 from samtranslator.model.types import IS_DICT, IS_STR, Validator, any_type, is_type
 from samtranslator.plugins import LifeCycleEvents
-from samtranslator.model.tags.resource_tagging import get_tag_list
 
 
 class PropertyType:

samtranslator/model/api/api_generator.py

@@ -1,39 +1,38 @@
 import logging
 from collections import namedtuple
-from typing import List, Optional, Set, Dict, Any, cast, Union, Tuple
+from typing import Any, Dict, List, Optional, Set, Tuple, Union, cast
 
 from samtranslator.metrics.method_decorator import cw_timer
-from samtranslator.model.intrinsics import ref, fnGetAtt, make_or_condition
 from samtranslator.model.apigateway import (
+    ApiGatewayApiKey,
+    ApiGatewayAuthorizer,
+    ApiGatewayBasePathMapping,
     ApiGatewayDeployment,
+    ApiGatewayDomainName,
+    ApiGatewayResponse,
     ApiGatewayRestApi,
     ApiGatewayStage,
-    ApiGatewayAuthorizer,
-    ApiGatewayResponse,
-    ApiGatewayDomainName,
-    ApiGatewayBasePathMapping,
     ApiGatewayUsagePlan,
     ApiGatewayUsagePlanKey,
-    ApiGatewayApiKey,
 )
-from samtranslator.model.route53 import Route53RecordSetGroup
 from samtranslator.model.exceptions import (
     ExpectedType,
     InvalidDocumentException,
     InvalidResourceException,
     InvalidTemplateException,
 )
+from samtranslator.model.intrinsics import fnGetAtt, fnSub, is_intrinsic, make_or_condition, ref
+from samtranslator.model.lambda_ import LambdaPermission
+from samtranslator.model.route53 import Route53RecordSetGroup
 from samtranslator.model.s3_utils.uri_parser import parse_s3_uri
+from samtranslator.model.tags.resource_tagging import get_tag_list
 from samtranslator.model.types import PassThrough
 from samtranslator.region_configuration import RegionConfiguration
 from samtranslator.swagger.swagger import SwaggerEditor
-from samtranslator.model.intrinsics import is_intrinsic, fnSub
-from samtranslator.model.lambda_ import LambdaPermission
-from samtranslator.translator.logical_id_generator import LogicalIdGenerator
 from samtranslator.translator.arn_generator import ArnGenerator
-from samtranslator.utils.types import Intrinsicable
-from samtranslator.model.tags.resource_tagging import get_tag_list
+from samtranslator.translator.logical_id_generator import LogicalIdGenerator
 from samtranslator.utils.py27hash_fix import Py27Dict, Py27UniStr
+from samtranslator.utils.types import Intrinsicable
 from samtranslator.utils.utils import InvalidValueType, dict_deep_get
 from samtranslator.validator.value_validator import sam_expect
 

samtranslator/model/api/http_api_generator.py

@@ -3,20 +3,19 @@
 from typing import Any, Dict, List, Optional, Tuple, Union, cast
 
 from samtranslator.metrics.method_decorator import cw_timer
-from samtranslator.model.intrinsics import ref, fnGetAtt
 from samtranslator.model.apigatewayv2 import (
-    ApiGatewayV2HttpApi,
-    ApiGatewayV2Stage,
+    ApiGatewayV2ApiMapping,
     ApiGatewayV2Authorizer,
     ApiGatewayV2DomainName,
-    ApiGatewayV2ApiMapping,
+    ApiGatewayV2HttpApi,
+    ApiGatewayV2Stage,
 )
 from samtranslator.model.exceptions import InvalidResourceException
+from samtranslator.model.intrinsics import fnGetAtt, is_intrinsic, is_intrinsic_no_value, ref
+from samtranslator.model.route53 import Route53RecordSetGroup
 from samtranslator.model.s3_utils.uri_parser import parse_s3_uri
 from samtranslator.open_api.open_api import OpenApiEditor
 from samtranslator.translator.logical_id_generator import LogicalIdGenerator
-from samtranslator.model.intrinsics import is_intrinsic, is_intrinsic_no_value
-from samtranslator.model.route53 import Route53RecordSetGroup
 from samtranslator.utils.types import Intrinsicable
 from samtranslator.utils.utils import InvalidValueType, dict_deep_get
 from samtranslator.validator.value_validator import sam_expect

samtranslator/model/apigateway.py

@@ -5,7 +5,7 @@
 from samtranslator.model import PropertyType, Resource
 from samtranslator.model.exceptions import InvalidResourceException
 from samtranslator.model.intrinsics import fnSub, ref
-from samtranslator.model.types import IS_DICT, IS_STR, is_type, list_of, one_of, PassThrough
+from samtranslator.model.types import IS_DICT, IS_STR, PassThrough, is_type, list_of, one_of
 from samtranslator.translator import logical_id_generator
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.utils.py27hash_fix import Py27Dict, Py27UniStr

samtranslator/model/apigatewayv2.py

@@ -1,9 +1,9 @@
 from typing import Any, Dict, List, Optional, Union
 
 from samtranslator.model import PropertyType, Resource
-from samtranslator.model.types import IS_DICT, is_type, one_of, IS_STR, list_of, PassThrough
-from samtranslator.model.intrinsics import ref, fnSub
 from samtranslator.model.exceptions import ExpectedType, InvalidResourceException
+from samtranslator.model.intrinsics import fnSub, ref
+from samtranslator.model.types import IS_DICT, IS_STR, PassThrough, is_type, list_of, one_of
 from samtranslator.translator.arn_generator import ArnGenerator
 from samtranslator.utils.types import Intrinsicable
 from samtranslator.validator.value_validator import sam_expect

samtranslator/model/cloudformation.py

@@ -1,6 +1,6 @@
 from samtranslator.model import PropertyType, Resource
-from samtranslator.model.types import IS_DICT, is_type, IS_STR, list_of, one_of
 from samtranslator.model.intrinsics import ref
+from samtranslator.model.types import IS_DICT, IS_STR, is_type, list_of, one_of
 
 
 class NestedStack(Resource):

samtranslator/model/codedeploy.py

@@ -1,6 +1,6 @@
 from samtranslator.model import PropertyType, Resource
 from samtranslator.model.intrinsics import ref
-from samtranslator.model.types import IS_DICT, is_type, one_of, IS_STR
+from samtranslator.model.types import IS_DICT, IS_STR, is_type, one_of
 
 
 class CodeDeployApplication(Resource):

samtranslator/model/cognito.py

@@ -1,6 +1,6 @@
 from samtranslator.model import PropertyType, Resource
-from samtranslator.model.types import IS_DICT, list_of, IS_STR
 from samtranslator.model.intrinsics import fnGetAtt, ref
+from samtranslator.model.types import IS_DICT, IS_STR, list_of
 
 
 class CognitoUserPool(Resource):

samtranslator/model/connector/connector.py

@@ -1,11 +1,10 @@
 from collections import namedtuple
-from typing import Any, Dict, List, Optional, Iterable
+from typing import Any, Dict, Iterable, List, Optional
 
 from samtranslator.model import ResourceResolver
-from samtranslator.model.intrinsics import get_logical_id_from_intrinsic, ref, fnGetAtt
+from samtranslator.model.intrinsics import fnGetAtt, get_logical_id_from_intrinsic, ref
 from samtranslator.utils.utils import as_array, insert_unique
 
-
 # TODO: Switch to dataclass
 ConnectorResourceReference = namedtuple(
     "ConnectorResourceReference",

samtranslator/model/dynamodb.py

@@ -1,6 +1,6 @@
 from samtranslator.model import PropertyType, Resource
-from samtranslator.model.types import IS_DICT, is_type, dict_of, list_of, IS_STR, one_of
 from samtranslator.model.intrinsics import fnGetAtt, ref
+from samtranslator.model.types import IS_DICT, IS_STR, dict_of, is_type, list_of, one_of
 
 
 class DynamoDBTable(Resource):

samtranslator/model/eventbridge_utils.py

@@ -1,5 +1,5 @@
-from samtranslator.model.sqs import SQSQueue, SQSQueuePolicy, SQSQueuePolicies
 from samtranslator.model.exceptions import InvalidEventException
+from samtranslator.model.sqs import SQSQueue, SQSQueuePolicies, SQSQueuePolicy
 
 
 class EventBridgeRuleUtils:

samtranslator/model/events.py

@@ -1,6 +1,6 @@
 from samtranslator.model import PropertyType, Resource
-from samtranslator.model.types import IS_DICT, list_of, IS_STR
 from samtranslator.model.intrinsics import fnGetAtt, ref
+from samtranslator.model.types import IS_DICT, IS_STR, list_of
 
 
 class EventsRule(Resource):