feat: Add schema validation tests

samcli/commands/_utils/options.py

@@ -820,7 +820,7 @@ def _space_separated_list_func_type(value):
     raise ValueError()
 
 
-_space_separated_list_func_type.__name__ = "LIST"
+_space_separated_list_func_type.__name__ = "list,string"
 
 
 def generate_next_command_recommendation(command_tuples: List[Tuple[str, str]]) -> str:

schema/samcli.json

@@ -1085,7 +1085,10 @@
                 },
                 "notification_arns": {
                   "title": "notification_arns",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "ARNs of SNS topics that AWS Cloudformation associates with the stack.",
                   "items": {
                     "type": "string"
@@ -1122,7 +1125,10 @@
                 },
                 "capabilities": {
                   "title": "capabilities",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "List of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html",
                   "items": {
                     "type": "string"
@@ -1560,7 +1566,10 @@
                 },
                 "notification_arns": {
                   "title": "notification_arns",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "ARNs of SNS topics that AWS Cloudformation associates with the stack.",
                   "items": {
                     "type": "string"
@@ -1576,7 +1585,10 @@
                 },
                 "capabilities": {
                   "title": "capabilities",
-                  "type": "array",
+                  "type": [
+                    "array",
+                    "string"
+                  ],
                   "description": "List of capabilities that one must specify before AWS Cloudformation can create certain stacks.\n\nAccepted Values: CAPABILITY_IAM, CAPABILITY_NAMED_IAM, CAPABILITY_RESOURCE_POLICY, CAPABILITY_AUTO_EXPAND.\n\nLearn more at: https://docs.aws.amazon.com/serverlessrepo/latest/devguide/acknowledging-application-capabilities.html",
                   "default": [
                     "CAPABILITY_NAMED_IAM",

tests/unit/schema/test_schema_validation.py

@@ -0,0 +1,65 @@
+import os
+from pathlib import Path
+import jsonschema
+from parameterized import parameterized
+from unittest import TestCase
+
+from samcli.lib.config.file_manager import FILE_MANAGER_MAPPER
+from schema.make_schema import generate_schema
+
+
+class TestSchemaValidation(TestCase):
+    schema = None
+    testdata_dir = None
+
+    @classmethod
+    def setUpClass(cls):
+        cls.schema = generate_schema()
+        testing_dir = Path(__name__).resolve().parents[0]
+        cls.testdata_dir = str(Path(testing_dir, "tests", "unit", "schema", "testdata"))
+
+    def test_samconfig_validates_against_schema(self):
+        self.assertIsNotNone(self.schema, "Schema was not set")
+
+        passing_tests_dir = Path(self.testdata_dir, "passing_tests")
+        tests_not_passing = []
+
+        # Read in and assert all files in passing_tests pass
+        for config_file_path in os.listdir(passing_tests_dir):
+            config_file = FILE_MANAGER_MAPPER[Path(config_file_path).suffix].read(
+                Path(str(passing_tests_dir), config_file_path)
+            )
+            self.assertNotEqual(config_file, {}, f"Config file {config_file_path} should be read correctly")
+
+            try:
+                jsonschema.validate(config_file, self.schema)
+            except jsonschema.ValidationError as e:
+                tests_not_passing.append(f"<{config_file_path}: {e.message}>")
+
+        if tests_not_passing:
+            self.assertTrue(False, f"Some config files are not validating against schema: {tests_not_passing}")
+
+    def test_samconfig_doesnt_validate_against_schema(self):
+        self.assertIsNotNone(self.schema, "Schema was not set")
+
+        failing_tests_dir = Path(self.testdata_dir, "failing_tests")
+        tests_not_failing = []
+
+        # Read in and assert all files in failing_tests fail
+        for config_file_path in os.listdir(failing_tests_dir):
+            config_file = FILE_MANAGER_MAPPER[Path(config_file_path).suffix].read(
+                Path(str(failing_tests_dir), config_file_path)
+            )
+            self.assertNotEqual(config_file, {}, f"Config file {config_file_path} should be read correctly")
+
+            with self.assertRaises(
+                jsonschema.ValidationError, msg=f"Config file {config_file_path} should not validate against schema"
+            ):
+                jsonschema.validate(config_file, self.schema)
+                tests_not_failing.append(config_file_path)
+                raise jsonschema.ValidationError("Test should have failed but didn't")
+
+        if tests_not_failing:
+            self.assertTrue(
+                False, f"Some config files are validating against schema when they shouldn't: {tests_not_failing}"
+            )

tests/unit/schema/testdata/failing_tests/buildcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.build]
+debug = false

tests/unit/schema/testdata/failing_tests/deletecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.delete]
+debug = false

tests/unit/schema/testdata/failing_tests/deploycmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.deploy]
+debug = false

tests/unit/schema/testdata/failing_tests/initcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.init]
+debug = false

tests/unit/schema/testdata/failing_tests/list_endpointscmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.list_endpoints]
+debug = false

tests/unit/schema/testdata/failing_tests/list_resourcescmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.list_resources]
+debug = false

tests/unit/schema/testdata/failing_tests/list_stack_outputscmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.list_stack_outputs]
+debug = false

tests/unit/schema/testdata/failing_tests/local_invokecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_invoke]
+debug = false

tests/unit/schema/testdata/failing_tests/local_start_apicmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_start_api]
+debug = false

tests/unit/schema/testdata/failing_tests/local_start_lambdacmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.local_start_lambda]
+debug = false

tests/unit/schema/testdata/failing_tests/logscmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.logs]
+debug = false

tests/unit/schema/testdata/failing_tests/noversion.toml

@@ -0,0 +1,26 @@
+# version = 0.1
+
+[default]
+[default.global]
+[default.global.parameters]
+stack_name = "sam-app"
+
+[default.build.parameters]
+cached = true
+parallel = true
+
+[default.deploy.parameters]
+capabilities = "CAPABILITY_IAM"
+confirm_changeset = true
+resolve_s3 = true
+
+[default.sync.parameters]
+watch = true
+
+[default.local_start_api.parameters]
+warm_containers = "EAGER"
+
+[prod]
+[prod.sync]
+[prod.sync.parameters]
+watch = false

tests/unit/schema/testdata/failing_tests/noversion.yaml

@@ -0,0 +1,4 @@
+default:
+  build:
+    parameters:
+      debug: false

tests/unit/schema/testdata/failing_tests/packagecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.package]
+debug = false

tests/unit/schema/testdata/failing_tests/pipeline_bootstrapcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.pipeline_bootstrap]
+debug = false

tests/unit/schema/testdata/failing_tests/pipeline_initcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.pipeline_init]
+debug = false

tests/unit/schema/testdata/failing_tests/publishcmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.publish]
+debug = false

tests/unit/schema/testdata/failing_tests/remote_invokecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.remote_invoke]
+debug = false

tests/unit/schema/testdata/failing_tests/synccmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.sync]
+debug = false

tests/unit/schema/testdata/failing_tests/tracescmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.traces]
+debug = false

tests/unit/schema/testdata/failing_tests/validatecmd_noparams.toml

@@ -0,0 +1,4 @@
+version = 0.1
+
+[default.validate]
+debug = false

tests/unit/schema/testdata/passing_tests/buildcmd.toml

@@ -0,0 +1,36 @@
+version = 0.1
+
+[default.build.parameters]
+hook_name = "Hook name"
+skip_prepare_infra = false
+use_container = false
+container_env_var = "Container env var"
+container_env_var_file = "Container env var file"
+build_image = "Build image"
+exclude = "Exclude"
+parallel = false
+mount_with = "READ"
+build_dir = "Build dir"
+cache_dir = "Cache dir"
+base_dir = "Base dir"
+manifest = "Manifest"
+cached = false
+template_file = "Template file"
+parameter_overrides = "Parameter overrides"
+skip_pull_image = false
+docker_network = "Docker network"
+beta_features = false
+debug = false
+profile = "Profile"
+region = "Region"
+
+[other.build.parameters]
+skip_prepare_infra = true
+use_container = true
+parallel = true
+mount_with = "WRITE"
+cached = true
+parameter_overrides = ["Parameter", "Overrides"]
+skip_pull_image = true
+beta_features = true
+debug = true

tests/unit/schema/testdata/passing_tests/deletecmd.toml

@@ -0,0 +1,16 @@
+version = 0.1
+
+[default.delete.parameters]
+stack_name = "Stack name"
+no_prompts = false
+s3_bucket = "S3 Bucket"
+s3_prefix = "S3 Prefix"
+profile = "Profile"
+region = "Region"
+beta_features = false
+debug = false
+
+[other.delete.parameters]
+no_prompts = true
+beta_features = true
+debug = true

tests/unit/schema/testdata/passing_tests/deploycmd.toml

@@ -0,0 +1,50 @@
+version = 0.1
+
+[default.deploy.parameters]
+guided = false
+template_file = "Template file"
+no_execute_changeset = false
+fail_on_empty_changeset = false
+confirm_changeset = false
+disable_rollback = false
+on_failure = "DELETE"
+stack_name = "Stack name"
+s3_bucket = "S3 Bucket"
+image_repository = "Image repository"
+image_repositories = ["Image", "Repositories"]
+force_upload = false
+s3_prefix = "S3 Prefix"
+kms_key_id = "KMS Key ID"
+role_arn = "Role ARN"
+use_json = false
+resolve_s3 = false
+resolve_image_repos = false
+metadata = "Metadata"
+notification_arns = "Notification ARNS"
+tags = ["Tags", "More tags", "Even more tags"]
+parameter_overrides = "Parameter overrides"
+signing_profiles = "Signing profiles"
+no_progressbar = false
+capabilities = "Capabilities"
+profile = "Profile"
+region = "Region"
+beta_features = false
+debug = false
+
+[other.deploy.parameters]
+guided = true
+no_execute_changeset = true
+fail_on_empty_changeset = true
+confirm_changeset = true
+disable_rollback = true
+on_failure = "ROLLBACK"
+force_upload = true
+use_json = true
+resolve_s3 = true
+resolve_image_repos = true
+notification_arns = ["Notification", "ARNS"]
+parameter_overrides = ["Parameter", "Overrides"]
+no_progressbar = true
+capabilities = ["Capabilities", "More capabilities"]
+beta_features = true
+debug = true

tests/unit/schema/testdata/passing_tests/initcmd.toml

@@ -0,0 +1,37 @@
+version = 0.1
+
+[default.init.parameters]
+no_interactive = false
+architecture = "arm64"
+location = "Location"
+runtime = "python3.11"
+package_type = "Image"
+base_image = "amazon/python3.11-base"
+dependency_manager = "pip"
+output_dir = "Output dir"
+name = "Name"
+app_template = "App template"
+no_input = false
+extra_context = "Extra context"
+tracing = false
+application_insights = false
+beta_features = false
+debug = false
+
+[other.init.parameters]
+no_interactive = true
+architecture = "x86_64"
+runtime = "java11"
+package_type = "Zip"
+base_image = "amazon/java11-base"
+dependency_manager = "gradle"
+no_input = true
+tracing = true
+application_insights = true
+beta_features = true
+debug = true
+
+[onemore.init.parameters]
+runtime = "ruby3.2"
+base_image = "amazon/ruby3.2-base"
+dependency_manager = "bundler"