Merge branch 'develop' into wip/developScalingTest

CHANGELOG.md

@@ -19,7 +19,7 @@ CHANGELOG
   - Add `DisableSudoAccessForDefaultUser` parameter to disable sudo access of default user in supported OSes.
 
 **CHANGES**
-- Upgrade Slurm to 23.11.1.
+- Upgrade Slurm to 23.11.3.
 - Add support for Python 3.11, 3.12 in pcluster CLI and aws-parallelcluster-batch-cli.
 - Upgrade Python to version 3.12 and NodeJS to version 18 in ParallelCluster Lambda Layer.
 - Build network interfaces using network card index from `NetworkCardIndex` list of EC2 DescribeInstances response, 

tests/integration-tests/configs/new_instance_types.yaml

@@ -87,12 +87,6 @@ test-suites:
           oss: {{ common.OSS_ONE_PER_DISTRO }}
           schedulers: ["slurm"]
   networking:
-    test_cluster_networking.py::test_cluster_in_private_subnet:
-      dimensions:
-        - regions: {{ NEW_REGIONS }}
-          instances: {{ NEW_INSTANCE_TYPES }}
-          oss: ["ubuntu2004"]
-          schedulers: ["slurm"]
 #     Useful for instances with multiple network interfaces
     test_multi_cidr.py::test_multi_cidr:
       dimensions:

tests/integration-tests/configs/new_os.yaml

@@ -199,12 +199,6 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_ARM }}
           oss: {{ NEW_OS }}
           schedulers: ["slurm"]
-    test_slurm.py::test_slurm_scaling:
-      dimensions:
-        - regions: ["us-west-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: {{ NEW_OS }}
-          schedulers: ["slurm"]
   storage:
     # Commercial regions that can't test FSx: ap-northeast-1, ap-southeast-1, ap-southeast-2, eu-central-1, eu-north-1, eu-west-1, eu-west-2, us-east-1, us-east-2, us-west-1, us-west-2
     test_fsx_lustre.py::test_fsx_lustre:

tests/integration-tests/configs/released.yaml

@@ -188,20 +188,7 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: ["ubuntu2204"]
           schedulers: ["slurm"]
-  multiple_nics:
-    test_multiple_nics.py::test_multiple_nics:
-      dimensions:
-        - regions: ["use1-az6"]
-          instances: ["c6in.32xlarge"]
-          oss: {{ common.OSS_COMMERCIAL_X86 }}
-          schedulers: ["slurm"]
   networking:
-    test_cluster_networking.py::test_cluster_in_private_subnet:
-      dimensions:
-        - regions: ["il-central-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
     test_cluster_networking.py::test_existing_eip:
       dimensions:
         - regions: ["me-south-1"]
@@ -240,24 +227,12 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: ["alinux2"]
           schedulers: ["slurm"]
-    test_security_groups.py::test_additional_sg_and_ssh_from:
-      dimensions:
-        - regions: ["eu-north-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["centos7"]
-          schedulers: ["slurm"]
     test_security_groups.py::test_overwrite_sg:
       dimensions:
         - regions: ["eu-north-1"]
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: {{ common.NOT_RELEASED_OSES }}
           schedulers: ["slurm"]
-    test_placement_group.py::test_placement_group:
-      dimensions:
-        - regions: ["eu-central-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
   pcluster_api:
     test_api_infrastructure.py::test_api_infrastructure_with_default_parameters:
       dimensions:
@@ -319,12 +294,6 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: ["ubuntu2204"]
           schedulers: ["slurm"]
-    test_slurm.py::test_slurm_from_login_nodes_in_private_network:
-      dimensions:
-        - regions: ["eu-west-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
     test_slurm.py::test_slurm_pmix:  # TODO: include in main test_slurm to reduce number of created clusters
       dimensions:
         - regions: ["ap-south-1"]
@@ -335,12 +304,6 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_ARM }}
           oss: ["rhel8"]
           schedulers: ["slurm"]
-    test_slurm.py::test_slurm_scaling:
-      dimensions:
-        - regions: ["use2-az2"]  # do not move, unless instance type support is moved as well
-          instances: [{{ common.instance("instance_type_1") }}]
-          oss: [ "alinux2" ]
-          schedulers: [ "slurm" ]
     test_slurm.py::test_error_handling:
       dimensions:
         - regions: ["ca-central-1"]
@@ -353,18 +316,6 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: ["rhel8"]
           schedulers: ["slurm"]
-    test_slurm.py::test_slurm_protected_mode_on_cluster_create:
-      dimensions:
-        - regions: ["ap-east-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["centos7"]
-          schedulers: ["slurm"]
-    test_slurm.py::test_fast_capacity_failover:
-      dimensions:
-        - regions: ["ap-east-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
     test_slurm.py::test_slurm_config_update:
       dimensions:
         - regions: [ "ap-east-1" ]
@@ -377,48 +328,6 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: ["ubuntu2204"]
           schedulers: ["slurm"]
-    test_slurm.py::test_scontrol_reboot:
-      dimensions:
-        - regions: ["us-east-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["ubuntu2004"]
-          schedulers: ["slurm"]
-    test_slurm.py::test_scontrol_reboot_ec2_health_checks:
-      dimensions:
-        - regions: ["us-east-2"]
-          instances: ["t2.medium"]
-          oss: ["ubuntu2204"]
-          schedulers: ["slurm"]
-    test_slurm.py::test_slurm_overrides:
-      dimensions:
-        - regions: ["me-south-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["ubuntu2004"]
-          schedulers: ["slurm"]
-    test_slurm_accounting.py::test_slurm_accounting:
-      dimensions:
-        - regions: ["ap-south-1"]
-          instances:  {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["ubuntu2204"]
-          schedulers: ["slurm"]
-    test_slurm_accounting.py::test_slurm_accounting_disabled_to_enabled_update:
-      dimensions:
-        - regions: ["us-west-2"]
-          instances:  {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["centos7"]
-          schedulers: ["slurm"]
-    test_slurm.py::test_slurm_custom_partitions:
-      dimensions:
-        - regions: ["ap-northeast-2"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["ubuntu2004"]
-          schedulers: ["slurm"]
-    test_custom_munge_key.py::test_custom_munge_key:
-      dimensions:
-        - regions: ["eu-west-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
   spot:
     test_spot.py::test_spot_default:
       dimensions:
@@ -427,22 +336,8 @@ test-suites:
           oss: ["centos7"]
           schedulers: ["slurm"]
   storage:
-    test_fsx_lustre.py::test_fsx_lustre:
-      dimensions:
-        - regions: ["eu-west-2"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
-        - regions: ["eu-north-1"]
-          instances: {{ common.INSTANCES_DEFAULT_ARM }}
-          oss: ["ubuntu2204"]
-          schedulers: ["slurm"]
     test_fsx_lustre.py::test_fsx_lustre_dra:
       dimensions:
-        - regions: [ "eu-west-2" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: [ "alinux2" ]
-          schedulers: [ "slurm" ]
         - regions: [ "eu-north-1" ]
           instances: {{ common.INSTANCES_DEFAULT_ARM }}
           oss: [ "ubuntu2204" ]
@@ -453,32 +348,6 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_ARM }}
           oss: [ "ubuntu2204" ]
           schedulers: [ "slurm" ]
-    # The checks performed in test_multiple_fsx is the same as test_fsx_lustre.
-    # We should consider this when assigning dimensions to each test.
-    test_fsx_lustre.py::test_multiple_fsx:
-      dimensions:
-        - regions: ["eu-west-2"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
-          schedulers: ["slurm"]
-        - regions: ["eu-west-2"]
-          instances: {{ common.INSTANCES_DEFAULT_ARM }}
-          oss: ["ubuntu2004"]
-          schedulers: ["slurm"]
-        - regions: ["cn-north-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
-        - regions: [ "us-gov-west-1" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: {{ common.NOT_RELEASED_OSES }}
-          schedulers: [ "slurm" ]
-    test_fsx_lustre.py::test_multi_az_fsx:
-      dimensions:
-        - regions: ["eu-west-2"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["centos7"]
-          schedulers: ["slurm"]
     test_fsx_lustre.py::test_fsx_lustre_configuration_options:
       dimensions:
         - regions: ["us-east-2"]
@@ -491,82 +360,17 @@ test-suites:
           instances: {{ common.INSTANCES_DEFAULT_ARM }}
           oss: ["rhel8"]
           schedulers: ["slurm"]
-    # EFS tests can be done in any region.
-    test_efs.py::test_efs_compute_az:
-      dimensions:
-        - regions: ["ca-central-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
-    test_efs.py::test_efs_same_az:
-      dimensions:
-        - regions: [ "ca-central-1" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: {{ common.NOT_RELEASED_OSES }}
-          schedulers: [ "slurm" ]
-    # The checks performed in test_efs_same_az is similar to test_multiple_efs.
-    # We should consider this when assigning dimensions to each test.
     test_efs.py::test_multiple_efs:
       dimensions:
-        - regions: ["cn-northwest-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["awsbatch"]
         - regions: [ "ca-central-1" ]
           instances: {{ common.INSTANCES_DEFAULT_ARM }}
           oss: ["ubuntu2204"]
           schedulers: [ "slurm" ]
-        - regions: ["us-gov-east-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
-          schedulers: ["slurm"]
-    test_raid.py::test_raid_fault_tolerance_mode:
-      dimensions:
-        - regions: ["cn-northwest-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
-    test_raid.py::test_raid_performance_mode:
-      dimensions:
-        - regions: ["ap-south-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["centos7"]
-          schedulers: ["slurm"]
-        - regions: [ "ap-south-1" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          schedulers: [ "slurm" ]
-          oss: {{ common.NOT_RELEASED_OSES }}
-        - regions: ["us-gov-east-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
-          schedulers: ["slurm"]
-    test_ebs.py::test_ebs_multiple:
-      dimensions:
-        - regions: ["us-east-2"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["ubuntu2204"]
-          schedulers: ["slurm"]
     test_ebs.py::test_ebs_single:
-      dimensions:
-        - regions: ["us-gov-east-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["alinux2"]
-          schedulers: ["slurm"]
-    test_ebs.py::test_ebs_snapshot:
-      dimensions:
-        - regions: [ "us-east-2" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: {{ common.NOT_RELEASED_OSES }}
-          schedulers: [ "slurm" ]
-        - regions: ["cn-northwest-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
-          schedulers: ["slurm"]
-    test_ebs.py::test_ebs_existing:
       dimensions:
         - regions: ["us-east-2"]
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
+          oss: ["ubuntu2204"]
           schedulers: ["slurm"]
     test_deletion_policy.py::test_retain_on_deletion:
       dimensions:
@@ -581,18 +385,6 @@ test-suites:
           instances: ["m5d.xlarge"]  # SSD based instance
           oss: ["alinux2"]
           schedulers: ["slurm"]
-    test_internal_efs.py::test_internal_efs:
-      dimensions:
-        - regions: [ "us-west-2" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: [ "ubuntu2204" ]
-          schedulers: [ "slurm" ]
-    test_shared_home.py::test_shared_home:
-      dimensions:
-        - regions: [ "us-west-1" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: [ "alinux2" ]
-          schedulers: [ "slurm" ]
   tags:
     test_tag_propagation.py::test_tag_propagation:
       dimensions:
@@ -606,35 +398,3 @@ test-suites:
         - regions: ["eu-central-1"]
           instances: {{ common.INSTANCES_DEFAULT_X86 }}
           oss: ["ubuntu2004"]
-    test_update.py::test_update_compute_ami:
-      dimensions:
-        - regions: ["eu-west-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
-    test_update.py::test_update_instance_list:
-      dimensions:
-        - regions: ["ap-south-1"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["rhel8"]
-          schedulers: ["slurm"]
-    test_update.py::test_queue_parameters_update:
-      dimensions:
-        - regions: [ "ap-south-1" ]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: {{ common.NOT_RELEASED_OSES }}
-          schedulers: [ "slurm" ]
-    test_update.py::test_dynamic_file_systems_update:
-      dimensions:
-        - regions: ["eu-west-2"]
-          instances: {{ common.INSTANCES_DEFAULT_X86 }}
-          oss: ["centos7"]
-          schedulers: ["slurm"]
-        - regions: ["ap-northeast-1"]
-          instances: {{ common.INSTANCES_DEFAULT_ARM }}
-          oss: ["ubuntu2204"]
-          schedulers: ["slurm"]
-    test_update.py::test_multi_az_create_and_update:
-      dimensions:
-        - regions: ["eu-west-2"]
-          schedulers: ["slurm"]
-          oss: ["alinux2"]

tests/integration-tests/test_runner.py

@@ -753,6 +753,14 @@ def _check_args(args):
             raise argparse.ArgumentTypeError("'{0}' is not a valid test config".format(args.tests_config))
 
 
+def unset_proxy():
+    """Unset proxies"""
+    os.environ.pop("HTTP_PROXY", None)
+    os.environ.pop("HTTPS_PROXY", None)
+    os.environ.pop("http_proxy", None)
+    os.environ.pop("https_proxy", None)
+
+
 def _run_sequential(args):
     # Redirect stdout to file
     if not args.show_output:
@@ -779,6 +787,9 @@ def main():
     _check_args(args)
     logger.info("Parsed test_runner parameters {0}".format(args))
 
+    # Unset any proxies used to avoid network issues with tests, such as DCV
+    unset_proxy()
+
     _make_logging_dirs(args.output_dir)
 
     if args.sequential: