diff --git a/CHANGELOG.md b/CHANGELOG.md index ceb2919e4f..e2f88ee387 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,7 +19,7 @@ CHANGELOG - Add `DisableSudoAccessForDefaultUser` parameter to disable sudo access of default user in supported OSes. **CHANGES** -- Upgrade Slurm to 23.11.1. +- Upgrade Slurm to 23.11.3. - Add support for Python 3.11, 3.12 in pcluster CLI and aws-parallelcluster-batch-cli. - Upgrade Python to version 3.12 and NodeJS to version 18 in ParallelCluster Lambda Layer. - Build network interfaces using network card index from `NetworkCardIndex` list of EC2 DescribeInstances response, diff --git a/tests/integration-tests/configs/new_instance_types.yaml b/tests/integration-tests/configs/new_instance_types.yaml index a0d4b35061..9a1d66cf17 100644 --- a/tests/integration-tests/configs/new_instance_types.yaml +++ b/tests/integration-tests/configs/new_instance_types.yaml @@ -87,12 +87,6 @@ test-suites: oss: {{ common.OSS_ONE_PER_DISTRO }} schedulers: ["slurm"] networking: - test_cluster_networking.py::test_cluster_in_private_subnet: - dimensions: - - regions: {{ NEW_REGIONS }} - instances: {{ NEW_INSTANCE_TYPES }} - oss: ["ubuntu2004"] - schedulers: ["slurm"] # Useful for instances with multiple network interfaces test_multi_cidr.py::test_multi_cidr: dimensions: diff --git a/tests/integration-tests/configs/new_os.yaml b/tests/integration-tests/configs/new_os.yaml index fd5968770f..e2850c645d 100644 --- a/tests/integration-tests/configs/new_os.yaml +++ b/tests/integration-tests/configs/new_os.yaml @@ -199,12 +199,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: {{ NEW_OS }} schedulers: ["slurm"] - test_slurm.py::test_slurm_scaling: - dimensions: - - regions: ["us-west-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: {{ NEW_OS }} - schedulers: ["slurm"] storage: # Commercial regions that can't test FSx: ap-northeast-1, ap-southeast-1, ap-southeast-2, eu-central-1, eu-north-1, eu-west-1, eu-west-2, us-east-1, us-east-2, us-west-1, us-west-2 test_fsx_lustre.py::test_fsx_lustre: diff --git a/tests/integration-tests/configs/released.yaml b/tests/integration-tests/configs/released.yaml index 4c8466ed77..ece65ee6a9 100644 --- a/tests/integration-tests/configs/released.yaml +++ b/tests/integration-tests/configs/released.yaml @@ -188,20 +188,7 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["ubuntu2204"] schedulers: ["slurm"] - multiple_nics: - test_multiple_nics.py::test_multiple_nics: - dimensions: - - regions: ["use1-az6"] - instances: ["c6in.32xlarge"] - oss: {{ common.OSS_COMMERCIAL_X86 }} - schedulers: ["slurm"] networking: - test_cluster_networking.py::test_cluster_in_private_subnet: - dimensions: - - regions: ["il-central-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] test_cluster_networking.py::test_existing_eip: dimensions: - regions: ["me-south-1"] @@ -240,24 +227,12 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["alinux2"] schedulers: ["slurm"] - test_security_groups.py::test_additional_sg_and_ssh_from: - dimensions: - - regions: ["eu-north-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["centos7"] - schedulers: ["slurm"] test_security_groups.py::test_overwrite_sg: dimensions: - regions: ["eu-north-1"] instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: {{ common.NOT_RELEASED_OSES }} schedulers: ["slurm"] - test_placement_group.py::test_placement_group: - dimensions: - - regions: ["eu-central-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] pcluster_api: test_api_infrastructure.py::test_api_infrastructure_with_default_parameters: dimensions: @@ -319,12 +294,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["ubuntu2204"] schedulers: ["slurm"] - test_slurm.py::test_slurm_from_login_nodes_in_private_network: - dimensions: - - regions: ["eu-west-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] test_slurm.py::test_slurm_pmix: # TODO: include in main test_slurm to reduce number of created clusters dimensions: - regions: ["ap-south-1"] @@ -335,12 +304,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: ["rhel8"] schedulers: ["slurm"] - test_slurm.py::test_slurm_scaling: - dimensions: - - regions: ["use2-az2"] # do not move, unless instance type support is moved as well - instances: [{{ common.instance("instance_type_1") }}] - oss: [ "alinux2" ] - schedulers: [ "slurm" ] test_slurm.py::test_error_handling: dimensions: - regions: ["ca-central-1"] @@ -353,18 +316,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["rhel8"] schedulers: ["slurm"] - test_slurm.py::test_slurm_protected_mode_on_cluster_create: - dimensions: - - regions: ["ap-east-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["centos7"] - schedulers: ["slurm"] - test_slurm.py::test_fast_capacity_failover: - dimensions: - - regions: ["ap-east-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] test_slurm.py::test_slurm_config_update: dimensions: - regions: [ "ap-east-1" ] @@ -377,48 +328,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["ubuntu2204"] schedulers: ["slurm"] - test_slurm.py::test_scontrol_reboot: - dimensions: - - regions: ["us-east-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["ubuntu2004"] - schedulers: ["slurm"] - test_slurm.py::test_scontrol_reboot_ec2_health_checks: - dimensions: - - regions: ["us-east-2"] - instances: ["t2.medium"] - oss: ["ubuntu2204"] - schedulers: ["slurm"] - test_slurm.py::test_slurm_overrides: - dimensions: - - regions: ["me-south-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["ubuntu2004"] - schedulers: ["slurm"] - test_slurm_accounting.py::test_slurm_accounting: - dimensions: - - regions: ["ap-south-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["ubuntu2204"] - schedulers: ["slurm"] - test_slurm_accounting.py::test_slurm_accounting_disabled_to_enabled_update: - dimensions: - - regions: ["us-west-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["centos7"] - schedulers: ["slurm"] - test_slurm.py::test_slurm_custom_partitions: - dimensions: - - regions: ["ap-northeast-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["ubuntu2004"] - schedulers: ["slurm"] - test_custom_munge_key.py::test_custom_munge_key: - dimensions: - - regions: ["eu-west-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] spot: test_spot.py::test_spot_default: dimensions: @@ -427,22 +336,8 @@ test-suites: oss: ["centos7"] schedulers: ["slurm"] storage: - test_fsx_lustre.py::test_fsx_lustre: - dimensions: - - regions: ["eu-west-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] - - regions: ["eu-north-1"] - instances: {{ common.INSTANCES_DEFAULT_ARM }} - oss: ["ubuntu2204"] - schedulers: ["slurm"] test_fsx_lustre.py::test_fsx_lustre_dra: dimensions: - - regions: [ "eu-west-2" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: [ "alinux2" ] - schedulers: [ "slurm" ] - regions: [ "eu-north-1" ] instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: [ "ubuntu2204" ] @@ -453,32 +348,6 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: [ "ubuntu2204" ] schedulers: [ "slurm" ] - # The checks performed in test_multiple_fsx is the same as test_fsx_lustre. - # We should consider this when assigning dimensions to each test. - test_fsx_lustre.py::test_multiple_fsx: - dimensions: - - regions: ["eu-west-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] - schedulers: ["slurm"] - - regions: ["eu-west-2"] - instances: {{ common.INSTANCES_DEFAULT_ARM }} - oss: ["ubuntu2004"] - schedulers: ["slurm"] - - regions: ["cn-north-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] - - regions: [ "us-gov-west-1" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: {{ common.NOT_RELEASED_OSES }} - schedulers: [ "slurm" ] - test_fsx_lustre.py::test_multi_az_fsx: - dimensions: - - regions: ["eu-west-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["centos7"] - schedulers: ["slurm"] test_fsx_lustre.py::test_fsx_lustre_configuration_options: dimensions: - regions: ["us-east-2"] @@ -491,82 +360,17 @@ test-suites: instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: ["rhel8"] schedulers: ["slurm"] - # EFS tests can be done in any region. - test_efs.py::test_efs_compute_az: - dimensions: - - regions: ["ca-central-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] - test_efs.py::test_efs_same_az: - dimensions: - - regions: [ "ca-central-1" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: {{ common.NOT_RELEASED_OSES }} - schedulers: [ "slurm" ] - # The checks performed in test_efs_same_az is similar to test_multiple_efs. - # We should consider this when assigning dimensions to each test. test_efs.py::test_multiple_efs: dimensions: - - regions: ["cn-northwest-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["awsbatch"] - regions: [ "ca-central-1" ] instances: {{ common.INSTANCES_DEFAULT_ARM }} oss: ["ubuntu2204"] schedulers: [ "slurm" ] - - regions: ["us-gov-east-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] - schedulers: ["slurm"] - test_raid.py::test_raid_fault_tolerance_mode: - dimensions: - - regions: ["cn-northwest-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] - test_raid.py::test_raid_performance_mode: - dimensions: - - regions: ["ap-south-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["centos7"] - schedulers: ["slurm"] - - regions: [ "ap-south-1" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - schedulers: [ "slurm" ] - oss: {{ common.NOT_RELEASED_OSES }} - - regions: ["us-gov-east-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] - schedulers: ["slurm"] - test_ebs.py::test_ebs_multiple: - dimensions: - - regions: ["us-east-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["ubuntu2204"] - schedulers: ["slurm"] test_ebs.py::test_ebs_single: - dimensions: - - regions: ["us-gov-east-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["alinux2"] - schedulers: ["slurm"] - test_ebs.py::test_ebs_snapshot: - dimensions: - - regions: [ "us-east-2" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: {{ common.NOT_RELEASED_OSES }} - schedulers: [ "slurm" ] - - regions: ["cn-northwest-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] - schedulers: ["slurm"] - test_ebs.py::test_ebs_existing: dimensions: - regions: ["us-east-2"] instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] + oss: ["ubuntu2204"] schedulers: ["slurm"] test_deletion_policy.py::test_retain_on_deletion: dimensions: @@ -581,18 +385,6 @@ test-suites: instances: ["m5d.xlarge"] # SSD based instance oss: ["alinux2"] schedulers: ["slurm"] - test_internal_efs.py::test_internal_efs: - dimensions: - - regions: [ "us-west-2" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: [ "ubuntu2204" ] - schedulers: [ "slurm" ] - test_shared_home.py::test_shared_home: - dimensions: - - regions: [ "us-west-1" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: [ "alinux2" ] - schedulers: [ "slurm" ] tags: test_tag_propagation.py::test_tag_propagation: dimensions: @@ -606,35 +398,3 @@ test-suites: - regions: ["eu-central-1"] instances: {{ common.INSTANCES_DEFAULT_X86 }} oss: ["ubuntu2004"] - test_update.py::test_update_compute_ami: - dimensions: - - regions: ["eu-west-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] - test_update.py::test_update_instance_list: - dimensions: - - regions: ["ap-south-1"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["rhel8"] - schedulers: ["slurm"] - test_update.py::test_queue_parameters_update: - dimensions: - - regions: [ "ap-south-1" ] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: {{ common.NOT_RELEASED_OSES }} - schedulers: [ "slurm" ] - test_update.py::test_dynamic_file_systems_update: - dimensions: - - regions: ["eu-west-2"] - instances: {{ common.INSTANCES_DEFAULT_X86 }} - oss: ["centos7"] - schedulers: ["slurm"] - - regions: ["ap-northeast-1"] - instances: {{ common.INSTANCES_DEFAULT_ARM }} - oss: ["ubuntu2204"] - schedulers: ["slurm"] - test_update.py::test_multi_az_create_and_update: - dimensions: - - regions: ["eu-west-2"] - schedulers: ["slurm"] - oss: ["alinux2"] diff --git a/tests/integration-tests/test_runner.py b/tests/integration-tests/test_runner.py index b94dc95aed..6ed4e648e5 100644 --- a/tests/integration-tests/test_runner.py +++ b/tests/integration-tests/test_runner.py @@ -753,6 +753,14 @@ def _check_args(args): raise argparse.ArgumentTypeError("'{0}' is not a valid test config".format(args.tests_config)) +def unset_proxy(): + """Unset proxies""" + os.environ.pop("HTTP_PROXY", None) + os.environ.pop("HTTPS_PROXY", None) + os.environ.pop("http_proxy", None) + os.environ.pop("https_proxy", None) + + def _run_sequential(args): # Redirect stdout to file if not args.show_output: @@ -779,6 +787,9 @@ def main(): _check_args(args) logger.info("Parsed test_runner parameters {0}".format(args)) + # Unset any proxies used to avoid network issues with tests, such as DCV + unset_proxy() + _make_logging_dirs(args.output_dir) if args.sequential: