From 777374ea22998d61f5aac93266c36a21edb877c7 Mon Sep 17 00:00:00 2001 From: Ivan Orlov Date: Wed, 17 Jul 2024 22:38:57 +0000 Subject: [PATCH] Add validator for Terraform version and SlurmGCP6 --- pkg/validators/adhoc.go | 72 +++++++++++++++++++ pkg/validators/validators.go | 5 +- pkg/validators/validators_test.go | 11 +-- .../tasks/create_deployment_directory.yml | 1 + tools/enforce_coverage.pl | 2 +- tools/validate_configs/validate_configs.sh | 2 +- 6 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 pkg/validators/adhoc.go diff --git a/pkg/validators/adhoc.go b/pkg/validators/adhoc.go new file mode 100644 index 0000000000..d7503593ca --- /dev/null +++ b/pkg/validators/adhoc.go @@ -0,0 +1,72 @@ +// Copyright 2023 "Google LLC" +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validators + +import ( + "encoding/json" + "fmt" + "hpc-toolkit/pkg/config" + "os/exec" + "strings" +) + +func testTfVersionForSlurm(bp config.Blueprint, _ config.Dict) error { + slurm := false + bp.WalkModulesSafe(func(_ config.ModulePath, m *config.Module) { + if strings.HasSuffix(m.Source, "slurm-gcp-v6-controller") { + slurm = true + } + }) + + if !slurm { + return nil + } + + ver, err := tfVersion() + if err != nil { + return nil + } + + if ver <= "1.4.0" { + return nil + } + + return fmt.Errorf("using a newer version of Terraform can lead to controller replacement on reconfigure for Slurm GCP v6\n\n" + + "Please be advised of this known issue: https://github.com/GoogleCloudPlatform/hpc-toolkit/issues/2774\n" + + "Until resolved it is advised to use Terraform 1.4.0 with Slurm deployments.\n\n" + + "To silence this warning, add flag: --skip-validators=test_tf_version_for_slurm") + +} + +func tfVersion() (string, error) { + path, err := exec.LookPath("terraform") + if err != nil { + return "", err + } + + out, err := exec.Command(path, "version", "--json").Output() + if err != nil { + return "", err + } + + var version struct { + TerraformVersion string `json:"terraform_version"` + } + if err := json.Unmarshal(out, &version); err != nil { + return "", err + } + + return version.TerraformVersion, nil +} diff --git a/pkg/validators/validators.go b/pkg/validators/validators.go index 21cf622144..18397bfaa6 100644 --- a/pkg/validators/validators.go +++ b/pkg/validators/validators.go @@ -54,6 +54,7 @@ const ( testZoneInRegionName = "test_zone_in_region" testModuleNotUsedName = "test_module_not_used" testDeploymentVariableNotUsedName = "test_deployment_variable_not_used" + testTfVersionForSlurmName = "test_tf_version_for_slurm" ) func implementations() map[string]func(config.Blueprint, config.Dict) error { @@ -65,6 +66,7 @@ func implementations() map[string]func(config.Blueprint, config.Dict) error { testZoneInRegionName: testZoneInRegion, testModuleNotUsedName: testModuleNotUsed, testDeploymentVariableNotUsedName: testDeploymentVariableNotUsed, + testTfVersionForSlurmName: testTfVersionForSlurm, } } @@ -165,7 +167,8 @@ func defaults(bp config.Blueprint) []config.Validator { defaults := []config.Validator{ {Validator: testModuleNotUsedName}, - {Validator: testDeploymentVariableNotUsedName}} + {Validator: testDeploymentVariableNotUsedName}, + {Validator: testTfVersionForSlurmName}} // always add the project ID validator before subsequent validators that can // only succeed if credentials can access the project. If the project ID diff --git a/pkg/validators/validators_test.go b/pkg/validators/validators_test.go index 3f690e5821..4f8fc578ad 100644 --- a/pkg/validators/validators_test.go +++ b/pkg/validators/validators_test.go @@ -73,6 +73,7 @@ func (s *MySuite) TestCheckInputs(c *C) { func (s *MySuite) TestDefaultValidators(c *C) { unusedMods := config.Validator{Validator: "test_module_not_used"} unusedVars := config.Validator{Validator: "test_deployment_variable_not_used"} + slurmTf := config.Validator{Validator: "test_tf_version_for_slurm"} prjInp := config.Dict{}.With("project_id", config.GlobalRef("project_id").AsValue()) regInp := prjInp.With("region", config.GlobalRef("region").AsValue()) @@ -93,14 +94,14 @@ func (s *MySuite) TestDefaultValidators(c *C) { { bp := config.Blueprint{} c.Check(defaults(bp), DeepEquals, []config.Validator{ - unusedMods, unusedVars}) + unusedMods, unusedVars, slurmTf}) } { bp := config.Blueprint{Vars: config.Dict{}. With("project_id", cty.StringVal("f00b"))} c.Check(defaults(bp), DeepEquals, []config.Validator{ - unusedMods, unusedVars, projectExists, apisEnabled}) + unusedMods, unusedVars, slurmTf, projectExists, apisEnabled}) } { @@ -109,7 +110,7 @@ func (s *MySuite) TestDefaultValidators(c *C) { With("region", cty.StringVal("narnia"))} c.Check(defaults(bp), DeepEquals, []config.Validator{ - unusedMods, unusedVars, projectExists, apisEnabled, regionExists}) + unusedMods, unusedVars, slurmTf, projectExists, apisEnabled, regionExists}) } { @@ -118,7 +119,7 @@ func (s *MySuite) TestDefaultValidators(c *C) { With("zone", cty.StringVal("danger"))} c.Check(defaults(bp), DeepEquals, []config.Validator{ - unusedMods, unusedVars, projectExists, apisEnabled, zoneExists}) + unusedMods, unusedVars, slurmTf, projectExists, apisEnabled, zoneExists}) } { @@ -128,6 +129,6 @@ func (s *MySuite) TestDefaultValidators(c *C) { With("zone", cty.StringVal("danger"))} c.Check(defaults(bp), DeepEquals, []config.Validator{ - unusedMods, unusedVars, projectExists, apisEnabled, regionExists, zoneExists, zoneInRegion}) + unusedMods, unusedVars, slurmTf, projectExists, apisEnabled, regionExists, zoneExists, zoneInRegion}) } } diff --git a/tools/cloud-build/daily-tests/ansible_playbooks/tasks/create_deployment_directory.yml b/tools/cloud-build/daily-tests/ansible_playbooks/tasks/create_deployment_directory.yml index 4f6d93eca8..f515e26aac 100644 --- a/tools/cloud-build/daily-tests/ansible_playbooks/tasks/create_deployment_directory.yml +++ b/tools/cloud-build/daily-tests/ansible_playbooks/tasks/create_deployment_directory.yml @@ -34,6 +34,7 @@ ansible.builtin.command: | ./ghpc create -l ERROR "{{ blueprint_yaml }}" \ --backend-config bucket={{ state_bucket }} \ + --skip-validators=test_tf_version_for_slurm \ --vars project_id={{ project }} \ --vars deployment_name={{ deployment_name }} \ {{ deployment_vars_str if deployment_vars_str is defined else '' }} diff --git a/tools/enforce_coverage.pl b/tools/enforce_coverage.pl index b258d07869..2f1f1211f0 100755 --- a/tools/enforce_coverage.pl +++ b/tools/enforce_coverage.pl @@ -24,7 +24,7 @@ cmd 40 pkg/shell 0 pkg/logging 0 - pkg/validators 13 + pkg/validators 10 pkg/inspect 60 pkg/modulewriter 79 pkg 80 diff --git a/tools/validate_configs/validate_configs.sh b/tools/validate_configs/validate_configs.sh index 3a3abde3e8..fb92005498 100755 --- a/tools/validate_configs/validate_configs.sh +++ b/tools/validate_configs/validate_configs.sh @@ -26,7 +26,7 @@ run_test() { exampleFile=$(basename "$example") DEPLOYMENT=$(echo "${exampleFile%.yaml}-$(basename "${tmpdir##*.}")" | sed -e 's/\(.*\)/\L\1/') PROJECT="invalid-project" - VALIDATORS_TO_SKIP="test_project_exists,test_apis_enabled,test_region_exists,test_zone_exists,test_zone_in_region" + VALIDATORS_TO_SKIP="test_project_exists,test_apis_enabled,test_region_exists,test_zone_exists,test_zone_in_region,test_tf_version_for_slurm" GHPC_PATH="${cwd}/ghpc" BP_PATH="${cwd}/${example}" # Cover the three possible starting sequences for local sources: ./ ../ /