Split spack functionality into setup and execute modules

community/examples/AMD/hpc-amd-slurm.yaml

@@ -39,11 +39,16 @@ deployment_groups:
     settings:
       local_mount: /sw
 
-  - id: spack
+  - id: spack-setup
     source: community/modules/scripts/spack-install
     settings:
       install_dir: /sw/spack
       spack_ref: v0.18.1
+
+  - id: spack-execute
+    source: community/modules/scripts/spack-execute
+    use: [spack-setup]
+    settings:
       log_file: /var/log/spack.log
       data_files:
       - destination: /tmp/projections-config.yaml
@@ -113,8 +118,7 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - $(spack.install_spack_deps_runner)
-      - $(spack.install_spack_runner)
+      - $(spack-execute.spack_runner)
       - type: shell
         destination: shutdown.sh
         content: |
@@ -128,7 +132,7 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - $(spack.setup_spack_runner)
+      - $(spack-setup.setup_spack_runner)
       # the following installation of AOCC may be automated in the future
       # with a clear direction to the user to read the EULA at
       # https://developer.amd.com/aocc-compiler-eula/

community/examples/hpc-slurm-gromacs.yaml

@@ -45,10 +45,15 @@ deployment_groups:
       local_mount: /home
 
   ## Install Scripts
-  - id: spack
+  - id: spack-setup
     source: community/modules/scripts/spack-install
     settings:
       install_dir: /sw/spack
+
+  - id: spack-execute
+    source: community/modules/scripts/spack-execute
+    use: [spack-setup]
+    settings:
       log_file: /var/log/spack.log
       data_files:
       - destination: /tmp/projections-config.yaml
@@ -77,13 +82,6 @@ deployment_groups:
         spack install intel-mpi@2018.4.274%gcc@10.3.0
         spack install gromacs@2023.1 %gcc@10.3.0 ^intel-mpi@2018.4.274 ^cmake@3.26.3 %gcc@4.8.5
 
-  - id: spack-startup
-    source: modules/scripts/startup-script
-    settings:
-      runners:
-      - $(spack.install_spack_deps_runner)
-      - $(spack.install_spack_runner)
-
   - id: compute_partition
     source: community/modules/compute/SchedMD-slurm-on-gcp-partition
     use:
@@ -111,7 +109,7 @@ deployment_groups:
     - homefs
     - appsfs
     - slurm_controller
-    - spack-startup
+    - spack-execute
     settings:
       login_machine_type: c2-standard-4
       login_scopes:

community/examples/hpc-slurm-ramble-gromacs.yaml

@@ -35,10 +35,15 @@ deployment_groups:
     source: modules/network/vpc
 
   ## Install Scripts
-  - id: spack
+  - id: spack-install
     source: community/modules/scripts/spack-install
     settings:
       install_dir: /opt/apps/spack
+
+  - id: spack-execute
+    source: community/modules/scripts/spack-execute
+    use: [spack-install]
+    settings:
       log_file: /var/log/spack.log
       commands: |
         # Un-comment and update mirror_url to install from spack cache
@@ -55,7 +60,7 @@ deployment_groups:
 
   - id: ramble-execute
     source: community/modules/scripts/ramble-execute
-    use: [spack, ramble-setup]
+    use: [spack-install, ramble-setup]
     settings:
       commands: |
         ramble workspace create gromacs -c /opt/apps/gromacs_scaling.yaml -t /opt/apps/execute_experiment.tpl
@@ -66,8 +71,7 @@ deployment_groups:
     source: modules/scripts/startup-script
     settings:
       runners:
-      - $(spack.install_spack_deps_runner)
-      - $(spack.install_spack_runner)
+      - $(spack-execute.spack_runner)
       - type: data
         destination: /opt/apps/gromacs_scaling.yaml
         content: |

community/modules/scripts/spack-execute/README.md

@@ -0,0 +1,63 @@
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+Copyright 2023 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
+| <a name="requirement_local"></a> [local](#requirement\_local) | ~> 2.0.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_local"></a> [local](#provider\_local) | ~> 2.0.0 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_startup_script"></a> [startup\_script](#module\_startup\_script) | github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script | v1.20.0 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [local_file.debug_file_ansible_execute](https://registry.terraform.io/providers/hashicorp/local/latest/docs/resources/file) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_commands"></a> [commands](#input\_commands) | String of commands to run within this module | `string` | `null` | no |
+| <a name="input_data_files"></a> [data\_files](#input\_data\_files) | A list of files to be transferred prior to running commands. <br>It must specify one of 'source' (absolute local file path) or 'content' (string).<br>It must specify a 'destination' with absolute path where file should be placed. | `list(map(string))` | `[]` | no |
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Name of deployment, used to name bucket containing spack scripts. | `string` | n/a | yes |
+| <a name="input_gcs_bucket_path"></a> [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | The GCS path for storage bucket and the object, starting with `gs://`. | `string` | n/a | yes |
+| <a name="input_labels"></a> [labels](#input\_labels) | Key-value pairs of labels to be added to created resources. | `map(string)` | n/a | yes |
+| <a name="input_log_file"></a> [log\_file](#input\_log\_file) | Defines the logfile that script output will be written to | `string` | `"/var/log/spack.log"` | no |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | Project in which the HPC deployment will be created. | `string` | n/a | yes |
+| <a name="input_region"></a> [region](#input\_region) | Region to place bucket containing spack scripts. | `string` | n/a | yes |
+| <a name="input_spack_runner"></a> [spack\_runner](#input\_spack\_runner) | Runner from previous spack-install or spack-execute to be chained with scripts generated by this module. | <pre>object({<br>    type        = string<br>    content     = string<br>    destination = string<br>  })</pre> | n/a | yes |
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_controller_startup_script"></a> [controller\_startup\_script](#output\_controller\_startup\_script) | Path to the Spack installation script, duplicate for SLURM controller. |
+| <a name="output_gcs_bucket_path"></a> [gcs\_bucket\_path](#output\_gcs\_bucket\_path) | Bucket containing the startup scripts for spack, to be reused by spack-execute module. |
+| <a name="output_spack_runner"></a> [spack\_runner](#output\_spack\_runner) | Single runner that combines scripts from this module and any previously chained spack-execute or spack-install modules. |
+| <a name="output_startup_script"></a> [startup\_script](#output\_startup\_script) | Path to the Spack installation script. |
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

community/modules/scripts/spack-execute/main.tf

@@ -0,0 +1,69 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  # This label allows for billing report tracking based on module.
+  labels = merge(var.labels, { ghpc_module = "spack-execute" })
+}
+
+locals {
+  commands_content = var.commands == null ? "echo 'no spack commands provided'" : indent(4, yamlencode(var.commands))
+
+  execute_contents = templatefile(
+    "${path.module}/templates/execute_commands.yml.tpl",
+    {
+      pre_script = ". /etc/profile.d/spack.sh"
+      log_file   = var.log_file
+      commands   = local.commands_content
+    }
+  )
+
+  data_runners = [for data_file in var.data_files : merge(data_file, { type = "data" })]
+
+  execute_md5 = substr(md5(local.execute_contents), 0, 4)
+  execute_runner = {
+    type        = "ansible-local"
+    content     = local.execute_contents
+    destination = "spack_execute_${local.execute_md5}.yml"
+  }
+
+  runners = concat([var.spack_runner], local.data_runners, [local.execute_runner])
+
+  # Destinations should be unique while also being known at time of apply
+  combined_unique_string = join("\n", [for runner in local.runners : runner["destination"]])
+  combined_md5           = substr(md5(local.combined_unique_string), 0, 4)
+  combined_runner = {
+    type        = "shell"
+    content     = module.startup_script.startup_script
+    destination = "combined_install_spack_${local.combined_md5}.sh"
+  }
+}
+
+module "startup_script" {
+  source = "github.com/GoogleCloudPlatform/hpc-toolkit//modules/scripts/startup-script?ref=v1.20.0"
+
+  labels          = local.labels
+  project_id      = var.project_id
+  deployment_name = var.deployment_name
+  region          = var.region
+  runners         = local.runners
+  gcs_bucket_path = var.gcs_bucket_path
+}
+
+resource "local_file" "debug_file_ansible_execute" {
+  content  = local.execute_contents
+  filename = "${path.module}/debug_execute_${local.execute_md5}.yml"
+}

community/modules/scripts/spack-execute/outputs.tf

@@ -0,0 +1,35 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+output "startup_script" {
+  description = "Path to the Spack installation script."
+  value       = module.startup_script.startup_script
+}
+
+output "controller_startup_script" {
+  description = "Path to the Spack installation script, duplicate for SLURM controller."
+  value       = module.startup_script.startup_script
+}
+
+output "spack_runner" {
+  description = "Single runner that combines scripts from this module and any previously chained spack-execute or spack-install modules."
+  value       = local.combined_runner
+}
+
+output "gcs_bucket_path" {
+  description = "Bucket containing the startup scripts for spack, to be reused by spack-execute module."
+  value       = var.gcs_bucket_path
+}

community/modules/scripts/spack-execute/variables.tf

@@ -0,0 +1,89 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "Project in which the HPC deployment will be created."
+  type        = string
+}
+
+variable "deployment_name" {
+  description = "Name of deployment, used to name bucket containing spack scripts."
+  type        = string
+}
+
+variable "region" {
+  description = "Region to place bucket containing spack scripts."
+  type        = string
+}
+
+variable "labels" {
+  description = "Key-value pairs of labels to be added to created resources."
+  type        = map(string)
+}
+
+variable "log_file" {
+  description = "Defines the logfile that script output will be written to"
+  default     = "/var/log/spack.log"
+  type        = string
+}
+
+variable "data_files" {
+  description = <<-EOT
+    A list of files to be transferred prior to running commands. 
+    It must specify one of 'source' (absolute local file path) or 'content' (string).
+    It must specify a 'destination' with absolute path where file should be placed.
+  EOT
+  type        = list(map(string))
+  default     = []
+  validation {
+    condition     = alltrue([for r in var.data_files : substr(r["destination"], 0, 1) == "/"])
+    error_message = "All destinations must be absolute paths and start with '/'."
+  }
+  validation {
+    condition = alltrue([
+      for r in var.data_files :
+      can(r["content"]) != can(r["source"])
+    ])
+    error_message = "A data_file must specify either 'content' or 'source', but never both."
+  }
+  validation {
+    condition = alltrue([
+      for r in var.data_files :
+      lookup(r, "content", lookup(r, "source", null)) != null
+    ])
+    error_message = "A data_file must specify a non-null 'content' or 'source'."
+  }
+}
+
+variable "commands" {
+  description = "String of commands to run within this module"
+  type        = string
+  default     = null
+}
+
+variable "spack_runner" {
+  description = "Runner from previous spack-install or spack-execute to be chained with scripts generated by this module."
+  type = object({
+    type        = string
+    content     = string
+    destination = string
+  })
+}
+
+variable "gcs_bucket_path" {
+  description = "The GCS path for storage bucket and the object, starting with `gs://`."
+  type        = string
+}

community/modules/scripts/spack-execute/versions.tf

@@ -0,0 +1,25 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+terraform {
+  required_version = ">= 1.0.0"
+  required_providers {
+    local = {
+      source  = "hashicorp/local"
+      version = "~> 2.0.0"
+    }
+  }
+}