From c0bc3624abf7db7ad3c949e906ebda3dabd9db10 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Thu, 12 May 2022 16:35:55 +0200
Subject: [PATCH 01/10] DAOSGCP-83 and DAOSGCP-84 Automatic format storage with
 pool and container creation that supports ACLs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 terraform/examples/daos_cluster/variables.tf  | 15 ++++-
 terraform/examples/io500/config/config.sh     | 41 ++++++++++++-
 .../examples/only_daos_server/variables.tf    | 15 ++++-
 terraform/modules/daos_server/main.tf         | 12 +++-
 terraform/modules/daos_server/outputs.tf      |  5 ++
 .../templates/pool_cont_create.tftpl          | 58 +++++++++++++++++++
 .../templates/startup_script.tftpl            |  2 +
 terraform/modules/daos_server/variables.tf    | 15 ++++-
 8 files changed, 151 insertions(+), 12 deletions(-)
 create mode 100644 terraform/modules/daos_server/templates/pool_cont_create.tftpl

diff --git a/terraform/examples/daos_cluster/variables.tf b/terraform/examples/daos_cluster/variables.tf
index ebe1f49..0701795 100644
--- a/terraform/examples/daos_cluster/variables.tf
+++ b/terraform/examples/daos_cluster/variables.tf
@@ -175,9 +175,18 @@ variable "server_pools" {
   description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
   default     = []
   type = list(object({
-    pool_name  = string
-    pool_size  = string
-    containers = list(string)
+    name  = string
+    size  = string
+    tier_ratio = number
+    acls = list(string)
+    properties = list(string)
+    containers = list(object({
+      name = string
+      type = string
+      acls = list(string)
+      properties = list(string)
+      user_attributes = list(string)
+      }))
     })
   )
 }
diff --git a/terraform/examples/io500/config/config.sh b/terraform/examples/io500/config/config.sh
index ac826ef..3a3b98b 100644
--- a/terraform/examples/io500/config/config.sh
+++ b/terraform/examples/io500/config/config.sh
@@ -30,13 +30,49 @@ ID=""
 PREEMPTIBLE_INSTANCES="true"
 SSH_USER="daos-user"
 DAOS_ALLOW_INSECURE="false"
+# DAOS_POOLS='[]'
+DAOS_POOLS='[
+  {
+    name = "io500_pool",
+    size = "1TB",
+    tier_ratio = 3
+    acls = [
+      "A::OWNER@:rwdtTaAo",
+      "A:G:GROUP@:rwtT",
+      "A::EVERYONE@:rwdtTaAo"
+    ],
+    properties = [
+      "reclaim:disabled"
+    ],
+    containers = [
+      {
+        name = "io500_cont",
+        type = "POSIX",
+        acls = [
+          "A::OWNER@:rwdtTaAo",
+          "A:G:GROUP@:rwtT",
+          "A::EVERYONE@:rwdtTaAo"
+        ],
+        properties = [
+          "cksum:sha1",
+          "dedup:hash",
+          "rf:0"
+        ],
+        user_attributes = [
+          "automount true",
+          "mount_path /mnt/daos"
+        ]
+      }
+    ]
+  }
+]'
 
 # Server(s)
 DAOS_SERVER_INSTANCE_COUNT="1"
-DAOS_SERVER_MACHINE_TYPE=n2-highmem-32 # n2-custom-20-131072 n2-custom-40-262144 n2-highmem-32 n2-standard-2
+DAOS_SERVER_MACHINE_TYPE=n2-standard-16 # n2-custom-20-131072 n2-custom-40-262144 n2-highmem-32 n2-standard-2
 DAOS_SERVER_DISK_COUNT=8
 DAOS_SERVER_CRT_TIMEOUT=300
-DAOS_SERVER_SCM_SIZE=100
+DAOS_SERVER_SCM_SIZE=45
 DAOS_SERVER_GVNIC=false
 
 # Client(s)
@@ -87,6 +123,7 @@ export TF_VAR_server_machine_type="${DAOS_SERVER_MACHINE_TYPE}"
 export TF_VAR_server_os_project="${TF_VAR_project_id}"
 export TF_VAR_server_os_family="daos-server-io500-centos-7"
 export TF_VAR_server_gvnic="${DAOS_SERVER_GVNIC}"
+export TF_VAR_server_pools="${DAOS_POOLS}"
 # Clients
 export TF_VAR_client_preemptible=${PREEMPTIBLE_INSTANCES}
 export TF_VAR_client_number_of_instances=${DAOS_CLIENT_INSTANCE_COUNT}
diff --git a/terraform/examples/only_daos_server/variables.tf b/terraform/examples/only_daos_server/variables.tf
index cbe8626..e557aef 100644
--- a/terraform/examples/only_daos_server/variables.tf
+++ b/terraform/examples/only_daos_server/variables.tf
@@ -155,9 +155,18 @@ variable "server_pools" {
   description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
   default     = []
   type = list(object({
-    pool_name  = string
-    pool_size  = string
-    containers = list(string)
+    name  = string
+    size  = string
+    tier_ratio = number
+    acls = list(string)
+    properties = list(string)
+    containers = list(object({
+      name = string
+      type = string
+      acls = list(string)
+      properties = list(string)
+      user_attributes = list(string)
+      }))
     })
   )
 }
diff --git a/terraform/modules/daos_server/main.tf b/terraform/modules/daos_server/main.tf
index 99ca3d7..1beb1a6 100644
--- a/terraform/modules/daos_server/main.tf
+++ b/terraform/modules/daos_server/main.tf
@@ -17,7 +17,7 @@
 locals {
   os_project         = var.os_project != null ? var.os_project : var.project_id
   subnetwork_project = var.subnetwork_project != null ? var.subnetwork_project : var.project_id
-  servers            = format("%s-[%04s-%04s]", var.instance_base_name, 1, var.number_of_instances)
+  servers            = var.number_of_instances == 1 ? local.first_server : format("%s-[%04s-%04s]", var.instance_base_name, 1, var.number_of_instances)
   first_server       = format("%s-%04s", var.instance_base_name, 1)
   max_aps            = var.number_of_instances > 5 ? 5 : (var.number_of_instances % 2) == 1 ? var.number_of_instances : var.number_of_instances - 1
   access_points      = formatlist("%s-%04s", var.instance_base_name, range(1, local.max_aps + 1))
@@ -28,6 +28,7 @@ locals {
   crt_timeout       = var.daos_crt_timeout
   daos_ca_secret_id = basename(google_secret_manager_secret.daos_ca.id)
   allow_insecure    = var.allow_insecure
+  pools             = var.pools
 
   # Google Virtual NIC (gVNIC) network interface
   nic_type                    = var.gvnic ? "GVNIC" : "VIRTIO_NET"
@@ -87,12 +88,21 @@ locals {
     }
   )
 
+  pool_cont_create_content = templatefile(
+    "${path.module}/templates/pool_cont_create.tftpl",
+    {
+      servers = local.servers
+      pools   = local.pools
+    }
+  )
+
   startup_script = templatefile(
     "${path.module}/templates/startup_script.tftpl",
     {
       first_server          = local.first_server
       certs_gen_content     = local.certs_gen_content
       certs_install_content = local.certs_install_content
+      pool_cont_create_content = local.pool_cont_create_content
     }
   )
 }
diff --git a/terraform/modules/daos_server/outputs.tf b/terraform/modules/daos_server/outputs.tf
index 8258aea..82c112a 100644
--- a/terraform/modules/daos_server/outputs.tf
+++ b/terraform/modules/daos_server/outputs.tf
@@ -46,3 +46,8 @@ output "certs_install_content" {
   description = "Cert installation content to include in the daos_client startup script"
   value       = local.certs_install_content
 }
+
+output "daos_pools" {
+  description = "Specification of pools and containers to create"
+  value       = local.pools
+}
diff --git a/terraform/modules/daos_server/templates/pool_cont_create.tftpl b/terraform/modules/daos_server/templates/pool_cont_create.tftpl
new file mode 100644
index 0000000..d36e1b7
--- /dev/null
+++ b/terraform/modules/daos_server/templates/pool_cont_create.tftpl
@@ -0,0 +1,58 @@
+%{ if length(pools) != 0 }
+set -x
+
+echo "BEGIN: DAOS pool and container creation"
+
+if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
+  # Wait for servers to start
+  until dmg network scan | grep --fixed-strings "${servers}"
+  do
+    sleep 5
+  done
+  echo "All DAOS Servers started"
+
+  echo "Formatting storage on servers: ${servers}"
+  dmg storage format
+  dmg system query -v
+  echo "Done formating DAOS server"
+
+  echo "Creating requested pools and containers"
+  %{for pool in pools}
+    dmg pool create \
+      --size=${pool.size} \
+      --tier-ratio=${pool.tier_ratio} \
+      ${pool.name}
+
+    %{for property in pool.properties}
+      dmg pool set-prop ${pool.name} ${property}
+    %{endfor}
+
+    %{for acl in pool.acls}
+      dmg pool update-acl --entry ${acl} ${pool.name}
+    %{endfor}
+
+    %{for container in pool.containers}
+      daos container create \
+        --type ${container.type} \
+        --properties=%{~ for property in container.properties ~}
+                        %{~ if property == element(container.properties, length(container.properties)-1) ~}
+                          ${~ property ~}
+                        %{~ else ~}${~ property ~},%{~ endif ~}
+                      %{~ endfor } \
+        ${pool.name} \
+        --label ${container.name}
+
+      %{for acl in container.acls}
+        daos cont update-acl ${pool.name} ${container.name} --entry ${acl}
+      %{endfor}
+
+      %{for attribute in container.user_attributes}
+        daos cont set-attr ${pool.name} ${container.name} ${attribute}
+      %{endfor}
+    %{endfor}
+  %{endfor}
+  echo "Done creating requested pools and containers"
+fi
+
+echo "END: DAOS pool and container creation"
+%{ endif }
diff --git a/terraform/modules/daos_server/templates/startup_script.tftpl b/terraform/modules/daos_server/templates/startup_script.tftpl
index e9d8723..e3b009e 100644
--- a/terraform/modules/daos_server/templates/startup_script.tftpl
+++ b/terraform/modules/daos_server/templates/startup_script.tftpl
@@ -38,4 +38,6 @@ systemctl start daos_server
 systemctl enable daos_agent
 systemctl start daos_agent
 
+${ pool_cont_create_content }
+
 echo "END: DAOS Server Startup Script"
diff --git a/terraform/modules/daos_server/variables.tf b/terraform/modules/daos_server/variables.tf
index a91bc96..eaffdaf 100644
--- a/terraform/modules/daos_server/variables.tf
+++ b/terraform/modules/daos_server/variables.tf
@@ -170,9 +170,18 @@ variable "pools" {
   description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
   default     = []
   type = list(object({
-    pool_name  = string
-    pool_size  = string
-    containers = list(string)
+    name  = string
+    size  = string
+    tier_ratio = number
+    acls = list(string)
+    properties = list(string)
+    containers = list(object({
+      name = string
+      type = string
+      acls = list(string)
+      properties = list(string)
+      user_attributes = list(string)
+      }))
     })
   )
 }

From 9294d5ad6add9f7723a8b741f5cdc978c2c686ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Thu, 12 May 2022 17:56:26 +0200
Subject: [PATCH 02/10] Update examples
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 .../terraform.tfvars.perf.example             | 33 ++++++++++++++++---
 .../daos_cluster/terraform.tfvars.tco.example | 33 ++++++++++++++++---
 .../terraform.tfvars.perf.example             | 33 ++++++++++++++++---
 .../terraform.tfvars.tco.example              | 33 ++++++++++++++++---
 4 files changed, 112 insertions(+), 20 deletions(-)

diff --git a/terraform/examples/daos_cluster/terraform.tfvars.perf.example b/terraform/examples/daos_cluster/terraform.tfvars.perf.example
index 77d6620..33009c8 100644
--- a/terraform/examples/daos_cluster/terraform.tfvars.perf.example
+++ b/terraform/examples/daos_cluster/terraform.tfvars.perf.example
@@ -26,11 +26,34 @@ server_daos_scm_size       = 45
 # server_os_disk_type        = "pd-ssd"
 
 # pools = [
-#  {
-#    pool_name = "test_pool"
-#    pool_size = "1TB"
-#    containers = [ "test_container1", "test_container2" ]
-#  }
+#   {
+#     name = "test_pool",
+#     size = "1TB",
+#     tier_ratio = 3
+#     acls = [
+#       "A::OWNER@:rwdtTaAo",
+#       "A:G:GROUP@:rwtT"
+#     ],
+#     properties = [
+#       "reclaim:disabled"
+#     ],
+#     containers = [
+#       {
+#         name = "test_container1",
+#         type = "POSIX",
+#         acls = [
+#           "A::OWNER@:rwdtTaAo",
+#           "A:G:GROUP@:rwtT"
+#         ],
+#         properties = [
+#           "rf:0"
+#         ],
+#         user_attributes = [
+#           "automount true"
+#         ]
+#       }
+#     ]
+#   }
 # ]
 
 # client_number_of_instances = 16
diff --git a/terraform/examples/daos_cluster/terraform.tfvars.tco.example b/terraform/examples/daos_cluster/terraform.tfvars.tco.example
index 9c324a8..2141c81 100644
--- a/terraform/examples/daos_cluster/terraform.tfvars.tco.example
+++ b/terraform/examples/daos_cluster/terraform.tfvars.tco.example
@@ -26,11 +26,34 @@ server_number_of_instances = 4
 # server_os_disk_type        = "pd-ssd"
 
 # pools = [
-#  {
-#    pool_name = "test_pool"
-#    pool_size = "1TB"
-#    containers = [ "test_container1", "test_container2" ]
-#  }
+#   {
+#     name = "test_pool",
+#     size = "1TB",
+#     tier_ratio = 3
+#     acls = [
+#       "A::OWNER@:rwdtTaAo",
+#       "A:G:GROUP@:rwtT"
+#     ],
+#     properties = [
+#       "reclaim:disabled"
+#     ],
+#     containers = [
+#       {
+#         name = "test_container1",
+#         type = "POSIX",
+#         acls = [
+#           "A::OWNER@:rwdtTaAo",
+#           "A:G:GROUP@:rwtT"
+#         ],
+#         properties = [
+#           "rf:0"
+#         ],
+#         user_attributes = [
+#           "automount true"
+#         ]
+#       }
+#     ]
+#   }
 # ]
 
 # client_number_of_instances = 16
diff --git a/terraform/examples/only_daos_server/terraform.tfvars.perf.example b/terraform/examples/only_daos_server/terraform.tfvars.perf.example
index 72efa6e..351d6cd 100644
--- a/terraform/examples/only_daos_server/terraform.tfvars.perf.example
+++ b/terraform/examples/only_daos_server/terraform.tfvars.perf.example
@@ -25,9 +25,32 @@ server_labels              = {
 # server_os_disk_type        = "pd-ssd"
 
 # pools = [
-#  {
-#    pool_name = "test_pool"
-#    pool_size = "1TB"
-#    containers = [ "test_container1", "test_container2" ]
-#  }
+#   {
+#     name = "test_pool",
+#     size = "1TB",
+#     tier_ratio = 3
+#     acls = [
+#       "A::OWNER@:rwdtTaAo",
+#       "A:G:GROUP@:rwtT"
+#     ],
+#     properties = [
+#       "reclaim:disabled"
+#     ],
+#     containers = [
+#       {
+#         name = "test_container1",
+#         type = "POSIX",
+#         acls = [
+#           "A::OWNER@:rwdtTaAo",
+#           "A:G:GROUP@:rwtT"
+#         ],
+#         properties = [
+#           "rf:0"
+#         ],
+#         user_attributes = [
+#           "automount true"
+#         ]
+#       }
+#     ]
+#   }
 # ]
diff --git a/terraform/examples/only_daos_server/terraform.tfvars.tco.example b/terraform/examples/only_daos_server/terraform.tfvars.tco.example
index c6e0dc7..018155d 100644
--- a/terraform/examples/only_daos_server/terraform.tfvars.tco.example
+++ b/terraform/examples/only_daos_server/terraform.tfvars.tco.example
@@ -25,9 +25,32 @@ server_labels              = {
 # server_os_disk_type        = "pd-ssd"
 
 # pools = [
-#  {
-#    pool_name = "test_pool"
-#    pool_size = "1TB"
-#    containers = [ "test_container1", "test_container2" ]
-#  }
+#   {
+#     name = "test_pool",
+#     size = "1TB",
+#     tier_ratio = 3
+#     acls = [
+#       "A::OWNER@:rwdtTaAo",
+#       "A:G:GROUP@:rwtT"
+#     ],
+#     properties = [
+#       "reclaim:disabled"
+#     ],
+#     containers = [
+#       {
+#         name = "test_container1",
+#         type = "POSIX",
+#         acls = [
+#           "A::OWNER@:rwdtTaAo",
+#           "A:G:GROUP@:rwtT"
+#         ],
+#         properties = [
+#           "rf:0"
+#         ],
+#         user_attributes = [
+#           "automount true"
+#         ]
+#       }
+#     ]
+#   }
 # ]

From 4a4b3972deba7577c3fbceb27ee648b0959b6b29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Thu, 12 May 2022 18:48:16 +0200
Subject: [PATCH 03/10] tflint + pre-commit check
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 terraform/examples/daos_cluster/README.md     |  2 +-
 terraform/examples/daos_cluster/module.json   |  2 +-
 terraform/examples/daos_cluster/variables.tf  | 19 +++++++++----------
 terraform/examples/only_daos_server/README.md |  2 +-
 .../examples/only_daos_server/module.json     |  2 +-
 .../examples/only_daos_server/variables.tf    | 19 +++++++++----------
 terraform/modules/daos_client/README.md       |  3 +--
 terraform/modules/daos_client/main.tf         | 16 ++++++++--------
 terraform/modules/daos_client/module.json     | 11 ++---------
 terraform/modules/daos_server/README.md       |  8 ++++----
 terraform/modules/daos_server/main.tf         |  6 +++---
 terraform/modules/daos_server/module.json     | 18 +++++++++---------
 terraform/modules/daos_server/variables.tf    | 19 +++++++++----------
 13 files changed, 58 insertions(+), 69 deletions(-)

diff --git a/terraform/examples/daos_cluster/README.md b/terraform/examples/daos_cluster/README.md
index 74fe8d7..6d11a71 100644
--- a/terraform/examples/daos_cluster/README.md
+++ b/terraform/examples/daos_cluster/README.md
@@ -282,7 +282,7 @@ No resources.
 | <a name="input_server_os_disk_type"></a> [server\_os\_disk\_type](#input\_server\_os\_disk\_type) | OS disk type ie. pd-ssd, pd-standard | `string` | `"pd-ssd"` | no |
 | <a name="input_server_os_family"></a> [server\_os\_family](#input\_server\_os\_family) | OS GCP image family | `string` | `"daos-server-centos-7"` | no |
 | <a name="input_server_os_project"></a> [server\_os\_project](#input\_server\_os\_project) | OS GCP image project name. Defaults to project\_id if null. | `string` | `null` | no |
-| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    pool_name  = string<br>    pool_size  = string<br>    containers = list(string)<br>    })<br>  )</pre> | `[]` | no |
+| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = list(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = list(string)<br>      user_attributes = list(string)<br>    }))<br>  }))</pre> | `[]` | no |
 | <a name="input_server_preemptible"></a> [server\_preemptible](#input\_server\_preemptible) | If preemptible instances | `string` | `false` | no |
 | <a name="input_server_service_account"></a> [server\_service\_account](#input\_server\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | <pre>object({<br>    email  = string,<br>    scopes = set(string)<br>  })</pre> | <pre>{<br>  "email": null,<br>  "scopes": [<br>    "https://www.googleapis.com/auth/devstorage.read_only",<br>    "https://www.googleapis.com/auth/logging.write",<br>    "https://www.googleapis.com/auth/monitoring.write",<br>    "https://www.googleapis.com/auth/servicecontrol",<br>    "https://www.googleapis.com/auth/service.management.readonly",<br>    "https://www.googleapis.com/auth/trace.append",<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ]<br>}</pre> | no |
 | <a name="input_server_template_name"></a> [server\_template\_name](#input\_server\_template\_name) | MIG template name | `string` | `"daos-server"` | no |
diff --git a/terraform/examples/daos_cluster/module.json b/terraform/examples/daos_cluster/module.json
index 18aecc7..7241284 100644
--- a/terraform/examples/daos_cluster/module.json
+++ b/terraform/examples/daos_cluster/module.json
@@ -232,7 +232,7 @@
     },
     {
       "name": "server_pools",
-      "type": "list(object({\n    pool_name  = string\n    pool_size  = string\n    containers = list(string)\n    })\n  )",
+      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = list(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = list(string)\n      user_attributes = list(string)\n    }))\n  }))",
       "description": "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos.",
       "default": [],
       "required": false
diff --git a/terraform/examples/daos_cluster/variables.tf b/terraform/examples/daos_cluster/variables.tf
index 0701795..c15a130 100644
--- a/terraform/examples/daos_cluster/variables.tf
+++ b/terraform/examples/daos_cluster/variables.tf
@@ -175,20 +175,19 @@ variable "server_pools" {
   description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
   default     = []
   type = list(object({
-    name  = string
-    size  = string
+    name       = string
+    size       = string
     tier_ratio = number
-    acls = list(string)
+    acls       = list(string)
     properties = list(string)
     containers = list(object({
-      name = string
-      type = string
-      acls = list(string)
-      properties = list(string)
+      name            = string
+      type            = string
+      acls            = list(string)
+      properties      = list(string)
       user_attributes = list(string)
-      }))
-    })
-  )
+    }))
+  }))
 }
 
 variable "client_labels" {
diff --git a/terraform/examples/only_daos_server/README.md b/terraform/examples/only_daos_server/README.md
index b1cd13d..3f99ba5 100644
--- a/terraform/examples/only_daos_server/README.md
+++ b/terraform/examples/only_daos_server/README.md
@@ -157,7 +157,7 @@ No resources.
 | <a name="input_server_os_disk_type"></a> [server\_os\_disk\_type](#input\_server\_os\_disk\_type) | OS disk type ie. pd-ssd, pd-standard | `string` | `"pd-ssd"` | no |
 | <a name="input_server_os_family"></a> [server\_os\_family](#input\_server\_os\_family) | OS GCP image family | `string` | `"daos-server-centos-7"` | no |
 | <a name="input_server_os_project"></a> [server\_os\_project](#input\_server\_os\_project) | OS GCP image project name. Defaults to project\_id if null. | `string` | `null` | no |
-| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    pool_name  = string<br>    pool_size  = string<br>    containers = list(string)<br>    })<br>  )</pre> | `[]` | no |
+| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = list(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = list(string)<br>      user_attributes = list(string)<br>    }))<br>  }))</pre> | `[]` | no |
 | <a name="input_server_preemptible"></a> [server\_preemptible](#input\_server\_preemptible) | If preemptible instances | `string` | `false` | no |
 | <a name="input_server_service_account"></a> [server\_service\_account](#input\_server\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | <pre>object({<br>    email  = string,<br>    scopes = set(string)<br>  })</pre> | <pre>{<br>  "email": null,<br>  "scopes": [<br>    "https://www.googleapis.com/auth/devstorage.read_only",<br>    "https://www.googleapis.com/auth/logging.write",<br>    "https://www.googleapis.com/auth/monitoring.write",<br>    "https://www.googleapis.com/auth/servicecontrol",<br>    "https://www.googleapis.com/auth/service.management.readonly",<br>    "https://www.googleapis.com/auth/trace.append",<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ]<br>}</pre> | no |
 | <a name="input_server_template_name"></a> [server\_template\_name](#input\_server\_template\_name) | MIG template name | `string` | `"daos-server"` | no |
diff --git a/terraform/examples/only_daos_server/module.json b/terraform/examples/only_daos_server/module.json
index 1bffb83..77e7147 100644
--- a/terraform/examples/only_daos_server/module.json
+++ b/terraform/examples/only_daos_server/module.json
@@ -130,7 +130,7 @@
     },
     {
       "name": "server_pools",
-      "type": "list(object({\n    pool_name  = string\n    pool_size  = string\n    containers = list(string)\n    })\n  )",
+      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = list(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = list(string)\n      user_attributes = list(string)\n    }))\n  }))",
       "description": "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos.",
       "default": [],
       "required": false
diff --git a/terraform/examples/only_daos_server/variables.tf b/terraform/examples/only_daos_server/variables.tf
index e557aef..46cabf8 100644
--- a/terraform/examples/only_daos_server/variables.tf
+++ b/terraform/examples/only_daos_server/variables.tf
@@ -155,20 +155,19 @@ variable "server_pools" {
   description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
   default     = []
   type = list(object({
-    name  = string
-    size  = string
+    name       = string
+    size       = string
     tier_ratio = number
-    acls = list(string)
+    acls       = list(string)
     properties = list(string)
     containers = list(object({
-      name = string
-      type = string
-      acls = list(string)
-      properties = list(string)
+      name            = string
+      type            = string
+      acls            = list(string)
+      properties      = list(string)
       user_attributes = list(string)
-      }))
-    })
-  )
+    }))
+  }))
 }
 
 variable "server_daos_scm_size" {
diff --git a/terraform/modules/daos_client/README.md b/terraform/modules/daos_client/README.md
index a82bebc..0b5bfc0 100644
--- a/terraform/modules/daos_client/README.md
+++ b/terraform/modules/daos_client/README.md
@@ -55,8 +55,7 @@ No modules.
 
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
-| <a name="input_allow_insecure"></a> [allow\_insecure](#input\_allow\_insecure) | Sets the allow\_insecure setting in the transport\_config section of the daos\_*.yml files | `bool` | `false` | no |
-| <a name="input_certs_install_script_content"></a> [certs\_install\_script\_content](#input\_certs\_install\_script\_content) | Script to install certificates for DAOS secure transport | `string` | n/a | yes |
+| <a name="input_certs_install_content"></a> [certs\_install\_content](#input\_certs\_install\_content) | DAOS certificate installation code to be included in the client startup script | `string` | n/a | yes |
 | <a name="input_daos_agent_yml"></a> [daos\_agent\_yml](#input\_daos\_agent\_yml) | YAML to configure the daos agent. | `string` | n/a | yes |
 | <a name="input_daos_control_yml"></a> [daos\_control\_yml](#input\_daos\_control\_yml) | YAML configuring DAOS control. | `string` | n/a | yes |
 | <a name="input_gvnic"></a> [gvnic](#input\_gvnic) | Use Google Virtual NIC (gVNIC) network interface | `bool` | `false` | no |
diff --git a/terraform/modules/daos_client/main.tf b/terraform/modules/daos_client/main.tf
index d5c1414..5828057 100644
--- a/terraform/modules/daos_client/main.tf
+++ b/terraform/modules/daos_client/main.tf
@@ -18,9 +18,9 @@ locals {
   os_project         = var.os_project != null ? var.os_project : var.project_id
   subnetwork_project = var.subnetwork_project != null ? var.subnetwork_project : var.project_id
   # Google Virtual NIC (gVNIC) network interface
-  nic_type                     = var.gvnic ? "GVNIC" : "VIRTIO_NET"
-  total_egress_bandwidth_tier  = var.gvnic ? "TIER_1" : "DEFAULT"
-  certs_install_content        = var.certs_install_content
+  nic_type                    = var.gvnic ? "GVNIC" : "VIRTIO_NET"
+  total_egress_bandwidth_tier = var.gvnic ? "TIER_1" : "DEFAULT"
+  certs_install_content       = var.certs_install_content
 
   startup_script = templatefile(
     "${path.module}/templates/daos_startup_script.tftpl",
@@ -100,11 +100,11 @@ resource "google_compute_per_instance_config" "named_instances" {
   name                   = format("%s-%04d", var.instance_base_name, sum([count.index, 1]))
   preserved_state {
     metadata = {
-      inst_type                    = "daos-client"
-      enable-oslogin               = "true"
-      daos_control_yaml_content    = var.daos_control_yml
-      daos_agent_yaml_content      = var.daos_agent_yml
-      startup-script               = local.startup_script
+      inst_type                 = "daos-client"
+      enable-oslogin            = "true"
+      daos_control_yaml_content = var.daos_control_yml
+      daos_agent_yaml_content   = var.daos_agent_yml
+      startup-script            = local.startup_script
       # Adding a reference to the instance template used causes the stateful instance to update
       # if the instance template changes. Otherwise there is no explicit dependency and template
       # changes may not occur on the stateful instance
diff --git a/terraform/modules/daos_client/module.json b/terraform/modules/daos_client/module.json
index 3534b9e..d7e141b 100644
--- a/terraform/modules/daos_client/module.json
+++ b/terraform/modules/daos_client/module.json
@@ -3,16 +3,9 @@
   "footer": "",
   "inputs": [
     {
-      "name": "allow_insecure",
-      "type": "bool",
-      "description": "Sets the allow_insecure setting in the transport_config section of the daos_*.yml files",
-      "default": false,
-      "required": false
-    },
-    {
-      "name": "certs_install_script_content",
+      "name": "certs_install_content",
       "type": "string",
-      "description": "Script to install certificates for DAOS secure transport",
+      "description": "DAOS certificate installation code to be included in the client startup script",
       "default": null,
       "required": true
     },
diff --git a/terraform/modules/daos_server/README.md b/terraform/modules/daos_server/README.md
index 2df10a2..a381935 100644
--- a/terraform/modules/daos_server/README.md
+++ b/terraform/modules/daos_server/README.md
@@ -75,7 +75,7 @@ No modules.
 | <a name="input_os_disk_type"></a> [os\_disk\_type](#input\_os\_disk\_type) | OS disk type ie. pd-ssd, pd-standard | `string` | `"pd-ssd"` | no |
 | <a name="input_os_family"></a> [os\_family](#input\_os\_family) | OS GCP image family | `string` | `"daos-server-centos-7"` | no |
 | <a name="input_os_project"></a> [os\_project](#input\_os\_project) | OS GCP image project name. Defaults to project\_id if null. | `string` | `null` | no |
-| <a name="input_pools"></a> [pools](#input\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    pool_name  = string<br>    pool_size  = string<br>    containers = list(string)<br>    })<br>  )</pre> | `[]` | no |
+| <a name="input_pools"></a> [pools](#input\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = list(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = list(string)<br>      user_attributes = list(string)<br>    }))<br>  }))</pre> | `[]` | no |
 | <a name="input_preemptible"></a> [preemptible](#input\_preemptible) | If preemptible instances | `string` | `false` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The GCP project to use | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The GCP region to create and test resources in | `string` | n/a | yes |
@@ -90,10 +90,10 @@ No modules.
 | Name | Description |
 |------|-------------|
 | <a name="output_access_points"></a> [access\_points](#output\_access\_points) | List of DAOS servers to use as access points |
-| <a name="output_certs_install_script_content"></a> [certs\_install\_script\_content](#output\_certs\_install\_script\_content) | Script to install certificates for DAOS secure transport |
+| <a name="output_certs_install_content"></a> [certs\_install\_content](#output\_certs\_install\_content) | Cert installation content to include in the daos\_client startup script |
 | <a name="output_daos_agent_yml"></a> [daos\_agent\_yml](#output\_daos\_agent\_yml) | YAML to configure the daos agent. This is typically saved in /etc/daos/daos\_agent.yml |
+| <a name="output_daos_client_config_script"></a> [daos\_client\_config\_script](#output\_daos\_client\_config\_script) | Script to configure the DAOS system. This will format the sytem with dmg -l and optionally create the specified pools. |
 | <a name="output_daos_client_install_script"></a> [daos\_client\_install\_script](#output\_daos\_client\_install\_script) | Script to install the DAOS client package. |
-| <a name="output_daos_config_script"></a> [daos\_config\_script](#output\_daos\_config\_script) | Script to configure the DAOS system. This will format the sytem with dmg -l and optionally create the specified pools. |
 | <a name="output_daos_control_yml"></a> [daos\_control\_yml](#output\_daos\_control\_yml) | YAML configuring DAOS control. This is typically saved in /etc/daos/daos\_control.yml |
-| <a name="output_default_service_account_email"></a> [default\_service\_account\_email](#output\_default\_service\_account\_email) | Default service account email |
+| <a name="output_daos_pools"></a> [daos\_pools](#output\_daos\_pools) | Specification of pools and containers to create |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/terraform/modules/daos_server/main.tf b/terraform/modules/daos_server/main.tf
index 1beb1a6..e5a2739 100644
--- a/terraform/modules/daos_server/main.tf
+++ b/terraform/modules/daos_server/main.tf
@@ -99,9 +99,9 @@ locals {
   startup_script = templatefile(
     "${path.module}/templates/startup_script.tftpl",
     {
-      first_server          = local.first_server
-      certs_gen_content     = local.certs_gen_content
-      certs_install_content = local.certs_install_content
+      first_server             = local.first_server
+      certs_gen_content        = local.certs_gen_content
+      certs_install_content    = local.certs_install_content
       pool_cont_create_content = local.pool_cont_create_content
     }
   )
diff --git a/terraform/modules/daos_server/module.json b/terraform/modules/daos_server/module.json
index a60c07e..28a959c 100644
--- a/terraform/modules/daos_server/module.json
+++ b/terraform/modules/daos_server/module.json
@@ -116,7 +116,7 @@
     },
     {
       "name": "pools",
-      "type": "list(object({\n    pool_name  = string\n    pool_size  = string\n    containers = list(string)\n    })\n  )",
+      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = list(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = list(string)\n      user_attributes = list(string)\n    }))\n  }))",
       "description": "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos.",
       "default": [],
       "required": false
@@ -196,28 +196,28 @@
       "description": "List of DAOS servers to use as access points"
     },
     {
-      "name": "certs_install_script_content",
-      "description": "Script to install certificates for DAOS secure transport"
+      "name": "certs_install_content",
+      "description": "Cert installation content to include in the daos_client startup script"
     },
     {
       "name": "daos_agent_yml",
       "description": "YAML to configure the daos agent. This is typically saved in /etc/daos/daos_agent.yml"
     },
     {
-      "name": "daos_client_install_script",
-      "description": "Script to install the DAOS client package."
+      "name": "daos_client_config_script",
+      "description": "Script to configure the DAOS system. This will format the sytem with dmg -l and optionally create the specified pools."
     },
     {
-      "name": "daos_config_script",
-      "description": "Script to configure the DAOS system. This will format the sytem with dmg -l and optionally create the specified pools."
+      "name": "daos_client_install_script",
+      "description": "Script to install the DAOS client package."
     },
     {
       "name": "daos_control_yml",
       "description": "YAML configuring DAOS control. This is typically saved in /etc/daos/daos_control.yml"
     },
     {
-      "name": "default_service_account_email",
-      "description": "Default service account email"
+      "name": "daos_pools",
+      "description": "Specification of pools and containers to create"
     }
   ],
   "providers": [
diff --git a/terraform/modules/daos_server/variables.tf b/terraform/modules/daos_server/variables.tf
index eaffdaf..0a4f581 100644
--- a/terraform/modules/daos_server/variables.tf
+++ b/terraform/modules/daos_server/variables.tf
@@ -170,18 +170,17 @@ variable "pools" {
   description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
   default     = []
   type = list(object({
-    name  = string
-    size  = string
+    name       = string
+    size       = string
     tier_ratio = number
-    acls = list(string)
+    acls       = list(string)
     properties = list(string)
     containers = list(object({
-      name = string
-      type = string
-      acls = list(string)
-      properties = list(string)
+      name            = string
+      type            = string
+      acls            = list(string)
+      properties      = list(string)
       user_attributes = list(string)
-      }))
-    })
-  )
+    }))
+  }))
 }

From d783351c5904e91efdb035ef74edc8a8eeedcb52 Mon Sep 17 00:00:00 2001
From: "Mark A. Olson" <mark.a.olson@intel.com>
Date: Thu, 12 May 2022 17:43:27 -0700
Subject: [PATCH 04/10] Updates to pool structure to allow for toolkit YAML
 mapping

Signed-off-by: Mark A. Olson <mark.a.olson@intel.com>
---
 terraform/examples/daos_cluster/README.md     |  8 ++-
 terraform/examples/daos_cluster/module.json   | 13 ++--
 .../terraform.tfvars.perf.example             | 32 ++++-----
 .../daos_cluster/terraform.tfvars.tco.example | 32 ++++-----
 terraform/examples/daos_cluster/variables.tf  | 11 ++--
 terraform/examples/only_daos_server/README.md |  2 +-
 .../examples/only_daos_server/module.json     |  4 +-
 .../terraform.tfvars.perf.example             | 32 ++++-----
 .../terraform.tfvars.tco.example              | 32 ++++-----
 .../examples/only_daos_server/variables.tf    |  8 +--
 terraform/modules/daos_server/README.md       |  2 +-
 terraform/modules/daos_server/main.tf         |  2 +-
 terraform/modules/daos_server/module.json     |  4 +-
 .../templates/pool_cont_create.inc.sh.tftpl   | 66 +++++++++++++++++++
 .../templates/pool_cont_create.tftpl          | 58 ----------------
 .../templates/startup_script.tftpl            |  2 +
 terraform/modules/daos_server/variables.tf    |  8 +--
 17 files changed, 170 insertions(+), 146 deletions(-)
 create mode 100644 terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
 delete mode 100644 terraform/modules/daos_server/templates/pool_cont_create.tftpl

diff --git a/terraform/examples/daos_cluster/README.md b/terraform/examples/daos_cluster/README.md
index 6d11a71..f69249b 100644
--- a/terraform/examples/daos_cluster/README.md
+++ b/terraform/examples/daos_cluster/README.md
@@ -272,7 +272,7 @@ No resources.
 | <a name="input_server_daos_disk_count"></a> [server\_daos\_disk\_count](#input\_server\_daos\_disk\_count) | Number of local ssd's to use | `number` | `16` | no |
 | <a name="input_server_daos_disk_type"></a> [server\_daos\_disk\_type](#input\_server\_daos\_disk\_type) | Daos disk type to use. For now only suported one is local-ssd | `string` | `"local-ssd"` | no |
 | <a name="input_server_daos_scm_size"></a> [server\_daos\_scm\_size](#input\_server\_daos\_scm\_size) | scm\_size | `number` | `200` | no |
-| <a name="input_server_gvnic"></a> [server\_gvnic](#input\_server\_gvnic) | Use Google Virtual NIC (gVNIC) network interface on DAOS servers | `bool` | `false` | no |
+| <a name="input_server_gvnic"></a> [server\_gvnic](#input\_server\_gvnic) | Use Google Virtual NIC (gVNIC) network interface | `bool` | `false` | no |
 | <a name="input_server_instance_base_name"></a> [server\_instance\_base\_name](#input\_server\_instance\_base\_name) | MIG instance base names to use | `string` | `"daos-server"` | no |
 | <a name="input_server_labels"></a> [server\_labels](#input\_server\_labels) | Set of key/value label pairs to assign to daos-server instances | `any` | `{}` | no |
 | <a name="input_server_machine_type"></a> [server\_machine\_type](#input\_server\_machine\_type) | GCP machine type. ie. e2-medium | `string` | `"n2-custom-36-215040"` | no |
@@ -282,7 +282,7 @@ No resources.
 | <a name="input_server_os_disk_type"></a> [server\_os\_disk\_type](#input\_server\_os\_disk\_type) | OS disk type ie. pd-ssd, pd-standard | `string` | `"pd-ssd"` | no |
 | <a name="input_server_os_family"></a> [server\_os\_family](#input\_server\_os\_family) | OS GCP image family | `string` | `"daos-server-centos-7"` | no |
 | <a name="input_server_os_project"></a> [server\_os\_project](#input\_server\_os\_project) | OS GCP image project name. Defaults to project\_id if null. | `string` | `null` | no |
-| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = list(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = list(string)<br>      user_attributes = list(string)<br>    }))<br>  }))</pre> | `[]` | no |
+| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | List of pools and containers to be created | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = map(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = map(string)<br>      user_attributes = map(any)<br>    }))<br>  }))</pre> | `[]` | no |
 | <a name="input_server_preemptible"></a> [server\_preemptible](#input\_server\_preemptible) | If preemptible instances | `string` | `false` | no |
 | <a name="input_server_service_account"></a> [server\_service\_account](#input\_server\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | <pre>object({<br>    email  = string,<br>    scopes = set(string)<br>  })</pre> | <pre>{<br>  "email": null,<br>  "scopes": [<br>    "https://www.googleapis.com/auth/devstorage.read_only",<br>    "https://www.googleapis.com/auth/logging.write",<br>    "https://www.googleapis.com/auth/monitoring.write",<br>    "https://www.googleapis.com/auth/servicecontrol",<br>    "https://www.googleapis.com/auth/service.management.readonly",<br>    "https://www.googleapis.com/auth/trace.append",<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ]<br>}</pre> | no |
 | <a name="input_server_template_name"></a> [server\_template\_name](#input\_server\_template\_name) | MIG template name | `string` | `"daos-server"` | no |
@@ -292,5 +292,7 @@ No resources.
 
 ## Outputs
 
-No outputs.
+| Name | Description |
+|------|-------------|
+| <a name="output_daos_pools"></a> [daos\_pools](#output\_daos\_pools) | Specification of pools and containers to create |
 <!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
diff --git a/terraform/examples/daos_cluster/module.json b/terraform/examples/daos_cluster/module.json
index 7241284..409d532 100644
--- a/terraform/examples/daos_cluster/module.json
+++ b/terraform/examples/daos_cluster/module.json
@@ -163,7 +163,7 @@
     {
       "name": "server_gvnic",
       "type": "bool",
-      "description": "Use Google Virtual NIC (gVNIC) network interface on DAOS servers",
+      "description": "Use Google Virtual NIC (gVNIC) network interface",
       "default": false,
       "required": false
     },
@@ -232,8 +232,8 @@
     },
     {
       "name": "server_pools",
-      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = list(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = list(string)\n      user_attributes = list(string)\n    }))\n  }))",
-      "description": "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos.",
+      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = map(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = map(string)\n      user_attributes = map(any)\n    }))\n  }))",
+      "description": "List of pools and containers to be created",
       "default": [],
       "required": false
     },
@@ -305,7 +305,12 @@
       "description": null
     }
   ],
-  "outputs": [],
+  "outputs": [
+    {
+      "name": "daos_pools",
+      "description": "Specification of pools and containers to create"
+    }
+  ],
   "providers": [],
   "requirements": [
     {
diff --git a/terraform/examples/daos_cluster/terraform.tfvars.perf.example b/terraform/examples/daos_cluster/terraform.tfvars.perf.example
index 33009c8..b24ca95 100644
--- a/terraform/examples/daos_cluster/terraform.tfvars.perf.example
+++ b/terraform/examples/daos_cluster/terraform.tfvars.perf.example
@@ -25,32 +25,34 @@ server_daos_scm_size       = 45
 # server_os_project          = "<project_id>"
 # server_os_disk_type        = "pd-ssd"
 
-# pools = [
+# server_pools = [
 #   {
-#     name = "test_pool",
-#     size = "1TB",
+#     name       = "pool1",
+#     size       = "1TB",
 #     tier_ratio = 3
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
-#       "A:G:GROUP@:rwtT"
-#     ],
-#     properties = [
-#       "reclaim:disabled"
+#       "A:G:GROUP@:rwtT",
+#       "A::EVERYONE@:r",
 #     ],
+#     properties = {
+#       reclaim = "lazy"
+#     },
 #     containers = [
 #       {
-#         name = "test_container1",
+#         name = "container1",
 #         type = "POSIX",
 #         acls = [
 #           "A::OWNER@:rwdtTaAo",
-#           "A:G:GROUP@:rwtT"
-#         ],
-#         properties = [
-#           "rf:0"
+#           "A:G:GROUP@:rwtT",
+#           "A::EVERYONE@:rwtT"
 #         ],
-#         user_attributes = [
-#           "automount true"
-#         ]
+#         properties = {
+#           rf = 0
+#         },
+#         user_attributes = {
+#           automount = true
+#         }
 #       }
 #     ]
 #   }
diff --git a/terraform/examples/daos_cluster/terraform.tfvars.tco.example b/terraform/examples/daos_cluster/terraform.tfvars.tco.example
index 2141c81..07ca1d1 100644
--- a/terraform/examples/daos_cluster/terraform.tfvars.tco.example
+++ b/terraform/examples/daos_cluster/terraform.tfvars.tco.example
@@ -25,32 +25,34 @@ server_number_of_instances = 4
 # server_os_project          = "<project_id>"
 # server_os_disk_type        = "pd-ssd"
 
-# pools = [
+# server_pools = [
 #   {
-#     name = "test_pool",
-#     size = "1TB",
+#     name       = "pool1",
+#     size       = "1TB",
 #     tier_ratio = 3
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
-#       "A:G:GROUP@:rwtT"
-#     ],
-#     properties = [
-#       "reclaim:disabled"
+#       "A:G:GROUP@:rwtT",
+#       "A::EVERYONE@:r",
 #     ],
+#     properties = {
+#       reclaim = "lazy"
+#     },
 #     containers = [
 #       {
-#         name = "test_container1",
+#         name = "container1",
 #         type = "POSIX",
 #         acls = [
 #           "A::OWNER@:rwdtTaAo",
-#           "A:G:GROUP@:rwtT"
-#         ],
-#         properties = [
-#           "rf:0"
+#           "A:G:GROUP@:rwtT",
+#           "A::EVERYONE@:rwtT"
 #         ],
-#         user_attributes = [
-#           "automount true"
-#         ]
+#         properties = {
+#           rf = 0
+#         },
+#         user_attributes = {
+#           automount = true
+#         }
 #       }
 #     ]
 #   }
diff --git a/terraform/examples/daos_cluster/variables.tf b/terraform/examples/daos_cluster/variables.tf
index c15a130..5a4782d 100644
--- a/terraform/examples/daos_cluster/variables.tf
+++ b/terraform/examples/daos_cluster/variables.tf
@@ -152,7 +152,6 @@ variable "server_preemptible" {
   type        = string
 }
 
-
 variable "server_daos_scm_size" {
   description = "scm_size"
   default     = 200
@@ -166,26 +165,26 @@ variable "server_daos_crt_timeout" {
 }
 
 variable "server_gvnic" {
-  description = "Use Google Virtual NIC (gVNIC) network interface on DAOS servers"
+  description = "Use Google Virtual NIC (gVNIC) network interface"
   default     = false
   type        = bool
 }
 
 variable "server_pools" {
-  description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
+  description = "List of pools and containers to be created"
   default     = []
   type = list(object({
     name       = string
     size       = string
     tier_ratio = number
     acls       = list(string)
-    properties = list(string)
+    properties = map(string)
     containers = list(object({
       name            = string
       type            = string
       acls            = list(string)
-      properties      = list(string)
-      user_attributes = list(string)
+      properties      = map(string)
+      user_attributes = map(any)
     }))
   }))
 }
diff --git a/terraform/examples/only_daos_server/README.md b/terraform/examples/only_daos_server/README.md
index 3f99ba5..18c40ed 100644
--- a/terraform/examples/only_daos_server/README.md
+++ b/terraform/examples/only_daos_server/README.md
@@ -157,7 +157,7 @@ No resources.
 | <a name="input_server_os_disk_type"></a> [server\_os\_disk\_type](#input\_server\_os\_disk\_type) | OS disk type ie. pd-ssd, pd-standard | `string` | `"pd-ssd"` | no |
 | <a name="input_server_os_family"></a> [server\_os\_family](#input\_server\_os\_family) | OS GCP image family | `string` | `"daos-server-centos-7"` | no |
 | <a name="input_server_os_project"></a> [server\_os\_project](#input\_server\_os\_project) | OS GCP image project name. Defaults to project\_id if null. | `string` | `null` | no |
-| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = list(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = list(string)<br>      user_attributes = list(string)<br>    }))<br>  }))</pre> | `[]` | no |
+| <a name="input_server_pools"></a> [server\_pools](#input\_server\_pools) | List of pools and containers to be created | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = map(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = map(string)<br>      user_attributes = map(any)<br>    }))<br>  }))</pre> | `[]` | no |
 | <a name="input_server_preemptible"></a> [server\_preemptible](#input\_server\_preemptible) | If preemptible instances | `string` | `false` | no |
 | <a name="input_server_service_account"></a> [server\_service\_account](#input\_server\_service\_account) | Service account to attach to the instance. See https://www.terraform.io/docs/providers/google/r/compute_instance_template.html#service_account. | <pre>object({<br>    email  = string,<br>    scopes = set(string)<br>  })</pre> | <pre>{<br>  "email": null,<br>  "scopes": [<br>    "https://www.googleapis.com/auth/devstorage.read_only",<br>    "https://www.googleapis.com/auth/logging.write",<br>    "https://www.googleapis.com/auth/monitoring.write",<br>    "https://www.googleapis.com/auth/servicecontrol",<br>    "https://www.googleapis.com/auth/service.management.readonly",<br>    "https://www.googleapis.com/auth/trace.append",<br>    "https://www.googleapis.com/auth/cloud-platform"<br>  ]<br>}</pre> | no |
 | <a name="input_server_template_name"></a> [server\_template\_name](#input\_server\_template\_name) | MIG template name | `string` | `"daos-server"` | no |
diff --git a/terraform/examples/only_daos_server/module.json b/terraform/examples/only_daos_server/module.json
index 77e7147..e06163d 100644
--- a/terraform/examples/only_daos_server/module.json
+++ b/terraform/examples/only_daos_server/module.json
@@ -130,8 +130,8 @@
     },
     {
       "name": "server_pools",
-      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = list(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = list(string)\n      user_attributes = list(string)\n    }))\n  }))",
-      "description": "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos.",
+      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = map(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = map(string)\n      user_attributes = map(any)\n    }))\n  }))",
+      "description": "List of pools and containers to be created",
       "default": [],
       "required": false
     },
diff --git a/terraform/examples/only_daos_server/terraform.tfvars.perf.example b/terraform/examples/only_daos_server/terraform.tfvars.perf.example
index 351d6cd..49acbc1 100644
--- a/terraform/examples/only_daos_server/terraform.tfvars.perf.example
+++ b/terraform/examples/only_daos_server/terraform.tfvars.perf.example
@@ -24,32 +24,34 @@ server_labels              = {
 # server_os_project          = "<project_id>"
 # server_os_disk_type        = "pd-ssd"
 
-# pools = [
+# server_pools = [
 #   {
-#     name = "test_pool",
-#     size = "1TB",
+#     name       = "pool1",
+#     size       = "1TB",
 #     tier_ratio = 3
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
-#       "A:G:GROUP@:rwtT"
-#     ],
-#     properties = [
-#       "reclaim:disabled"
+#       "A:G:GROUP@:rwtT",
+#       "A::EVERYONE@:r",
 #     ],
+#     properties = {
+#       reclaim = "lazy"
+#     },
 #     containers = [
 #       {
-#         name = "test_container1",
+#         name = "container1",
 #         type = "POSIX",
 #         acls = [
 #           "A::OWNER@:rwdtTaAo",
-#           "A:G:GROUP@:rwtT"
-#         ],
-#         properties = [
-#           "rf:0"
+#           "A:G:GROUP@:rwtT",
+#           "A::EVERYONE@:rwtT"
 #         ],
-#         user_attributes = [
-#           "automount true"
-#         ]
+#         properties = {
+#           rf = 0
+#         },
+#         user_attributes = {
+#           automount = true
+#         }
 #       }
 #     ]
 #   }
diff --git a/terraform/examples/only_daos_server/terraform.tfvars.tco.example b/terraform/examples/only_daos_server/terraform.tfvars.tco.example
index 018155d..2d6db34 100644
--- a/terraform/examples/only_daos_server/terraform.tfvars.tco.example
+++ b/terraform/examples/only_daos_server/terraform.tfvars.tco.example
@@ -24,32 +24,34 @@ server_labels              = {
 # server_os_project          = "<project_id>"
 # server_os_disk_type        = "pd-ssd"
 
-# pools = [
+# server_pools = [
 #   {
-#     name = "test_pool",
-#     size = "1TB",
+#     name       = "pool1",
+#     size       = "1TB",
 #     tier_ratio = 3
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
-#       "A:G:GROUP@:rwtT"
-#     ],
-#     properties = [
-#       "reclaim:disabled"
+#       "A:G:GROUP@:rwtT",
+#       "A::EVERYONE@:r",
 #     ],
+#     properties = {
+#       reclaim = "lazy"
+#     },
 #     containers = [
 #       {
-#         name = "test_container1",
+#         name = "container1",
 #         type = "POSIX",
 #         acls = [
 #           "A::OWNER@:rwdtTaAo",
-#           "A:G:GROUP@:rwtT"
-#         ],
-#         properties = [
-#           "rf:0"
+#           "A:G:GROUP@:rwtT",
+#           "A::EVERYONE@:rwtT"
 #         ],
-#         user_attributes = [
-#           "automount true"
-#         ]
+#         properties = {
+#           rf = 0
+#         },
+#         user_attributes = {
+#           automount = true
+#         }
 #       }
 #     ]
 #   }
diff --git a/terraform/examples/only_daos_server/variables.tf b/terraform/examples/only_daos_server/variables.tf
index 46cabf8..85404fa 100644
--- a/terraform/examples/only_daos_server/variables.tf
+++ b/terraform/examples/only_daos_server/variables.tf
@@ -152,20 +152,20 @@ variable "server_preemptible" {
 }
 
 variable "server_pools" {
-  description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
+  description = "List of pools and containers to be created"
   default     = []
   type = list(object({
     name       = string
     size       = string
     tier_ratio = number
     acls       = list(string)
-    properties = list(string)
+    properties = map(string)
     containers = list(object({
       name            = string
       type            = string
       acls            = list(string)
-      properties      = list(string)
-      user_attributes = list(string)
+      properties      = map(string)
+      user_attributes = map(any)
     }))
   }))
 }
diff --git a/terraform/modules/daos_server/README.md b/terraform/modules/daos_server/README.md
index a381935..322e9bb 100644
--- a/terraform/modules/daos_server/README.md
+++ b/terraform/modules/daos_server/README.md
@@ -75,7 +75,7 @@ No modules.
 | <a name="input_os_disk_type"></a> [os\_disk\_type](#input\_os\_disk\_type) | OS disk type ie. pd-ssd, pd-standard | `string` | `"pd-ssd"` | no |
 | <a name="input_os_family"></a> [os\_family](#input\_os\_family) | OS GCP image family | `string` | `"daos-server-centos-7"` | no |
 | <a name="input_os_project"></a> [os\_project](#input\_os\_project) | OS GCP image project name. Defaults to project\_id if null. | `string` | `null` | no |
-| <a name="input_pools"></a> [pools](#input\_pools) | If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos. | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = list(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = list(string)<br>      user_attributes = list(string)<br>    }))<br>  }))</pre> | `[]` | no |
+| <a name="input_pools"></a> [pools](#input\_pools) | List of pools and containers to be created | <pre>list(object({<br>    name       = string<br>    size       = string<br>    tier_ratio = number<br>    acls       = list(string)<br>    properties = map(string)<br>    containers = list(object({<br>      name            = string<br>      type            = string<br>      acls            = list(string)<br>      properties      = map(string)<br>      user_attributes = map(any)<br>    }))<br>  }))</pre> | `[]` | no |
 | <a name="input_preemptible"></a> [preemptible](#input\_preemptible) | If preemptible instances | `string` | `false` | no |
 | <a name="input_project_id"></a> [project\_id](#input\_project\_id) | The GCP project to use | `string` | n/a | yes |
 | <a name="input_region"></a> [region](#input\_region) | The GCP region to create and test resources in | `string` | n/a | yes |
diff --git a/terraform/modules/daos_server/main.tf b/terraform/modules/daos_server/main.tf
index e5a2739..5b81dea 100644
--- a/terraform/modules/daos_server/main.tf
+++ b/terraform/modules/daos_server/main.tf
@@ -89,7 +89,7 @@ locals {
   )
 
   pool_cont_create_content = templatefile(
-    "${path.module}/templates/pool_cont_create.tftpl",
+    "${path.module}/templates/pool_cont_create.inc.sh.tftpl",
     {
       servers = local.servers
       pools   = local.pools
diff --git a/terraform/modules/daos_server/module.json b/terraform/modules/daos_server/module.json
index 28a959c..438a221 100644
--- a/terraform/modules/daos_server/module.json
+++ b/terraform/modules/daos_server/module.json
@@ -116,8 +116,8 @@
     },
     {
       "name": "pools",
-      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = list(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = list(string)\n      user_attributes = list(string)\n    }))\n  }))",
-      "description": "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos.",
+      "type": "list(object({\n    name       = string\n    size       = string\n    tier_ratio = number\n    acls       = list(string)\n    properties = map(string)\n    containers = list(object({\n      name            = string\n      type            = string\n      acls            = list(string)\n      properties      = map(string)\n      user_attributes = map(any)\n    }))\n  }))",
+      "description": "List of pools and containers to be created",
       "default": [],
       "required": false
     },
diff --git a/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl b/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
new file mode 100644
index 0000000..5e67f34
--- /dev/null
+++ b/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
@@ -0,0 +1,66 @@
+%{ if length(pools) != 0 }
+echo "BEGIN: DAOS pool and container creation"
+
+if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
+  # Wait for servers to start
+  until dmg network scan | grep --fixed-strings "${servers}"
+  do
+    sleep 5
+  done
+  echo "All DAOS Servers started"
+
+  # ---------------------------------------------------------------------------
+  # TODO: Request by Keith Mannthey
+  #       Move storage format into a separate storage_format.inc.sh.tftpl file.
+  #       This will allow the storage format to be done either on the server
+  #       or the client separately from the pool creation. In other words it
+  #       would allow storage format on the server and pool creation on the
+  #       client.
+  echo "BEGIN: Storage format"
+  dmg storage format
+  dmg system query -v
+  echo "END: Storage format"
+  # ---------------------------------------------------------------------------
+
+  %{ if length(pools) != 0 }
+    echo "BEGIN: Create pools and containers from definition in 'pool' variable"
+    %{for pool in pools}
+  dmg pool create \
+    --size=${pool.size} \
+    --tier-ratio="${pool.tier_ratio}" \
+    "${pool.name}"
+
+      %{~ for property_key, property_value in pool.properties ~}
+  dmg pool set-prop "${pool.name}" "${property_key}:${property_value}"
+      %{~ endfor ~}
+
+      %{~ for acl in pool.acls ~}
+  dmg pool update-acl --entry "${acl}" "${pool.name}"
+      %{~ endfor ~}
+
+      %{~ for container in pool.containers ~}
+  daos container create \
+    --type ${container.type} \
+      %{~ if length(container.properties) != 0 ~}
+    --properties="%{~ for idx, cprop_key in keys(container.properties)  ~}
+  ${cprop_key}:${container.properties[cprop_key]}%{~ if idx != length(container.properties)-1 ~},%{~ endif ~}%{~ endfor ~}" \
+      %{~ endif ~}
+    "${pool.name}" \
+    --label "${container.name}"
+
+        %{~ for acl in container.acls ~}
+  daos cont update-acl "${pool.name}" "${container.name}" --entry "${acl}"
+        %{~ endfor ~}
+
+        %{~ for attr_key, attr_value in container.user_attributes ~}
+  daos cont set-attr "${pool.name}" "${container.name}" "${attr_key}" "${attr_value}"
+        %{~ endfor ~}
+      %{~ endfor ~}
+    %{~endfor ~}
+    echo "END: Create pools and containers from definition in 'pool' variable"
+  %{~ endif ~}
+
+fi # if first daos server instance
+
+echo "END: DAOS pool and container creation"
+%{~ endif ~}
diff --git a/terraform/modules/daos_server/templates/pool_cont_create.tftpl b/terraform/modules/daos_server/templates/pool_cont_create.tftpl
deleted file mode 100644
index d36e1b7..0000000
--- a/terraform/modules/daos_server/templates/pool_cont_create.tftpl
+++ /dev/null
@@ -1,58 +0,0 @@
-%{ if length(pools) != 0 }
-set -x
-
-echo "BEGIN: DAOS pool and container creation"
-
-if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
-  # Wait for servers to start
-  until dmg network scan | grep --fixed-strings "${servers}"
-  do
-    sleep 5
-  done
-  echo "All DAOS Servers started"
-
-  echo "Formatting storage on servers: ${servers}"
-  dmg storage format
-  dmg system query -v
-  echo "Done formating DAOS server"
-
-  echo "Creating requested pools and containers"
-  %{for pool in pools}
-    dmg pool create \
-      --size=${pool.size} \
-      --tier-ratio=${pool.tier_ratio} \
-      ${pool.name}
-
-    %{for property in pool.properties}
-      dmg pool set-prop ${pool.name} ${property}
-    %{endfor}
-
-    %{for acl in pool.acls}
-      dmg pool update-acl --entry ${acl} ${pool.name}
-    %{endfor}
-
-    %{for container in pool.containers}
-      daos container create \
-        --type ${container.type} \
-        --properties=%{~ for property in container.properties ~}
-                        %{~ if property == element(container.properties, length(container.properties)-1) ~}
-                          ${~ property ~}
-                        %{~ else ~}${~ property ~},%{~ endif ~}
-                      %{~ endfor } \
-        ${pool.name} \
-        --label ${container.name}
-
-      %{for acl in container.acls}
-        daos cont update-acl ${pool.name} ${container.name} --entry ${acl}
-      %{endfor}
-
-      %{for attribute in container.user_attributes}
-        daos cont set-attr ${pool.name} ${container.name} ${attribute}
-      %{endfor}
-    %{endfor}
-  %{endfor}
-  echo "Done creating requested pools and containers"
-fi
-
-echo "END: DAOS pool and container creation"
-%{ endif }
diff --git a/terraform/modules/daos_server/templates/startup_script.tftpl b/terraform/modules/daos_server/templates/startup_script.tftpl
index e3b009e..7ac5790 100644
--- a/terraform/modules/daos_server/templates/startup_script.tftpl
+++ b/terraform/modules/daos_server/templates/startup_script.tftpl
@@ -38,6 +38,8 @@ systemctl start daos_server
 systemctl enable daos_agent
 systemctl start daos_agent
 
+# TODO: Storage format to go here
+# $ { storage_format_content }
 ${ pool_cont_create_content }
 
 echo "END: DAOS Server Startup Script"
diff --git a/terraform/modules/daos_server/variables.tf b/terraform/modules/daos_server/variables.tf
index 0a4f581..7575e58 100644
--- a/terraform/modules/daos_server/variables.tf
+++ b/terraform/modules/daos_server/variables.tf
@@ -167,20 +167,20 @@ variable "allow_insecure" {
 }
 
 variable "pools" {
-  description = "If provided, this module will generate a script to create a list of pools. pool attributes have to be specified in a format acceptable by [dmg](https://docs.daos.io/v2.0/admin/pool_operations/) and daos."
+  description = "List of pools and containers to be created"
   default     = []
   type = list(object({
     name       = string
     size       = string
     tier_ratio = number
     acls       = list(string)
-    properties = list(string)
+    properties = map(string)
     containers = list(object({
       name            = string
       type            = string
       acls            = list(string)
-      properties      = list(string)
-      user_attributes = list(string)
+      properties      = map(string)
+      user_attributes = map(any)
     }))
   }))
 }

From 4b889e9e75b99f3f3d8db4192554c3084bab0c5b Mon Sep 17 00:00:00 2001
From: "Mark A. Olson" <mark.a.olson@intel.com>
Date: Fri, 13 May 2022 00:41:32 -0700
Subject: [PATCH 05/10] Changed map of strings to map of any in pool properties

Signed-off-by: Mark A. Olson <mark.a.olson@intel.com>
---
 terraform/examples/daos_cluster/variables.tf  |  4 +-
 terraform/examples/io500/config/config.sh     | 42 ++-----------------
 terraform/examples/io500/start.sh             |  2 +-
 .../examples/only_daos_server/variables.tf    |  4 +-
 terraform/modules/daos_server/variables.tf    |  4 +-
 5 files changed, 10 insertions(+), 46 deletions(-)

diff --git a/terraform/examples/daos_cluster/variables.tf b/terraform/examples/daos_cluster/variables.tf
index 5a4782d..b7e6168 100644
--- a/terraform/examples/daos_cluster/variables.tf
+++ b/terraform/examples/daos_cluster/variables.tf
@@ -178,12 +178,12 @@ variable "server_pools" {
     size       = string
     tier_ratio = number
     acls       = list(string)
-    properties = map(string)
+    properties = map(any)
     containers = list(object({
       name            = string
       type            = string
       acls            = list(string)
-      properties      = map(string)
+      properties      = map(any)
       user_attributes = map(any)
     }))
   }))
diff --git a/terraform/examples/io500/config/config.sh b/terraform/examples/io500/config/config.sh
index 3a3b98b..b1ef7e4 100644
--- a/terraform/examples/io500/config/config.sh
+++ b/terraform/examples/io500/config/config.sh
@@ -30,49 +30,14 @@ ID=""
 PREEMPTIBLE_INSTANCES="true"
 SSH_USER="daos-user"
 DAOS_ALLOW_INSECURE="false"
-# DAOS_POOLS='[]'
-DAOS_POOLS='[
-  {
-    name = "io500_pool",
-    size = "1TB",
-    tier_ratio = 3
-    acls = [
-      "A::OWNER@:rwdtTaAo",
-      "A:G:GROUP@:rwtT",
-      "A::EVERYONE@:rwdtTaAo"
-    ],
-    properties = [
-      "reclaim:disabled"
-    ],
-    containers = [
-      {
-        name = "io500_cont",
-        type = "POSIX",
-        acls = [
-          "A::OWNER@:rwdtTaAo",
-          "A:G:GROUP@:rwtT",
-          "A::EVERYONE@:rwdtTaAo"
-        ],
-        properties = [
-          "cksum:sha1",
-          "dedup:hash",
-          "rf:0"
-        ],
-        user_attributes = [
-          "automount true",
-          "mount_path /mnt/daos"
-        ]
-      }
-    ]
-  }
-]'
+
 
 # Server(s)
 DAOS_SERVER_INSTANCE_COUNT="1"
-DAOS_SERVER_MACHINE_TYPE=n2-standard-16 # n2-custom-20-131072 n2-custom-40-262144 n2-highmem-32 n2-standard-2
+DAOS_SERVER_MACHINE_TYPE=n2-highmem-32 # n2-custom-20-131072 n2-custom-40-262144 n2-highmem-32 n2-standard-2
 DAOS_SERVER_DISK_COUNT=8
 DAOS_SERVER_CRT_TIMEOUT=300
-DAOS_SERVER_SCM_SIZE=45
+DAOS_SERVER_SCM_SIZE=100
 DAOS_SERVER_GVNIC=false
 
 # Client(s)
@@ -123,7 +88,6 @@ export TF_VAR_server_machine_type="${DAOS_SERVER_MACHINE_TYPE}"
 export TF_VAR_server_os_project="${TF_VAR_project_id}"
 export TF_VAR_server_os_family="daos-server-io500-centos-7"
 export TF_VAR_server_gvnic="${DAOS_SERVER_GVNIC}"
-export TF_VAR_server_pools="${DAOS_POOLS}"
 # Clients
 export TF_VAR_client_preemptible=${PREEMPTIBLE_INSTANCES}
 export TF_VAR_client_number_of_instances=${DAOS_CLIENT_INSTANCE_COUNT}
diff --git a/terraform/examples/io500/start.sh b/terraform/examples/io500/start.sh
index 5f03d99..e4e4271 100755
--- a/terraform/examples/io500/start.sh
+++ b/terraform/examples/io500/start.sh
@@ -522,7 +522,7 @@ To run the IO500 benchmark:
    ./login
 
 2. Run IO500
-   ~/run_io500-sc21.sh
+   ./run_io500-sc21.sh
 
 EOF
 }
diff --git a/terraform/examples/only_daos_server/variables.tf b/terraform/examples/only_daos_server/variables.tf
index 85404fa..d05eaf1 100644
--- a/terraform/examples/only_daos_server/variables.tf
+++ b/terraform/examples/only_daos_server/variables.tf
@@ -159,12 +159,12 @@ variable "server_pools" {
     size       = string
     tier_ratio = number
     acls       = list(string)
-    properties = map(string)
+    properties = map(any)
     containers = list(object({
       name            = string
       type            = string
       acls            = list(string)
-      properties      = map(string)
+      properties      = map(any)
       user_attributes = map(any)
     }))
   }))
diff --git a/terraform/modules/daos_server/variables.tf b/terraform/modules/daos_server/variables.tf
index 7575e58..12f5f24 100644
--- a/terraform/modules/daos_server/variables.tf
+++ b/terraform/modules/daos_server/variables.tf
@@ -174,12 +174,12 @@ variable "pools" {
     size       = string
     tier_ratio = number
     acls       = list(string)
-    properties = map(string)
+    properties = map(any)
     containers = list(object({
       name            = string
       type            = string
       acls            = list(string)
-      properties      = map(string)
+      properties      = map(any)
       user_attributes = map(any)
     }))
   }))

From 03895cbfa55fdece50983f392eff4b664ace351b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Fri, 13 May 2022 12:32:23 +0200
Subject: [PATCH 06/10] Move storage format functionality to separate file
 include
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 terraform/examples/io500/config/config.sh     |  1 -
 terraform/modules/daos_server/main.tf         |  8 +++
 .../templates/pool_cont_create.inc.sh.tftpl   | 52 +++++--------------
 .../templates/startup_script.tftpl            |  3 +-
 .../templates/storage_format.inc.sh.tftpl     | 13 +++++
 5 files changed, 36 insertions(+), 41 deletions(-)
 create mode 100644 terraform/modules/daos_server/templates/storage_format.inc.sh.tftpl

diff --git a/terraform/examples/io500/config/config.sh b/terraform/examples/io500/config/config.sh
index b1ef7e4..ac826ef 100644
--- a/terraform/examples/io500/config/config.sh
+++ b/terraform/examples/io500/config/config.sh
@@ -31,7 +31,6 @@ PREEMPTIBLE_INSTANCES="true"
 SSH_USER="daos-user"
 DAOS_ALLOW_INSECURE="false"
 
-
 # Server(s)
 DAOS_SERVER_INSTANCE_COUNT="1"
 DAOS_SERVER_MACHINE_TYPE=n2-highmem-32 # n2-custom-20-131072 n2-custom-40-262144 n2-highmem-32 n2-standard-2
diff --git a/terraform/modules/daos_server/main.tf b/terraform/modules/daos_server/main.tf
index 5b81dea..a5c8210 100644
--- a/terraform/modules/daos_server/main.tf
+++ b/terraform/modules/daos_server/main.tf
@@ -88,6 +88,13 @@ locals {
     }
   )
 
+  storage_format_content = templatefile(
+    "${path.module}/templates/storage_format.inc.sh.tftpl",
+    {
+      servers = local.servers
+    }
+  )
+
   pool_cont_create_content = templatefile(
     "${path.module}/templates/pool_cont_create.inc.sh.tftpl",
     {
@@ -102,6 +109,7 @@ locals {
       first_server             = local.first_server
       certs_gen_content        = local.certs_gen_content
       certs_install_content    = local.certs_install_content
+      storage_format_content   = local.storage_format_content
       pool_cont_create_content = local.pool_cont_create_content
     }
   )
diff --git a/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl b/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
index 5e67f34..c19b919 100644
--- a/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
+++ b/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
@@ -1,44 +1,22 @@
 %{ if length(pools) != 0 }
-echo "BEGIN: DAOS pool and container creation"
+echo "BEGIN: DAOS pool and container creation from definition in 'pool' variable"
 
 if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
-  # Wait for servers to start
-  until dmg network scan | grep --fixed-strings "${servers}"
-  do
-    sleep 5
-  done
-  echo "All DAOS Servers started"
-
-  # ---------------------------------------------------------------------------
-  # TODO: Request by Keith Mannthey
-  #       Move storage format into a separate storage_format.inc.sh.tftpl file.
-  #       This will allow the storage format to be done either on the server
-  #       or the client separately from the pool creation. In other words it
-  #       would allow storage format on the server and pool creation on the
-  #       client.
-  echo "BEGIN: Storage format"
-  dmg storage format
-  dmg system query -v
-  echo "END: Storage format"
-  # ---------------------------------------------------------------------------
-
-  %{ if length(pools) != 0 }
-    echo "BEGIN: Create pools and containers from definition in 'pool' variable"
-    %{for pool in pools}
+  %{for pool in pools}
   dmg pool create \
     --size=${pool.size} \
     --tier-ratio="${pool.tier_ratio}" \
     "${pool.name}"
 
-      %{~ for property_key, property_value in pool.properties ~}
+    %{~ for property_key, property_value in pool.properties ~}
   dmg pool set-prop "${pool.name}" "${property_key}:${property_value}"
-      %{~ endfor ~}
+    %{~ endfor ~}
 
-      %{~ for acl in pool.acls ~}
+    %{~ for acl in pool.acls ~}
   dmg pool update-acl --entry "${acl}" "${pool.name}"
-      %{~ endfor ~}
+    %{~ endfor ~}
 
-      %{~ for container in pool.containers ~}
+    %{~ for container in pool.containers ~}
   daos container create \
     --type ${container.type} \
       %{~ if length(container.properties) != 0 ~}
@@ -48,19 +26,17 @@ if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
     "${pool.name}" \
     --label "${container.name}"
 
-        %{~ for acl in container.acls ~}
+      %{~ for acl in container.acls ~}
   daos cont update-acl "${pool.name}" "${container.name}" --entry "${acl}"
-        %{~ endfor ~}
+      %{~ endfor ~}
 
-        %{~ for attr_key, attr_value in container.user_attributes ~}
+      %{~ for attr_key, attr_value in container.user_attributes ~}
   daos cont set-attr "${pool.name}" "${container.name}" "${attr_key}" "${attr_value}"
-        %{~ endfor ~}
       %{~ endfor ~}
-    %{~endfor ~}
-    echo "END: Create pools and containers from definition in 'pool' variable"
-  %{~ endif ~}
+    %{~ endfor ~}
+  %{~endfor ~}
 
-fi # if first daos server instance
+fi
 
-echo "END: DAOS pool and container creation"
+echo "END: DAOS pool and container creation from definition in 'pool' variable"
 %{~ endif ~}
diff --git a/terraform/modules/daos_server/templates/startup_script.tftpl b/terraform/modules/daos_server/templates/startup_script.tftpl
index 7ac5790..4694506 100644
--- a/terraform/modules/daos_server/templates/startup_script.tftpl
+++ b/terraform/modules/daos_server/templates/startup_script.tftpl
@@ -38,8 +38,7 @@ systemctl start daos_server
 systemctl enable daos_agent
 systemctl start daos_agent
 
-# TODO: Storage format to go here
-# $ { storage_format_content }
+${ storage_format_content }
 ${ pool_cont_create_content }
 
 echo "END: DAOS Server Startup Script"
diff --git a/terraform/modules/daos_server/templates/storage_format.inc.sh.tftpl b/terraform/modules/daos_server/templates/storage_format.inc.sh.tftpl
new file mode 100644
index 0000000..02c00ee
--- /dev/null
+++ b/terraform/modules/daos_server/templates/storage_format.inc.sh.tftpl
@@ -0,0 +1,13 @@
+if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
+  # Wait for servers to start
+  until dmg network scan | grep --fixed-strings "${servers}"
+  do
+    sleep 5
+  done
+  echo "All DAOS Servers started"
+
+  echo "BEGIN: Storage format"
+  dmg storage format
+  dmg system query -v
+  echo "END: Storage format"
+fi

From 854672fef737bcf8b09bd1836fb40a3570593c4a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Fri, 13 May 2022 15:28:23 +0200
Subject: [PATCH 07/10] Change tier-ratio value to default 6
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 terraform/examples/daos_cluster/terraform.tfvars.perf.example   | 2 +-
 terraform/examples/daos_cluster/terraform.tfvars.tco.example    | 2 +-
 .../examples/only_daos_server/terraform.tfvars.tco.example      | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/terraform/examples/daos_cluster/terraform.tfvars.perf.example b/terraform/examples/daos_cluster/terraform.tfvars.perf.example
index b24ca95..1d1884d 100644
--- a/terraform/examples/daos_cluster/terraform.tfvars.perf.example
+++ b/terraform/examples/daos_cluster/terraform.tfvars.perf.example
@@ -29,7 +29,7 @@ server_daos_scm_size       = 45
 #   {
 #     name       = "pool1",
 #     size       = "1TB",
-#     tier_ratio = 3
+#     tier_ratio = 6
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
 #       "A:G:GROUP@:rwtT",
diff --git a/terraform/examples/daos_cluster/terraform.tfvars.tco.example b/terraform/examples/daos_cluster/terraform.tfvars.tco.example
index 07ca1d1..35fc6b4 100644
--- a/terraform/examples/daos_cluster/terraform.tfvars.tco.example
+++ b/terraform/examples/daos_cluster/terraform.tfvars.tco.example
@@ -29,7 +29,7 @@ server_number_of_instances = 4
 #   {
 #     name       = "pool1",
 #     size       = "1TB",
-#     tier_ratio = 3
+#     tier_ratio = 6
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
 #       "A:G:GROUP@:rwtT",
diff --git a/terraform/examples/only_daos_server/terraform.tfvars.tco.example b/terraform/examples/only_daos_server/terraform.tfvars.tco.example
index 2d6db34..23f9907 100644
--- a/terraform/examples/only_daos_server/terraform.tfvars.tco.example
+++ b/terraform/examples/only_daos_server/terraform.tfvars.tco.example
@@ -28,7 +28,7 @@ server_labels              = {
 #   {
 #     name       = "pool1",
 #     size       = "1TB",
-#     tier_ratio = 3
+#     tier_ratio = 6
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
 #       "A:G:GROUP@:rwtT",

From 7684a6bec07ea391ac47e55670ce669d3b7ecb66 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Fri, 13 May 2022 15:32:36 +0200
Subject: [PATCH 08/10] fixup! Change tier-ratio value to default 6
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 .../examples/only_daos_server/terraform.tfvars.perf.example     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/terraform/examples/only_daos_server/terraform.tfvars.perf.example b/terraform/examples/only_daos_server/terraform.tfvars.perf.example
index 49acbc1..340cedb 100644
--- a/terraform/examples/only_daos_server/terraform.tfvars.perf.example
+++ b/terraform/examples/only_daos_server/terraform.tfvars.perf.example
@@ -28,7 +28,7 @@ server_labels              = {
 #   {
 #     name       = "pool1",
 #     size       = "1TB",
-#     tier_ratio = 3
+#     tier_ratio = 6
 #     acls = [
 #       "A::OWNER@:rwdtTaAo",
 #       "A:G:GROUP@:rwtT",

From 3a860c811c380d8b531ca40a364f76e4a5587c44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C5=81ukasz=20Sitkiewicz?= <lukasz.sitkiewicz@intel.com>
Date: Fri, 13 May 2022 16:19:37 +0200
Subject: [PATCH 09/10] Apply pool properties at pool creation time
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Signed-off-by: Łukasz Sitkiewicz <lukasz.sitkiewicz@intel.com>
---
 .../templates/pool_cont_create.inc.sh.tftpl          | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl b/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
index c19b919..c8f24ee 100644
--- a/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
+++ b/terraform/modules/daos_server/templates/pool_cont_create.inc.sh.tftpl
@@ -6,12 +6,12 @@ if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
   dmg pool create \
     --size=${pool.size} \
     --tier-ratio="${pool.tier_ratio}" \
+    %{~ if length(pool.properties) != 0 ~}
+    --properties="%{~ for idx, pprop_key in keys(pool.properties) ~}
+        ${~ pprop_key}:${pool.properties[pprop_key]}%{~ if idx != length(pool.properties)-1 ~},%{~ endif ~}%{~ endfor ~}" \
+      %{~ endif ~}
     "${pool.name}"
 
-    %{~ for property_key, property_value in pool.properties ~}
-  dmg pool set-prop "${pool.name}" "${property_key}:${property_value}"
-    %{~ endfor ~}
-
     %{~ for acl in pool.acls ~}
   dmg pool update-acl --entry "${acl}" "${pool.name}"
     %{~ endfor ~}
@@ -20,8 +20,8 @@ if [[ "$${HOSTNAME,,}" == "$${FIRST_DAOS_SERVER_HOSTNAME,,}" ]]; then
   daos container create \
     --type ${container.type} \
       %{~ if length(container.properties) != 0 ~}
-    --properties="%{~ for idx, cprop_key in keys(container.properties)  ~}
-  ${cprop_key}:${container.properties[cprop_key]}%{~ if idx != length(container.properties)-1 ~},%{~ endif ~}%{~ endfor ~}" \
+    --properties="%{~ for idx, cprop_key in keys(container.properties) ~}
+        ${~ cprop_key}:${container.properties[cprop_key]}%{~ if idx != length(container.properties)-1 ~},%{~ endif ~}%{~ endfor ~}" \
       %{~ endif ~}
     "${pool.name}" \
     --label "${container.name}"

From a7b28e9e25412dea01b763db697f84c297b02f41 Mon Sep 17 00:00:00 2001
From: "Mark A. Olson" <mark.a.olson@intel.com>
Date: Fri, 13 May 2022 08:29:04 -0700
Subject: [PATCH 10/10] Add numa config to daos_agent.yml

Signed-off-by: Mark A. Olson <mark.a.olson@intel.com>
---
 .../modules/daos_server/templates/daos_agent.yml.tftpl      | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/terraform/modules/daos_server/templates/daos_agent.yml.tftpl b/terraform/modules/daos_server/templates/daos_agent.yml.tftpl
index fd4cf5f..d3d8ef9 100644
--- a/terraform/modules/daos_server/templates/daos_agent.yml.tftpl
+++ b/terraform/modules/daos_server/templates/daos_agent.yml.tftpl
@@ -13,3 +13,9 @@ transport_config:
   cert: /etc/daos/certs/agent.crt
   key: /etc/daos/certs/agent.key
   %{ endif }
+
+fabric_ifaces:
+- numa_node: 0
+  devices:
+  - iface: eth0
+    domain: eth0