Tutorial for FSI VaR MonteCarlo

community/examples/fsi-montecarlo-on-batch.yaml

@@ -0,0 +1,116 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+blueprint_name: fsi-montecarlo-on-batch
+
+vars:
+  project_id:  ## Set GCP Project ID Here ##
+  deployment_name: fsimontecarlo
+  region: us-central1
+  zone: us-central1-a
+deployment_groups:
+- group: setup
+  modules:
+
+  - id: enable_apis
+    source: community/modules/project/service-enablement
+    settings:
+      gcp_service_list: [
+        "bigquery.googleapis.com",
+        "cloudresourcemanager.googleapis.com",
+        "container.googleapis.com",
+        "logging.googleapis.com",
+        "notebooks.googleapis.com",
+        "batch.googleapis.com",
+        "pubsub.googleapis.com",
+        "compute.googleapis.com"
+      ]
+- group: primary
+  modules:
+
+  - id: fsi_bucket
+    source: community/modules/file-system/cloud-storage-bucket
+    settings:
+      name_prefix: fsi_bucket
+      random_suffix: true
+      force_destroy: true
+      local_mount: /home/jupyter/fsi
+
+  - id: pubsub_topic
+    source: community/modules/pubsub/topic
+
+  - id: bq-dataset
+    source: community/modules/database/bigquery_dataset
+    settings:
+
+  - id: bq-table
+    source: community/modules/database/bigquery_table
+    use: [bq-dataset]
+    settings:
+      table_schema:
+        '
+        [
+          {
+            "name": "subscription_name", "type": "STRING"
+          },
+          {
+            "name": "message_id", "type": "STRING"
+          },
+          {
+            "name": "publish_time", "type": "TIMESTAMP"
+          },
+          {
+            "name": "simulation_results", "type": "RECORD", "mode": "REPEATED",
+            "fields": [
+              {
+                "name" : "price",
+                "type" : "NUMERIC"
+              }
+            ]
+          },
+          {
+            "name": "ticker", "type": "STRING"
+          }
+          ,{
+            "name": "epoch_time", "type": "INT64"
+          }
+          ,{
+            "name": "iteration", "type": "INT64"
+          }
+          ,{
+            "name": "start_date", "type": "STRING"
+          }
+          ,{
+            "name": "end_date", "type": "STRING"
+          }
+          ,{
+            "name": "attributes", "type": "STRING"
+          }
+        ]
+        '
+
+  - id: fsi_notebook
+    source: community/modules/compute/notebook
+    use: [fsi_bucket]
+    settings:
+      machine_type: n1-standard-4
+
+  - id: fsi_tutorial_files
+    source: community/modules/files/fsi-montecarlo-on-batch
+    use: [bq-dataset, bq-table, fsi_bucket, pubsub_topic]
+
+  - id: bq_subscription
+    source: community/modules/pubsub/bigquery-sub
+    use: [bq-table, pubsub_topic]

community/modules/compute/notebook/README.md

@@ -0,0 +1,89 @@
+# Description
+
+This module creates the Vertex AI Notebook, to be used in tutorials.
+
+Primarily used for FSI - MonteCarlo Tutorial: **[fsi-montecarlo-on-batch-tutorial]**.
+
+[fsi-montecarlo-on-batch-tutorial]: ../docs/tutorials/fsi-montecarlo-on-batch/README.md
+
+## Usage
+
+This is a simple usage:
+
+```yaml
+  - id: bucket
+    source: community/modules/file-system/cloud-storage-bucket
+    settings: 
+      name_prefix: my-bucket
+      local_mount: /home/jupyter/my-bucket
+
+  - id: notebook
+    source: community/modules/compute/notebook
+    use: [bucket]
+    settings:
+      name_prefix: notebook
+      machine_type: n1-standard-4
+
+```
+
+## License
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+Copyright 2023 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.0.0 |
+| <a name="requirement_google"></a> [google](#requirement\_google) | >= 4.42 |
+| <a name="requirement_random"></a> [random](#requirement\_random) | ~> 3.0 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_google"></a> [google](#provider\_google) | >= 4.42 |
+| <a name="provider_random"></a> [random](#provider\_random) | ~> 3.0 |
+
+## Modules
+
+No modules.
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [google_notebooks_instance.instance](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/notebooks_instance) | resource |
+| [google_storage_bucket_object.mount_script](https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/storage_bucket_object) | resource |
+| [random_id.resource_name_suffix](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/id) | resource |
+
+## Inputs
+
+| Name | Description | Type | Default | Required |
+|------|-------------|------|---------|:--------:|
+| <a name="input_deployment_name"></a> [deployment\_name](#input\_deployment\_name) | Name of the HPC deployment; used as part of name of the notebook. | `string` | n/a | yes |
+| <a name="input_gcs_bucket_path"></a> [gcs\_bucket\_path](#input\_gcs\_bucket\_path) | Bucket name, can be provided from the google-cloud-storage module | `string` | `null` | no |
+| <a name="input_instance_image"></a> [instance\_image](#input\_instance\_image) | Instance Image | `map(string)` | <pre>{<br>  "family": "tf-latest-cpu",<br>  "name": null,<br>  "project": "deeplearning-platform-release"<br>}</pre> | no |
+| <a name="input_labels"></a> [labels](#input\_labels) | Labels to add to the resource Key-value pairs. | `map(string)` | n/a | yes |
+| <a name="input_machine_type"></a> [machine\_type](#input\_machine\_type) | The machine type to employ | `string` | n/a | yes |
+| <a name="input_mount_runner"></a> [mount\_runner](#input\_mount\_runner) | mount content from the google-cloud-storage module | `any` | n/a | yes |
+| <a name="input_project_id"></a> [project\_id](#input\_project\_id) | ID of project in which the notebook will be created. | `string` | n/a | yes |
+| <a name="input_zone"></a> [zone](#input\_zone) | The zone to deploy to | `string` | n/a | yes |
+
+## Outputs
+
+No outputs.
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

community/modules/compute/notebook/main.tf

@@ -0,0 +1,68 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+locals {
+  # This label allows for billing report tracking based on module.
+  labels = merge(var.labels, { ghpc_module = "notebook", ghpc_role = "compute" })
+}
+
+locals {
+  suffix               = random_id.resource_name_suffix.hex
+  name                 = "${var.deployment_name}-notebook-${local.suffix}"
+  bucket               = replace(var.gcs_bucket_path, "gs://", "")
+  post_script_filename = "mount-${local.suffix}.sh"
+
+  # mount_runner_args is defined here: https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/3abddcfbd245b0e6747917a4e55b30658414ffd7/community/modules/file-system/cloud-storage-bucket/outputs.tf#L40
+  mount_args = split(" ", var.mount_runner.args)
+
+  unused       = local.mount_args[0]
+  remote_mount = local.mount_args[1]
+  local_mount  = local.mount_args[2]
+  fs_type      = local.mount_args[3]
+  # These options provide a "rw" mount of the GCS bucket
+  mount_options = "defaults,_netdev,allow_other,implicit_dirs,gid=1000,uid=1000"
+
+  content0 = var.mount_runner.content
+  content1 = replace(local.content0, "$1", local.unused)
+  content2 = replace(local.content1, "$2", local.remote_mount)
+  content3 = replace(local.content2, "$3", local.local_mount)
+  content4 = replace(local.content3, "$4", local.fs_type)
+  content5 = replace(local.content4, "$5", local.mount_options)
+
+}
+
+resource "random_id" "resource_name_suffix" {
+  byte_length = 4
+}
+
+resource "google_notebooks_instance" "instance" {
+  name                = local.name
+  location            = var.zone
+  machine_type        = var.machine_type
+  project             = var.project_id
+  post_startup_script = "${var.gcs_bucket_path}/${local.post_script_filename}"
+  labels              = local.labels
+  vm_image {
+    project      = var.instance_image.project
+    image_family = var.instance_image.family
+  }
+}
+
+resource "google_storage_bucket_object" "mount_script" {
+  name    = local.post_script_filename
+  content = local.content5
+  bucket  = local.bucket
+}

community/modules/compute/notebook/metadata.yaml

@@ -0,0 +1,20 @@
+# Copyright 2023 "Google LLC"
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+
+spec:
+  requirements:
+    services:
+    - notebooks.googleapis.com
+    - storage.googleapis.com

community/modules/compute/notebook/variables.tf

@@ -0,0 +1,71 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+variable "project_id" {
+  description = "ID of project in which the notebook will be created."
+  type        = string
+}
+
+variable "deployment_name" {
+  description = "Name of the HPC deployment; used as part of name of the notebook."
+  type        = string
+}
+
+variable "zone" {
+  description = "The zone to deploy to"
+  type        = string
+}
+
+variable "machine_type" {
+  description = "The machine type to employ"
+  type        = string
+}
+
+variable "labels" {
+  description = "Labels to add to the resource Key-value pairs."
+  type        = map(string)
+}
+
+variable "instance_image" {
+  description = "Instance Image"
+  type        = map(string)
+  default = {
+    project = "deeplearning-platform-release"
+    family  = "tf-latest-cpu"
+    name    = null
+  }
+
+  validation {
+    condition     = can(coalesce(var.instance_image.project))
+    error_message = "In var.instance_image, the \"project\" field must be a string set to the Cloud project ID."
+  }
+
+  validation {
+    condition     = can(coalesce(var.instance_image.name)) != can(coalesce(var.instance_image.family))
+    error_message = "In var.instance_image, exactly one of \"family\" or \"name\" fields must be set to desired image family or name."
+  }
+}
+
+variable "gcs_bucket_path" {
+  description = "Bucket name, can be provided from the google-cloud-storage module"
+  type        = string
+  default     = null
+}
+
+variable "mount_runner" {
+  description = "mount content from the google-cloud-storage module"
+  type        = any
+}

community/modules/compute/notebook/versions.tf

@@ -0,0 +1,29 @@
+/**
+ * Copyright 2023 Google LLC
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+terraform {
+  required_version = ">= 1.0.0"
+  required_providers {
+    google = {
+      source  = "hashicorp/google"
+      version = ">= 4.42"
+    }
+    random = {
+      source  = "hashicorp/random"
+      version = "~> 3.0"
+    }
+  }
+}