diff --git a/layered/cloud-run/main.py b/layered/cloud-run/main.py index b5f9c021..183fb90f 100644 --- a/layered/cloud-run/main.py +++ b/layered/cloud-run/main.py @@ -15,6 +15,7 @@ app = Flask(__name__) project = os.environ.get("PROJECT_NAME") or "flint-cloud" +gcs_bucket_name = os.environ.get("gcs_bucket_name") or "simulation_data_flint-cloud" with open("service_account.json", "w") as saf: saf.write(os.environ.get("SERVICE_ACCOUNT") or "ERROR") os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "service_account.json" @@ -40,7 +41,7 @@ def publish_message(topic, attribs, project=project): def upload_blob(title, source_file_name): """Uploads a file to the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to upload # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -58,7 +59,7 @@ def upload_blob(title, source_file_name): def delete_blob(title, source_file_name): """Deletes a file to from the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to upload # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -76,7 +77,7 @@ def delete_blob(title, source_file_name): def download_blob(title, source_blob_name): """Downloads a file from the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to download # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -100,7 +101,7 @@ def generate_download_signed_url_v4(project_dir): Engine or from the Google Cloud SDK. """ - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name blob_path = "simulations/simulation-" + project_dir + "/" storage_client = storage.Client() @@ -144,7 +145,7 @@ def create_log(title): def check_log(title): """Check for log""" storage_client = storage.Client() - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name bucket = storage_client.bucket(bucket_name) blob_path = f"simulations/simulation-{title}/log.txt" return storage.Blob(bucket=bucket, name=blob_path).exists(storage_client) diff --git a/layered/compute-engine/main.py b/layered/compute-engine/main.py index e6003d56..a421bd0b 100644 --- a/layered/compute-engine/main.py +++ b/layered/compute-engine/main.py @@ -12,6 +12,7 @@ from googleapiclient import discovery project = os.environ.get("PROJECT_NAME") or "flint-cloud" +gcs_bucket_name = os.environ.get("gcs_bucket_name") or "simulation_data_flint-cloud" zone = os.environ.get("GCE_ZONE") or "us-central1-a" instance = os.environ.get("GCE_NAME") or "instance-1" with open("service_account.json", "w") as saf: @@ -39,7 +40,7 @@ def publish_message(topic, attribs, project="flint-cloud"): def delete_blob(title, source_file_name): """Deletes a file to from the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to upload # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -57,7 +58,7 @@ def delete_blob(title, source_file_name): def upload_blob(title, source_file_name): """Uploads a file to the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to upload # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -75,7 +76,7 @@ def upload_blob(title, source_file_name): def download_blob(title, source_blob_name): """Downloads a file from the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to download # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -99,7 +100,7 @@ def generate_download_signed_url_v4(project_dir): Engine or from the Google Cloud SDK. """ - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name blob_path = "simulations/simulation-" + project_dir + "/" storage_client = storage.Client() @@ -143,7 +144,7 @@ def create_log(title): def check_log(title): """Check for log""" storage_client = storage.Client() - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name bucket = storage_client.bucket(bucket_name) blob_path = f"simulations/simulation-{title}/log.txt" return storage.Blob(bucket=bucket, name=blob_path).exists(storage_client) diff --git a/layered/ingress/main.py b/layered/ingress/main.py index 0ec015ed..7c95a477 100644 --- a/layered/ingress/main.py +++ b/layered/ingress/main.py @@ -13,6 +13,7 @@ app = Flask(__name__) project = os.environ.get("PROJECT_NAME") or "flint-cloud" +gcs_bucket_name = os.environ.get("gcs_bucket_name") or "simulation_data_flint-cloud" zone = os.environ.get("GCE_ZONE") or "us-central1-a" instance = os.environ.get("GCE_NAME") or "instance-1" with open("service_account.json", "w") as saf: @@ -55,7 +56,7 @@ def publish_message(topic, data, project=project): def upload_blob(title, source_file_name): """Uploads a file to the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to upload # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -73,7 +74,7 @@ def upload_blob(title, source_file_name): def download_blob(title, source_blob_name): """Downloads a file from the bucket.""" # The ID of your GCS bucket - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name # The path to your file to download # source_file_name = "local/path/to/file" # The ID of your GCS object @@ -124,7 +125,7 @@ def gcbm_new(): title = "".join(c for c in title if c.isalnum()) name = "simulations/simulation-" + title + "/input.zip" storage_client = storage.Client() - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name bucket = storage_client.bucket(bucket_name) stats = storage.Blob(bucket=bucket, name=name).exists(storage_client) @@ -258,7 +259,7 @@ def status(): # Verify output exists storage_client = storage.Client() - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name bucket = storage_client.bucket(bucket_name) blob_path = f"simulations/simulation-{title}/output.zip" stats = storage.Blob(bucket=bucket, name=blob_path).exists(storage_client) @@ -305,7 +306,7 @@ def gcbm_dynamic(): # Verify simulation exists storage_client = storage.Client() - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name bucket = storage_client.bucket(bucket_name) blob_path = f"simulations/simulation-{title}/input.zip" stats = storage.Blob(bucket=bucket, name=blob_path).exists(storage_client) @@ -343,7 +344,7 @@ def generate_download_signed_url_v4(project_dir): Engine or from the Google Cloud SDK. """ - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name blob_path = "simulations/simulation-" + project_dir + "/" storage_client = storage.Client() @@ -416,7 +417,7 @@ def gcbm_list_simulations(): description: GCBM Simulations List """ storage_client = storage.Client() - bucket_name = "simulation_data_flint-cloud" + bucket_name = gcs_bucket_name blobs = storage_client.list_blobs(bucket_name, prefix="simulations/") blob_map = {} blob_list = [] diff --git a/layered/main.tf b/layered/main.tf index f02bb73b..3d24abe4 100644 --- a/layered/main.tf +++ b/layered/main.tf @@ -4,6 +4,11 @@ variable "project" { default = "flint-cloud" } +variable "gcs_bucket_name" { + type = string + default = "simulation_data_flint-cloud" +} + variable "region" { type = string default = "us-central1" @@ -133,7 +138,7 @@ resource "google_pubsub_topic" "large-simulations" { resource "google_cloud_run_service" "fc-ingress" { name = var.ingress_name location = var.region - + template { spec { containers { @@ -190,7 +195,7 @@ resource "google_cloud_run_service_iam_policy" "noauth-ingress" { resource "google_cloud_run_service" "fc-cr-processor" { name = var.cr_processor_name location = var.region - + template { spec { containers { @@ -255,9 +260,20 @@ resource "google_pubsub_subscription" "small-simulations-sub" { depends_on = [google_pubsub_topic.small-simulations, google_cloud_run_service.fc-cr-processor] } +# The following example shows how to generate a unique name for an GCS bucket + +resource "random_id" "server" { + keepers = { + gcs_bucket_id = "${var.gcs_bucket_id}" + } + + byte_length = 8 +} + # GCS bucket resource "google_storage_bucket" "simulation_data_flint-cloud" { - name = "simulation_data_flint-cloud" + gcs_bucket_name = "simulation_data_flint-cloud_${random_id.server.hex}" + name = gcs_bucket_name location = var.region force_destroy = true