Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Generate globally unique Google Cloud Storage bucket name #48

Closed
wants to merge 1 commit into from
+39 −20
Closed

feat: Generate globally unique Google Cloud Storage bucket name #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions layered/cloud-run/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
app = Flask(__name__)

project = os.environ.get("PROJECT_NAME") or "flint-cloud"
gcs_bucket_name = os.environ.get("gcs_bucket_name") or "simulation_data_flint-cloud"
with open("service_account.json", "w") as saf:
saf.write(os.environ.get("SERVICE_ACCOUNT") or "ERROR")
os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = "service_account.json"
Expand All @@ -40,7 +41,7 @@ def publish_message(topic, attribs, project=project):
def upload_blob(title, source_file_name):
"""Uploads a file to the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to upload
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -58,7 +59,7 @@ def upload_blob(title, source_file_name):
def delete_blob(title, source_file_name):
"""Deletes a file to from the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to upload
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -76,7 +77,7 @@ def delete_blob(title, source_file_name):
def download_blob(title, source_blob_name):
"""Downloads a file from the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to download
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -100,7 +101,7 @@ def generate_download_signed_url_v4(project_dir):
Engine or from the Google Cloud SDK.
"""

bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
blob_path = "simulations/simulation-" + project_dir + "/"

storage_client = storage.Client()
Expand Down Expand Up @@ -144,7 +145,7 @@ def create_log(title):
def check_log(title):
"""Check for log"""
storage_client = storage.Client()
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
bucket = storage_client.bucket(bucket_name)
blob_path = f"simulations/simulation-{title}/log.txt"
return storage.Blob(bucket=bucket, name=blob_path).exists(storage_client)
Expand Down
11 changes: 6 additions & 5 deletions layered/compute-engine/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
from googleapiclient import discovery

project = os.environ.get("PROJECT_NAME") or "flint-cloud"
gcs_bucket_name = os.environ.get("gcs_bucket_name") or "simulation_data_flint-cloud"
zone = os.environ.get("GCE_ZONE") or "us-central1-a"
instance = os.environ.get("GCE_NAME") or "instance-1"
with open("service_account.json", "w") as saf:
Expand Down Expand Up @@ -39,7 +40,7 @@ def publish_message(topic, attribs, project="flint-cloud"):
def delete_blob(title, source_file_name):
"""Deletes a file to from the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to upload
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -57,7 +58,7 @@ def delete_blob(title, source_file_name):
def upload_blob(title, source_file_name):
"""Uploads a file to the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to upload
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -75,7 +76,7 @@ def upload_blob(title, source_file_name):
def download_blob(title, source_blob_name):
"""Downloads a file from the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to download
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -99,7 +100,7 @@ def generate_download_signed_url_v4(project_dir):
Engine or from the Google Cloud SDK.
"""

bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
blob_path = "simulations/simulation-" + project_dir + "/"

storage_client = storage.Client()
Expand Down Expand Up @@ -143,7 +144,7 @@ def create_log(title):
def check_log(title):
"""Check for log"""
storage_client = storage.Client()
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
bucket = storage_client.bucket(bucket_name)
blob_path = f"simulations/simulation-{title}/log.txt"
return storage.Blob(bucket=bucket, name=blob_path).exists(storage_client)
Expand Down
15 changes: 8 additions & 7 deletions layered/ingress/main.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
app = Flask(__name__)

project = os.environ.get("PROJECT_NAME") or "flint-cloud"
gcs_bucket_name = os.environ.get("gcs_bucket_name") or "simulation_data_flint-cloud"
zone = os.environ.get("GCE_ZONE") or "us-central1-a"
instance = os.environ.get("GCE_NAME") or "instance-1"
with open("service_account.json", "w") as saf:
Expand Down Expand Up @@ -55,7 +56,7 @@ def publish_message(topic, data, project=project):
def upload_blob(title, source_file_name):
"""Uploads a file to the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to upload
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand All @@ -73,7 +74,7 @@ def upload_blob(title, source_file_name):
def download_blob(title, source_blob_name):
"""Downloads a file from the bucket."""
# The ID of your GCS bucket
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
# The path to your file to download
# source_file_name = "local/path/to/file"
# The ID of your GCS object
Expand Down Expand Up @@ -124,7 +125,7 @@ def gcbm_new():
title = "".join(c for c in title if c.isalnum())
name = "simulations/simulation-" + title + "/input.zip"
storage_client = storage.Client()
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
bucket = storage_client.bucket(bucket_name)
stats = storage.Blob(bucket=bucket, name=name).exists(storage_client)

Expand Down Expand Up @@ -258,7 +259,7 @@ def status():

# Verify output exists
storage_client = storage.Client()
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
bucket = storage_client.bucket(bucket_name)
blob_path = f"simulations/simulation-{title}/output.zip"
stats = storage.Blob(bucket=bucket, name=blob_path).exists(storage_client)
Expand Down Expand Up @@ -305,7 +306,7 @@ def gcbm_dynamic():

# Verify simulation exists
storage_client = storage.Client()
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
bucket = storage_client.bucket(bucket_name)
blob_path = f"simulations/simulation-{title}/input.zip"
stats = storage.Blob(bucket=bucket, name=blob_path).exists(storage_client)
Expand Down Expand Up @@ -343,7 +344,7 @@ def generate_download_signed_url_v4(project_dir):
Engine or from the Google Cloud SDK.
"""

bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
blob_path = "simulations/simulation-" + project_dir + "/"

storage_client = storage.Client()
Expand Down Expand Up @@ -416,7 +417,7 @@ def gcbm_list_simulations():
description: GCBM Simulations List
"""
storage_client = storage.Client()
bucket_name = "simulation_data_flint-cloud"
bucket_name = gcs_bucket_name
blobs = storage_client.list_blobs(bucket_name, prefix="simulations/")
blob_map = {}
blob_list = []
Expand Down
22 changes: 19 additions & 3 deletions layered/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,11 @@ variable "project" {
default = "flint-cloud"
}

variable "gcs_bucket_name" {
type = string
default = "simulation_data_flint-cloud"
}

variable "region" {
type = string
default = "us-central1"
Expand Down Expand Up @@ -133,7 +138,7 @@ resource "google_pubsub_topic" "large-simulations" {
resource "google_cloud_run_service" "fc-ingress" {
name = var.ingress_name
location = var.region

template {
spec {
containers {
Expand Down Expand Up @@ -190,7 +195,7 @@ resource "google_cloud_run_service_iam_policy" "noauth-ingress" {
resource "google_cloud_run_service" "fc-cr-processor" {
name = var.cr_processor_name
location = var.region

template {
spec {
containers {
Expand Down Expand Up @@ -255,9 +260,20 @@ resource "google_pubsub_subscription" "small-simulations-sub" {
depends_on = [google_pubsub_topic.small-simulations, google_cloud_run_service.fc-cr-processor]
}

# The following example shows how to generate a unique name for an GCS bucket

resource "random_id" "server" {
keepers = {
gcs_bucket_id = "${var.gcs_bucket_id}"
}

byte_length = 8
}

# GCS bucket
resource "google_storage_bucket" "simulation_data_flint-cloud" {
name = "simulation_data_flint-cloud"
gcs_bucket_name = "simulation_data_flint-cloud_${random_id.server.hex}"
name = gcs_bucket_name
location = var.region
force_destroy = true

Expand Down