From 0a4e1671b03bdd683c39dfc25f7de3a9ee3dd960 Mon Sep 17 00:00:00 2001 From: Walkoss Date: Sun, 9 Apr 2023 01:12:49 +0200 Subject: [PATCH] feat(weavier-gcp): add gcs backup --- weaviate/helm/weaviate/Chart.yaml | 2 +- .../weaviate/templates/serviceaccount.yaml | 12 ------ weaviate/helm/weaviate/values.yaml | 13 ------ weaviate/helm/weaviate/values.yaml.tpl | 12 ++++++ weaviate/plural/recipes/weaviate-gcp.yaml | 4 ++ weaviate/terraform/aws/deps.yaml | 13 ++++++ weaviate/terraform/{kube => aws}/main.tf | 10 +++++ .../terraform/{kube => aws}/terraform.tfvars | 0 weaviate/terraform/{kube => aws}/variables.tf | 0 weaviate/terraform/azure/deps.yaml | 13 ++++++ weaviate/terraform/azure/main.tf | 20 +++++++++ weaviate/terraform/azure/terraform.tfvars | 1 + weaviate/terraform/azure/variables.tf | 4 ++ weaviate/terraform/gcp/deps.yaml | 13 ++++++ weaviate/terraform/gcp/main.tf | 43 +++++++++++++++++++ weaviate/terraform/gcp/terraform.tfvars | 5 +++ weaviate/terraform/gcp/variables.tf | 20 +++++++++ weaviate/terraform/kube/deps.yaml | 26 ----------- 18 files changed, 159 insertions(+), 52 deletions(-) delete mode 100644 weaviate/helm/weaviate/templates/serviceaccount.yaml create mode 100644 weaviate/terraform/aws/deps.yaml rename weaviate/terraform/{kube => aws}/main.tf (53%) rename weaviate/terraform/{kube => aws}/terraform.tfvars (100%) rename weaviate/terraform/{kube => aws}/variables.tf (100%) create mode 100644 weaviate/terraform/azure/deps.yaml create mode 100644 weaviate/terraform/azure/main.tf create mode 100644 weaviate/terraform/azure/terraform.tfvars create mode 100644 weaviate/terraform/azure/variables.tf create mode 100644 weaviate/terraform/gcp/deps.yaml create mode 100644 weaviate/terraform/gcp/main.tf create mode 100644 weaviate/terraform/gcp/terraform.tfvars create mode 100644 weaviate/terraform/gcp/variables.tf delete mode 100644 weaviate/terraform/kube/deps.yaml diff --git a/weaviate/helm/weaviate/Chart.yaml b/weaviate/helm/weaviate/Chart.yaml index 3d43e3c59..19b9dc338 100644 --- a/weaviate/helm/weaviate/Chart.yaml +++ b/weaviate/helm/weaviate/Chart.yaml @@ -2,7 +2,7 @@ apiVersion: v2 name: weaviate description: helm chart for weaviate type: application -version: 0.1.0 +version: 0.1.1 appVersion: 1.18.0 dependencies: - name: weaviate diff --git a/weaviate/helm/weaviate/templates/serviceaccount.yaml b/weaviate/helm/weaviate/templates/serviceaccount.yaml deleted file mode 100644 index bd0e77856..000000000 --- a/weaviate/helm/weaviate/templates/serviceaccount.yaml +++ /dev/null @@ -1,12 +0,0 @@ -{{- if .Values.serviceAccount.create -}} -apiVersion: v1 -kind: ServiceAccount -metadata: - name: {{ include "weaviate.serviceAccountName" . }} - labels: - {{- include "weaviate.labels" . | nindent 4 }} - {{- with .Values.serviceAccount.annotations }} - annotations: - {{- toYaml . | nindent 4 }} - {{- end }} -{{- end }} diff --git a/weaviate/helm/weaviate/values.yaml b/weaviate/helm/weaviate/values.yaml index 0d2929583..7e2b9dadb 100644 --- a/weaviate/helm/weaviate/values.yaml +++ b/weaviate/helm/weaviate/values.yaml @@ -2,18 +2,6 @@ # # This is a YAML-formatted file. # # Declare variables to be passed into your templates. -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: true - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - ingress: enabled: true className: "nginx" @@ -36,4 +24,3 @@ weaviate: protocol: TCP port: 80 type: ClusterIP - annotations: {} diff --git a/weaviate/helm/weaviate/values.yaml.tpl b/weaviate/helm/weaviate/values.yaml.tpl index bcc5e4a1a..ace6f82a6 100644 --- a/weaviate/helm/weaviate/values.yaml.tpl +++ b/weaviate/helm/weaviate/values.yaml.tpl @@ -1,3 +1,5 @@ +{{ $isGcp := or (eq .Provider "google") (eq .Provider "gcp") }} + ingress: hosts: - host: {{ .Values.hostname }} @@ -8,3 +10,13 @@ ingress: - secretName: weaviate-tls hosts: - {{ .Values.hostname }} + +weaviate: + backups: + {{ if $isGcp }} + gcs: + enabled: true + envconfig: + BACKUP_GCS_BUCKET: {{ .Values.weaviateBucket }} + BACKUP_GCS_USE_AUTH: "true" + {{ end }} diff --git a/weaviate/plural/recipes/weaviate-gcp.yaml b/weaviate/plural/recipes/weaviate-gcp.yaml index 871c78e62..06c9cc655 100644 --- a/weaviate/plural/recipes/weaviate-gcp.yaml +++ b/weaviate/plural/recipes/weaviate-gcp.yaml @@ -12,6 +12,10 @@ sections: - name: hostname type: DOMAIN documentation: the fully qualified domain name your Weaviate instance will be available at + - name: weaviateBucket + type: BUCKET + documentation: gcs bucket for storing weaviate backups + default: weaviate items: - type: TERRAFORM name: kube diff --git a/weaviate/terraform/aws/deps.yaml b/weaviate/terraform/aws/deps.yaml new file mode 100644 index 000000000..fff3562f9 --- /dev/null +++ b/weaviate/terraform/aws/deps.yaml @@ -0,0 +1,13 @@ +apiVersion: plural.sh/v1alpha1 +kind: Dependencies +metadata: + description: weaviate aws setup + version: 0.1.0 +spec: + dependencies: + - name: aws-bootstrap + repo: bootstrap + type: terraform + version: '>= 0.1.1' + providers: + - aws diff --git a/weaviate/terraform/kube/main.tf b/weaviate/terraform/aws/main.tf similarity index 53% rename from weaviate/terraform/kube/main.tf rename to weaviate/terraform/aws/main.tf index 5a15fb602..9adef4458 100644 --- a/weaviate/terraform/kube/main.tf +++ b/weaviate/terraform/aws/main.tf @@ -8,3 +8,13 @@ resource "kubernetes_namespace" "weaviate" { } } +resource "kubernetes_service_account" "weaviate" { + metadata { + name = "weaviate" + namespace = var.namespace + } + + depends_on = [ + kubernetes_namespace.weaviate + ] +} diff --git a/weaviate/terraform/kube/terraform.tfvars b/weaviate/terraform/aws/terraform.tfvars similarity index 100% rename from weaviate/terraform/kube/terraform.tfvars rename to weaviate/terraform/aws/terraform.tfvars diff --git a/weaviate/terraform/kube/variables.tf b/weaviate/terraform/aws/variables.tf similarity index 100% rename from weaviate/terraform/kube/variables.tf rename to weaviate/terraform/aws/variables.tf diff --git a/weaviate/terraform/azure/deps.yaml b/weaviate/terraform/azure/deps.yaml new file mode 100644 index 000000000..bc0936d3b --- /dev/null +++ b/weaviate/terraform/azure/deps.yaml @@ -0,0 +1,13 @@ +apiVersion: plural.sh/v1alpha1 +kind: Dependencies +metadata: + description: weaviate azure setup + version: 0.1.0 +spec: + dependencies: + - name: azure-bootstrap + repo: bootstrap + type: terraform + version: '>= 0.1.1' + providers: + - azure diff --git a/weaviate/terraform/azure/main.tf b/weaviate/terraform/azure/main.tf new file mode 100644 index 000000000..9adef4458 --- /dev/null +++ b/weaviate/terraform/azure/main.tf @@ -0,0 +1,20 @@ +resource "kubernetes_namespace" "weaviate" { + metadata { + name = var.namespace + labels = { + "app.kubernetes.io/managed-by" = "plural" + "app.plural.sh/name" = "weaviate" + } + } +} + +resource "kubernetes_service_account" "weaviate" { + metadata { + name = "weaviate" + namespace = var.namespace + } + + depends_on = [ + kubernetes_namespace.weaviate + ] +} diff --git a/weaviate/terraform/azure/terraform.tfvars b/weaviate/terraform/azure/terraform.tfvars new file mode 100644 index 000000000..1a055746b --- /dev/null +++ b/weaviate/terraform/azure/terraform.tfvars @@ -0,0 +1 @@ +namespace = {{ .Namespace | quote }} diff --git a/weaviate/terraform/azure/variables.tf b/weaviate/terraform/azure/variables.tf new file mode 100644 index 000000000..81fa5ca1f --- /dev/null +++ b/weaviate/terraform/azure/variables.tf @@ -0,0 +1,4 @@ +variable "namespace" { + type = string + default = "weaviate" +} diff --git a/weaviate/terraform/gcp/deps.yaml b/weaviate/terraform/gcp/deps.yaml new file mode 100644 index 000000000..161dfd13d --- /dev/null +++ b/weaviate/terraform/gcp/deps.yaml @@ -0,0 +1,13 @@ +apiVersion: plural.sh/v1alpha1 +kind: Dependencies +metadata: + description: weaviate gcp setup + version: 0.1.0 +spec: + dependencies: + - name: gcp-bootstrap + repo: bootstrap + type: terraform + version: '>= 0.1.1' + providers: + - gcp diff --git a/weaviate/terraform/gcp/main.tf b/weaviate/terraform/gcp/main.tf new file mode 100644 index 000000000..c15c76be1 --- /dev/null +++ b/weaviate/terraform/gcp/main.tf @@ -0,0 +1,43 @@ +resource "kubernetes_namespace" "weaviate" { + metadata { + name = var.namespace + labels = { + "app.kubernetes.io/managed-by" = "plural" + "app.plural.sh/name" = "weaviate" + } + } +} + +module "weaviate-workload-identity" { + source = "terraform-google-modules/kubernetes-engine/google//modules/workload-identity" + name = "${var.cluster_name}-weaviate-sa" + namespace = var.namespace + project_id = var.project_id + use_existing_k8s_sa = true + annotate_k8s_sa = false + k8s_sa_name = "weaviate" + roles = ["roles/storage.admin"] +} + +module "gcs_buckets" { + source = "github.com/pluralsh/module-library//terraform/gcs-buckets" + + project_id = var.project_id + bucket_names = [var.weaviate_bucket] + service_account_email = module.weaviate-workload-identity.gcp_service_account_email + location = var.bucket_location +} + +resource "kubernetes_service_account" "weaviate" { + metadata { + name = "weaviate" + namespace = var.namespace + annotations = { + "iam.gke.io/gcp-service-account" = module.weaviate-workload-identity.gcp_service_account_email + } + } + + depends_on = [ + kubernetes_namespace.weaviate + ] +} diff --git a/weaviate/terraform/gcp/terraform.tfvars b/weaviate/terraform/gcp/terraform.tfvars new file mode 100644 index 000000000..a9e69e954 --- /dev/null +++ b/weaviate/terraform/gcp/terraform.tfvars @@ -0,0 +1,5 @@ +namespace = {{ .Namespace | quote }} +weaviate_bucket = {{ .Values.weaviateBucket | quote }} +cluster_name = {{ .Cluster | quote }} +project_id = {{ .Project | quote }} +bucket_location = {{ .Context.BucketLocation | quote }} diff --git a/weaviate/terraform/gcp/variables.tf b/weaviate/terraform/gcp/variables.tf new file mode 100644 index 000000000..06e244e2a --- /dev/null +++ b/weaviate/terraform/gcp/variables.tf @@ -0,0 +1,20 @@ +variable "namespace" { + type = string + default = "weaviate" +} + +variable "weaviate_bucket" { + type = string +} + +variable "cluster_name" { + type = string +} + +variable "project_id" { + type = string +} + +variable "bucket_location" { + type = string +} diff --git a/weaviate/terraform/kube/deps.yaml b/weaviate/terraform/kube/deps.yaml deleted file mode 100644 index 98474a21e..000000000 --- a/weaviate/terraform/kube/deps.yaml +++ /dev/null @@ -1,26 +0,0 @@ -apiVersion: plural.sh/v1alpha1 -kind: Dependencies -metadata: - description: weaviate kube setup - version: 0.1.0 -spec: - dependencies: - - name: aws-bootstrap - repo: bootstrap - type: terraform - version: '>= 0.1.1' - optional: true - - name: azure-bootstrap - repo: bootstrap - type: terraform - version: '>= 0.1.1' - optional: true - - name: gcp-bootstrap - repo: bootstrap - type: terraform - version: '>= 0.1.1' - optional: true - providers: - - aws - - azure - - gcp