diff --git a/infra/gcp/bash/ensure-main-project.sh b/infra/gcp/bash/ensure-main-project.sh index 789fab9be3c..0c5f8f8e3be 100755 --- a/infra/gcp/bash/ensure-main-project.sh +++ b/infra/gcp/bash/ensure-main-project.sh @@ -74,12 +74,13 @@ readonly DNS_GROUP="k8s-infra-dns-admins@kubernetes.io" readonly TERRAFORM_STATE_BUCKET_ENTRIES=( "${LEGACY_CLUSTER_TERRAFORM_BUCKET}:${CLUSTER_ADMINS_GROUP}" k8s-infra-tf-aws:k8s-infra-aws-admins@kubernetes.io + k8s-infra-tf-gcp:k8s-infra-gcp-org-admins@kubernetes.io k8s-infra-tf-monitoring:"${CLUSTER_ADMINS_GROUP}" k8s-infra-tf-prow-clusters:k8s-infra-prow-oncall@kubernetes.io k8s-infra-tf-public-clusters:"${CLUSTER_ADMINS_GROUP}" k8s-infra-tf-public-pii:"${CLUSTER_ADMINS_GROUP}" - k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io k8s-infra-tf-sandbox-capg:k8s-infra-sandbox-capg@kubernetes.io + k8s-infra-tf-sandbox-ii:k8s-infra-ii-coop@kubernetes.io ) # The services we explicitly want enabled for the main project diff --git a/infra/gcp/terraform/k8s-infra-kubernetes-io/main.tf b/infra/gcp/terraform/k8s-infra-kubernetes-io/main.tf new file mode 100644 index 00000000000..2866fa356fb --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-kubernetes-io/main.tf @@ -0,0 +1,139 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +data "google_billing_account" "account" { + billing_account = "018801-93540E-22A20E" +} + +data "google_organization" "org" { + domain = "kubernetes.io" +} + +resource "google_project" "project" { + name = "k8s-infra-kubernetes-io" + project_id = "k8s-infra-kubernetes-io" + org_id = data.google_organization.org.org_id + billing_account = data.google_billing_account.account.billing_account +} + +resource "google_project_service" "project" { + project = google_project.project.id + for_each = toset([ + "billingbudgets.googleapis.com", + "cloudbilling.googleapis.com", + "cloudresourcemanager.googleapis.com", + "monitoring.googleapis.com", + "serviceusage.googleapis.com", + ]) + service = each.value +} + +data "google_project" "kubernetes_public" { + project_id = "kubernetes-public" +} + +resource "google_resource_manager_lien" "kubernetes_public" { + parent = "projects/${data.google_project.kubernetes_public.number}" + restrictions = ["resourcemanager.projects.delete"] + origin = "do-not-delete-kubernetes-public" + reason = "kubernetes-public hosts public-facing kubernetes project infrastructure" +} + +data "google_project" "k8s_infra_sandbox_capg" { + project_id = "k8s-infra-sandbox-capg" +} + +data "google_monitoring_notification_channel" "sig_k8s_infra_leads" { + project = data.google_project.kubernetes_public.project_id + display_name = "sig-k8s-infra-leads@kubernetes.io" +} + +resource "google_billing_budget" "capg_budget" { + billing_account = data.google_billing_account.account.billing_account + display_name = "k8s-infra-sandbox-capg" + budget_filter { + # calendar_period = "MONTH" # TODO: terraform doesn't support this? + projects = [ "projects/${data.google_project.k8s_infra_sandbox_capg.number}" ] + # exclude promotions, which is where our credits come from, since that zeros out cost + credit_types_treatment = "INCLUDE_SPECIFIED_CREDITS" + credit_types = [ + "SUSTAINED_USAGE_DISCOUNT", + "DISCOUNT", + "COMMITTED_USAGE_DISCOUNT", + "FREE_TIER", + "COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE", + "SUBSCRIPTION_BENEFIT", + ] + } + amount { + specified_amount { + currency_code = "USD" + units = "5000" + } + } + all_updates_rule { + # Don't send to users with Billing Account Administrators and + # Billing Account Users IAM roles for the billing account + disable_default_iam_recipients = true + monitoring_notification_channels = [ + data.google_monitoring_notification_channel.sig_k8s_infra_leads.name + ] + } + dynamic "threshold_rules" { + for_each = toset([0.2, 0.5, 0.9, 1.0]) + content { + threshold_percent = threshold_rules.value + } + } +} + +resource "google_billing_budget" "k8s_infra" { + billing_account = data.google_billing_account.account.billing_account + display_name = "k8s-infra-monthly" + budget_filter { + # calendar_period = "MONTH" # TODO: terraform doesn't support this? + # exclude promotions, which is where our credits come from, since that zeros out cost + credit_types_treatment = "INCLUDE_SPECIFIED_CREDITS" + credit_types = [ + "SUSTAINED_USAGE_DISCOUNT", + "DISCOUNT", + "COMMITTED_USAGE_DISCOUNT", + "FREE_TIER", + "COMMITTED_USAGE_DISCOUNT_DOLLAR_BASE", + "SUBSCRIPTION_BENEFIT", + ] + } + amount { + specified_amount { + currency_code = "USD" + units = "250000" # 3M/yr / 12mo + } + } + all_updates_rule { + # Don't send to users with Billing Account Administrators and + # Billing Account Users IAM roles for the billing account + disable_default_iam_recipients = true + monitoring_notification_channels = [ + data.google_monitoring_notification_channel.sig_k8s_infra_leads.name + ] + } + dynamic "threshold_rules" { + for_each = toset([0.9, 1.0]) + content { + threshold_percent = threshold_rules.value + } + } +} diff --git a/infra/gcp/terraform/k8s-infra-kubernetes-io/provider.tf b/infra/gcp/terraform/k8s-infra-kubernetes-io/provider.tf new file mode 100644 index 00000000000..8927cd59617 --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-kubernetes-io/provider.tf @@ -0,0 +1,50 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This file defines: +- Required provider versions +- Storage backend details +*/ + +terraform { + + backend "gcs" { + bucket = "k8s-infra-tf-gcp" + prefix = "kubernetes-io" + } + + required_providers { + google = { + source = "hashicorp/google" + version = "~> 3.87.0" + } + google-beta = { + source = "hashicorp/google-beta" + version = "~> 3.87.0" + } + } +} + +provider "google" { + # necessary to create google_billing_budget resources + # NOTE: bootstrapping was necessary to use this, involved setting to false + # and creating the project / enabling the required services using + # another gcloud project with cloudresourcemanager and serviceusage + # APIs enabled + user_project_override = true + billing_project = "k8s-infra-kubernetes-io" +} diff --git a/infra/gcp/terraform/k8s-infra-kubernetes-io/versions.tf b/infra/gcp/terraform/k8s-infra-kubernetes-io/versions.tf new file mode 100644 index 00000000000..1a7d1da6605 --- /dev/null +++ b/infra/gcp/terraform/k8s-infra-kubernetes-io/versions.tf @@ -0,0 +1,24 @@ +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +/* +This file defines: +- Required Terraform version +*/ + +terraform { + required_version = "~> 1.0.0" +}