From bb32e7e0ea3e2ad9e69f6ed807665336f32b8c1a Mon Sep 17 00:00:00 2001 From: Damien Duportal Date: Mon, 5 Aug 2024 16:44:39 +0200 Subject: [PATCH] feat(dockerhub-mirror) set up a new dedicated ACR to mirror DockerHub inside the Jenkins Azure infrastructure Signed-off-by: Damien Duportal --- cert.ci.jenkins.io.tf | 38 ++++++++++ ci.jenkins.io.tf | 160 +++++---------------------------------- dockerhub-mirror.tf | 150 ++++++++++++++++++++++++++++++++++++ infra.ci.jenkins.io.tf | 38 ++++++++++ trusted.ci.jenkins.io.tf | 36 +++++++++ 5 files changed, 281 insertions(+), 141 deletions(-) create mode 100644 dockerhub-mirror.tf diff --git a/cert.ci.jenkins.io.tf b/cert.ci.jenkins.io.tf index 048b8040..73d621ea 100644 --- a/cert.ci.jenkins.io.tf +++ b/cert.ci.jenkins.io.tf @@ -117,3 +117,41 @@ resource "azurerm_dns_a_record" "cert_ci_jenkins_io" { ttl = 60 records = [module.cert_ci_jenkins_io.controller_private_ipv4] } + +## Allow access to/from ACR endpoint +resource "azurerm_network_security_rule" "allow_out_https_from_cert_ephemeral_agents_to_acr" { + provider = azurerm.jenkins-sponsorship + name = "allow-out-https-from-ephemeral-agents-to-acr" + priority = 4050 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = data.azurerm_subnet.cert_ci_jenkins_io_sponsorship_ephemeral_agents.address_prefixes + destination_address_prefixes = distinct( + flatten( + [for rs in azurerm_private_endpoint.dockerhub_mirror["certcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + ) + ) + resource_group_name = azurerm_resource_group.cert_ci_jenkins_io_controller_jenkins_sponsorship.name + network_security_group_name = module.cert_ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name +} +resource "azurerm_network_security_rule" "allow_in_https_from_cert_ephemeral_agents_to_acr" { + provider = azurerm.jenkins-sponsorship + name = "allow-in-https-from-ephemeral-agents-to-acr" + priority = 4050 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = data.azurerm_subnet.cert_ci_jenkins_io_sponsorship_ephemeral_agents.address_prefixes + destination_address_prefixes = distinct( + flatten( + [for rs in azurerm_private_endpoint.dockerhub_mirror["certcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + ) + ) + resource_group_name = azurerm_resource_group.cert_ci_jenkins_io_controller_jenkins_sponsorship.name + network_security_group_name = module.cert_ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name +} diff --git a/ci.jenkins.io.tf b/ci.jenkins.io.tf index 1a127a79..24bf7494 100644 --- a/ci.jenkins.io.tf +++ b/ci.jenkins.io.tf @@ -95,117 +95,7 @@ resource "azurerm_network_security_rule" "allow_outbound_https_from_cijio_to_cij network_security_group_name = module.ci_jenkins_io_sponsorship.controller_nsg_name } -## Service DNS records -resource "azurerm_dns_cname_record" "ci_jenkins_io" { - name = trimsuffix(trimsuffix(local.ci_jenkins_io_fqdn, data.azurerm_dns_zone.jenkinsio.name), ".") - zone_name = data.azurerm_dns_zone.jenkinsio.name - resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name - ttl = 60 - record = module.ci_jenkins_io_sponsorship.controller_public_fqdn - tags = local.default_tags -} -resource "azurerm_dns_cname_record" "assets_ci_jenkins_io" { - name = "assets.${azurerm_dns_cname_record.ci_jenkins_io.name}" - zone_name = data.azurerm_dns_zone.jenkinsio.name - resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name - ttl = 60 - record = module.ci_jenkins_io_sponsorship.controller_public_fqdn - tags = local.default_tags -} - -#### ACR to use as DockerHub (and other) Registry mirror -data "azurerm_resource_group" "cijio_agents" { - name = "ci-jenkins-io-ephemeral-agents" - provider = azurerm.jenkins-sponsorship -} - -resource "azurerm_container_registry" "cijenkinsio" { - name = "cijenkinsio" - provider = azurerm.jenkins-sponsorship - resource_group_name = data.azurerm_resource_group.cijio_agents.name - location = data.azurerm_resource_group.cijio_agents.location - sku = "Premium" - admin_enabled = false - public_network_access_enabled = false # private links are used to reach the registry - anonymous_pull_enabled = true # Require "Standard" or "Premium" sku. Docker Engine cannot use auth. for pull trough cache - ref. https://github.com/moby/moby/issues/30880 - data_endpoint_enabled = true # Required for endpoint private link - - tags = local.default_tags -} - -locals { - # CredentialSet is not supported by Terraform, so we have to specify its name - acr_cijenkinsio_dockerhub_credentialset = "dockerhub" -} - -resource "azurerm_container_registry_cache_rule" "mirror_dockerhub" { - name = "mirror" - provider = azurerm.jenkins-sponsorship - container_registry_id = azurerm_container_registry.cijenkinsio.id - source_repo = "docker.io/*" - target_repo = "*" - # Credential created manually (unsupported by Terraform) - credential_set_id = "${azurerm_container_registry.cijenkinsio.id}/credentialSets/${local.acr_cijenkinsio_dockerhub_credentialset}" -} - -resource "azurerm_private_endpoint" "acr_cijenkinsio_agents" { - name = "acr-cijenkinsio-agents" - provider = azurerm.jenkins-sponsorship - location = data.azurerm_resource_group.cijio_agents.location - resource_group_name = data.azurerm_resource_group.cijio_agents.name - subnet_id = data.azurerm_subnet.ci_jenkins_io_ephemeral_agents_jenkins_sponsorship.id - - private_service_connection { - name = "acr-cijenkinsio-agents" - private_connection_resource_id = azurerm_container_registry.cijenkinsio.id - subresource_names = ["registry"] - is_manual_connection = false - } - private_dns_zone_group { - name = "privatelink.azurecr.io" - private_dns_zone_ids = [azurerm_private_dns_zone.acr_ci_jenkins_io.id] - } - tags = local.default_tags -} - -resource "azurerm_private_dns_zone" "acr_ci_jenkins_io" { - name = "privatelink.azurecr.io" - provider = azurerm.jenkins-sponsorship - resource_group_name = data.azurerm_resource_group.cijio_agents.name - - tags = local.default_tags -} - -resource "azurerm_private_dns_zone_virtual_network_link" "acr_ci_jenkins_io_vnet_dns" { - name = "acr-ci-jenkins-io-vnet_-dns" - provider = azurerm.jenkins-sponsorship - resource_group_name = data.azurerm_resource_group.cijio_agents.name - private_dns_zone_name = azurerm_private_dns_zone.acr_ci_jenkins_io.name - virtual_network_id = data.azurerm_virtual_network.public_jenkins_sponsorship.id - - registration_enabled = true - tags = local.default_tags -} - -resource "azurerm_key_vault" "ci_jenkins_io" { - name = "ddutest" # "ci-jenkins-io" - provider = azurerm.jenkins-sponsorship - location = data.azurerm_resource_group.cijio_agents.location - resource_group_name = data.azurerm_resource_group.cijio_agents.name - - tenant_id = data.azurerm_client_config.current.tenant_id - soft_delete_retention_days = 7 - purge_protection_enabled = false - enable_rbac_authorization = true - enabled_for_deployment = true - enabled_for_disk_encryption = true - enabled_for_template_deployment = true - - sku_name = "standard" - - tags = local.default_tags -} - +## Allow access to/from ACR endpoint resource "azurerm_network_security_rule" "allow_out_https_from_cijio_agents_to_acr" { provider = azurerm.jenkins-sponsorship name = "allow-out-https-from-cijio-agents-to-acr" @@ -215,42 +105,30 @@ resource "azurerm_network_security_rule" "allow_out_https_from_cijio_agents_to_a protocol = "Tcp" source_port_range = "*" destination_port_range = "443" - source_address_prefixes = data.azurerm_subnet.ci_jenkins_io_ephemeral_agents.address_prefixes + source_address_prefixes = data.azurerm_subnet.ci_jenkins_io_ephemeral_agents_jenkins_sponsorship.address_prefixes destination_address_prefixes = distinct( flatten( - [for rs in azurerm_private_endpoint.acr_cijenkinsio_agents.private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + [for rs in azurerm_private_endpoint.dockerhub_mirror["cijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] ) ) resource_group_name = module.ci_jenkins_io_sponsorship.controller_resourcegroup_name - network_security_group_name = module.ci_jenkins_io_sponsorship.controller_nsg_name + network_security_group_name = module.ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name } -resource "azurerm_network_security_rule" "allow_in_https_from_cijio_agents_to_acr" { - provider = azurerm.jenkins-sponsorship - name = "allow-in-https-from-cijio-agents-to-acr" - priority = 4050 - direction = "Inbound" - access = "Allow" - protocol = "Tcp" - source_port_range = "*" - destination_port_range = "443" - source_address_prefixes = distinct( - flatten( - [for rs in azurerm_private_endpoint.acr_cijenkinsio_agents.private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] - ) - ) - destination_address_prefixes = data.azurerm_subnet.ci_jenkins_io_ephemeral_agents.address_prefixes - resource_group_name = module.ci_jenkins_io_sponsorship.controller_resourcegroup_name - network_security_group_name = module.ci_jenkins_io_sponsorship.controller_nsg_name +## Service DNS records +resource "azurerm_dns_cname_record" "ci_jenkins_io" { + name = trimsuffix(trimsuffix(local.ci_jenkins_io_fqdn, data.azurerm_dns_zone.jenkinsio.name), ".") + zone_name = data.azurerm_dns_zone.jenkinsio.name + resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name + ttl = 60 + record = module.ci_jenkins_io_sponsorship.controller_public_fqdn + tags = local.default_tags } - -# This role allows the ACR registry to read secrets -# Note: an admin must insert secrets into the keyvault manually and then create the credentialset in ACR manually -# which requires the "Key Vault Secrets Officer" or "Owner" role temporarily -resource "azurerm_role_assignment" "acr_read_keyvault_secrets" { - provider = azurerm.jenkins-sponsorship - scope = azurerm_key_vault.ci_jenkins_io.id - role_definition_name = "Key Vault Secrets User" - principal_id = "201fed0a-6e86-4600-a12b-945f2c1c0eb2" - skip_service_principal_aad_check = true +resource "azurerm_dns_cname_record" "assets_ci_jenkins_io" { + name = "assets.${azurerm_dns_cname_record.ci_jenkins_io.name}" + zone_name = data.azurerm_dns_zone.jenkinsio.name + resource_group_name = data.azurerm_resource_group.proddns_jenkinsio.name + ttl = 60 + record = module.ci_jenkins_io_sponsorship.controller_public_fqdn + tags = local.default_tags } diff --git a/dockerhub-mirror.tf b/dockerhub-mirror.tf new file mode 100644 index 00000000..0af51326 --- /dev/null +++ b/dockerhub-mirror.tf @@ -0,0 +1,150 @@ +#### ACR to use as DockerHub (and other) Registry mirror +resource "azurerm_resource_group" "dockerhub_mirror" { + name = "dockerhub-mirror" + provider = azurerm.jenkins-sponsorship + location = var.location +} + +resource "azurerm_container_registry" "dockerhub_mirror" { + name = "dockerhubmirror" + provider = azurerm.jenkins-sponsorship + resource_group_name = azurerm_resource_group.dockerhub_mirror.name + location = azurerm_resource_group.dockerhub_mirror.location + sku = "Premium" + admin_enabled = false + public_network_access_enabled = false # private links are used to reach the registry + anonymous_pull_enabled = true # Requires "Standard" or "Premium" sku. Docker Engine cannot use auth. for pull trough cache - ref. https://github.com/moby/moby/issues/30880 + data_endpoint_enabled = true # Required for endpoint private link. Requires "Premium" sku. + + tags = local.default_tags +} + +locals { + acr_private_links = { + "cijenkinsio" = { + "subnet_id" = data.azurerm_subnet.ci_jenkins_io_kubernetes_sponsorship.id + "vnet_id" = data.azurerm_virtual_network.public_jenkins_sponsorship.id + "rg_name" = data.azurerm_virtual_network.public_jenkins_sponsorship.resource_group_name + }, + "certcijenkinsio" = { + "subnet_id" = data.azurerm_subnet.cert_ci_jenkins_io_sponsorship_ephemeral_agents.id, + "vnet_id" = data.azurerm_virtual_network.cert_ci_jenkins_io_sponsorship.id + "rg_name" = data.azurerm_virtual_network.cert_ci_jenkins_io_sponsorship.resource_group_name + }, + "trustedcijenkinsio" = { + "subnet_id" = data.azurerm_subnet.trusted_ci_jenkins_io_sponsorship_ephemeral_agents.id, + "vnet_id" = data.azurerm_virtual_network.trusted_ci_jenkins_io_sponsorship.id + "rg_name" = data.azurerm_virtual_network.trusted_ci_jenkins_io_sponsorship.resource_group_name + }, + "infracijenkinsio" = { + "subnet_id" = data.azurerm_subnet.infra_ci_jenkins_io_sponsorship_ephemeral_agents.id, + "vnet_id" = data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.id + "rg_name" = data.azurerm_virtual_network.infra_ci_jenkins_io_sponsorship.resource_group_name + }, + } +} + +resource "azurerm_private_endpoint" "dockerhub_mirror" { + for_each = local.acr_private_links + + name = "acr-${each.key}" + provider = azurerm.jenkins-sponsorship + + location = azurerm_resource_group.dockerhub_mirror.location + resource_group_name = azurerm_resource_group.dockerhub_mirror.name + subnet_id = each.value.subnet_id + + custom_network_interface_name = "acr-${each.key}-nic" + + private_service_connection { + name = "acr-${each.key}" + private_connection_resource_id = azurerm_container_registry.dockerhub_mirror.id + subresource_names = ["registry"] + is_manual_connection = false + } + private_dns_zone_group { + name = "privatelink.azurecr.io" + private_dns_zone_ids = [azurerm_private_dns_zone.dockerhub_mirror[each.key].id] + } + tags = local.default_tags +} + +resource "azurerm_private_dns_zone" "dockerhub_mirror" { + for_each = local.acr_private_links + + # Conventional and static name required by Azure (otherwise automatic record creation does not work) + name = "privatelink.azurecr.io" + provider = azurerm.jenkins-sponsorship + + # Private DNS zone name is static: we can only have one per RG + resource_group_name = each.value.rg_name + + tags = local.default_tags +} + +resource "azurerm_private_dns_zone_virtual_network_link" "dockerhub_mirror" { + for_each = local.acr_private_links + + name = "privatelink.azurecr.io" + provider = azurerm.jenkins-sponsorship + # Private DNS zone name is static: we can only have one per RG + resource_group_name = each.value.rg_name + private_dns_zone_name = azurerm_private_dns_zone.dockerhub_mirror[each.key].name + virtual_network_id = each.value.vnet_id + + registration_enabled = true + tags = local.default_tags +} + +#trivy:ignore:avd-azu-0016 +resource "azurerm_key_vault" "dockerhub_mirror" { + name = "dockerhubmirror" + provider = azurerm.jenkins-sponsorship + location = azurerm_resource_group.dockerhub_mirror.location + resource_group_name = azurerm_resource_group.dockerhub_mirror.name + + tenant_id = data.azurerm_client_config.current.tenant_id + soft_delete_retention_days = 7 + purge_protection_enabled = false + enable_rbac_authorization = true + enabled_for_deployment = true + enabled_for_disk_encryption = true + enabled_for_template_deployment = true + public_network_access_enabled = false + + network_acls { + bypass = "AzureServices" + default_action = "Deny" + } + + sku_name = "standard" + + tags = local.default_tags +} + +# IMPORTANT: when bootstraping, multiple Terraform apply are required until ACR CredentialSet can be created by Terraform (unsupported by Terraform until https://github.com/hashicorp/terraform-provider-azurerm/issues/26539 is done). +# 1. Start by creating the dockerhub-username and docker-password in the Keyvault (once created) which requires the "Key Vault Secrets Officer" or "Owner" role temporarily +# 2. Then create the CredentialSet in the registry (once created) with the name 'dockerhub'. It will be marked as "Unhealthy" (expected). +# 3. Then retrieve the principal ID and set it in the attributes below. +# 4. Finally re-run terraform apply one last time to create this role_assignement and the ACR cache rule. The CrednetialSet in ACR willb e marked as "Helathy" right after this apply. +resource "azurerm_role_assignment" "acr_read_keyvault_secrets" { + provider = azurerm.jenkins-sponsorship + scope = azurerm_key_vault.dockerhub_mirror.id + role_definition_name = "Key Vault Secrets User" + skip_service_principal_aad_check = true + # Need to be retrieved manually from Azure UI -> Container Registries -> Select the "azurerm_key_vault.dockerhub_mirror" resource -> Services -> Cache -> Crerdentials -> select "dockerhub" + principal_id = "90872c87-43ab-446d-89b2-741693c34b90" +} + +resource "azurerm_container_registry_cache_rule" "mirror_dockerhub" { + name = "mirror" + provider = azurerm.jenkins-sponsorship + container_registry_id = azurerm_container_registry.dockerhub_mirror.id + source_repo = "docker.io/*" + target_repo = "*" + + # Credential created manually (unsupported by Terraform until https://github.com/hashicorp/terraform-provider-azurerm/issues/26539 is done). + # Check dependent resource + depends_on = [azurerm_role_assignment.acr_read_keyvault_secrets] + credential_set_id = "${azurerm_container_registry.dockerhub_mirror.id}/credentialSets/dockerhub" +} diff --git a/infra.ci.jenkins.io.tf b/infra.ci.jenkins.io.tf index 2d678916..873ee2d5 100644 --- a/infra.ci.jenkins.io.tf +++ b/infra.ci.jenkins.io.tf @@ -318,3 +318,41 @@ resource "azurerm_role_assignment" "infra_ci_jenkins_io_allow_azurerm_privatek8s role_definition_id = azurerm_role_definition.infra_ci_jenkins_io_controller_disk_reader.role_definition_resource_id principal_id = azurerm_kubernetes_cluster.privatek8s.identity[0].principal_id } + +## Allow access to/from ACR endpoint +resource "azurerm_network_security_rule" "allow_out_https_from_infra_ephemeral_agents_to_acr" { + provider = azurerm.jenkins-sponsorship + name = "allow-out-https-from-ephemeral-agents-to-acr" + priority = 4050 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = data.azurerm_subnet.infra_ci_jenkins_io_sponsorship_ephemeral_agents.address_prefixes + destination_address_prefixes = distinct( + flatten( + [for rs in azurerm_private_endpoint.dockerhub_mirror["infracijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + ) + ) + resource_group_name = azurerm_resource_group.infra_ci_jenkins_io_controller_jenkins_sponsorship.name + network_security_group_name = module.infra_ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name +} +resource "azurerm_network_security_rule" "allow_in_https_from_infra_ephemeral_agents_to_acr" { + provider = azurerm.jenkins-sponsorship + name = "allow-in-https-from-ephemeral-agents-to-acr" + priority = 4050 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = data.azurerm_subnet.infra_ci_jenkins_io_sponsorship_ephemeral_agents.address_prefixes + destination_address_prefixes = distinct( + flatten( + [for rs in azurerm_private_endpoint.dockerhub_mirror["infracijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + ) + ) + resource_group_name = azurerm_resource_group.infra_ci_jenkins_io_controller_jenkins_sponsorship.name + network_security_group_name = module.infra_ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name +} diff --git a/trusted.ci.jenkins.io.tf b/trusted.ci.jenkins.io.tf index c69352ef..2dc3bdd7 100644 --- a/trusted.ci.jenkins.io.tf +++ b/trusted.ci.jenkins.io.tf @@ -345,6 +345,24 @@ resource "azurerm_subnet_network_security_group_association" "trusted_ci_permane } ## Outbound Rules (different set of priorities than Inbound rules) ## +resource "azurerm_network_security_rule" "allow_out_https_from_trusted_ephemeral_agents_to_acr" { + provider = azurerm.jenkins-sponsorship + name = "allow-out-https-from-ephemeral-agents-to-acr" + priority = 4050 + direction = "Outbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = data.azurerm_subnet.trusted_ci_jenkins_io_sponsorship_ephemeral_agents.address_prefixes + destination_address_prefixes = distinct( + flatten( + [for rs in azurerm_private_endpoint.dockerhub_mirror["trustedcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + ) + ) + resource_group_name = azurerm_resource_group.trusted_ci_jenkins_io_controller_jenkins_sponsorship.name + network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name +} # Ignore the rule as it does not detect the IP restriction to only update.jenkins.io"s host #trivy:ignore:azure-network-no-public-egress resource "azurerm_network_security_rule" "allow_outbound_ssh_from_permanent_agent_to_updatecenter" { @@ -453,6 +471,24 @@ resource "azurerm_network_security_rule" "allow_inbound_ssh_from_bounce_to_ephem resource_group_name = module.trusted_ci_jenkins_io.controller_resourcegroup_name network_security_group_name = module.trusted_ci_jenkins_io.controller_nsg_name } +resource "azurerm_network_security_rule" "allow_in_https_from_trusted_ephemeral_agents_to_acr" { + provider = azurerm.jenkins-sponsorship + name = "allow-in-https-from-ephemeral-agents-to-acr" + priority = 4050 + direction = "Inbound" + access = "Allow" + protocol = "Tcp" + source_port_range = "*" + destination_port_range = "443" + source_address_prefixes = data.azurerm_subnet.trusted_ci_jenkins_io_sponsorship_ephemeral_agents.address_prefixes + destination_address_prefixes = distinct( + flatten( + [for rs in azurerm_private_endpoint.dockerhub_mirror["trustedcijenkinsio"].private_dns_zone_configs.*.record_sets : rs.*.ip_addresses] + ) + ) + resource_group_name = azurerm_resource_group.trusted_ci_jenkins_io_controller_jenkins_sponsorship.name + network_security_group_name = module.trusted_ci_jenkins_io_azurevm_agents_jenkins_sponsorship.ephemeral_agents_nsg_name +} #trivy:ignore:azure-network-no-public-ingress resource "azurerm_network_security_rule" "allow_inbound_ssh_from_internet_to_bounce" { name = "allow-inbound-ssh-from-internet-to-bounce"