Feature/add support for all types of azure auth mechanism (#126)

* 785124 - initial impl of additional Azure auth options

* HDP 785124 - standardize variable names, implement unit testing for auth validator

* HDP 785124 - implement cerbot manager unit tests and fix arising issues

* HDP 785124 - formatting changes

* HDP 785124 - streamline if branching in new logic

* HDP 785124 - improve logging

* HDP 785124 - add missing tests and test files

* HDP 785124 - cleanup imports

* HDP 785124 - update readme

* HDP 785124 - fix formatting

* HDP 785124 - delete unused file

* HDP 785124 - reformatting by black

* HDP 785124 - complement tests, increase validator robustness

* HDP devops 785124 - add service principal certification handling, improve tests and refactor credentials validator logic

* Update README.md

Co-authored-by: felixZdi <93919627+felixZdi@users.noreply.github.com>

---------

Co-authored-by: felixZdi <93919627+felixZdi@users.noreply.github.com>

.gitignore

@@ -51,6 +51,7 @@ src/
 # Local testing
 acme_dns_azure/config.yaml
 tests/**/*/*.yaml
+!tests/app/resources/**/*.yaml
 
 # test results 
 .coverage

README.md

@@ -135,11 +135,23 @@ The other placeholders are specified separately.
 See [examples](examples/README.md) for configuration examples.
 
 ```yml
-# Client ID of managed identity
+# Azure credentials choice section. Only one of the following flags should be set to true to indicate which credentials to use. Otherwise an exception would be raised by the validator. 
+# These values are translated into ini file as specified here: https://docs.certbot-dns-azure.co.uk/en/latest/index.html#certbot-azure-workload-identity-ini
+# If no flag is provided the program will try to use sp_client_* values to use service principal credentials first. If those are not both present it will try to use managed_identity_id.
+[use_system_assigned_identity_credentials: <boolean>]
+[use_azure_cli_credentials: <boolean>]
+[use_workload_identity_credentials: <boolean>]
+[use_managed_identity_credentials: <boolean>]
+[use_provided_service_principal_credentials: <boolean>]
+
+# Client ID of managed identity. Must be provided if use_managed_identity_credentials is true. Will be used even if all use_*_credentials flags are set to false, but only if sp_client_* values are not all provided.
 [managed_identity_id: <string>]
 
+# sp_client_* values must be provided if use_provided_service_principal_credentials is true. Will be used even if all use_*_credentials flags are set to false. User must specify id and either secret or certificate path. If both values (id and pwd/cert path) are provided and none of the flags is set to true it has precedence over the use of provided managed_identity_id.
 [sp_client_id: <string>]
 [sp_client_secret: <secret>]
+[sp_certificate_path: <string>]
+# End of Azure credentials choice section.
 
 [azure_environment: <string> | default = "AzurePublicCloud"]
 

acme_dns_azure/certbot_manager.py

@@ -116,24 +116,42 @@ def _create_certbot_ini(self) -> List[str]:
 
     def _create_certbot_dns_azure_ini(self) -> List[str]:
         lines = []
-        if "sp_client_id" in self._config:
+        # decide on credentials to be use to interact with Azure, based on which fag is set to true
+        if self._config.get("use_system_assigned_identity_credentials"):
+            logger.info("Using Azure system assigned identity.")
+            lines.append("dns_azure_msi_system_assigned = true")
+        elif self._config.get("use_azure_cli_credentials"):
+            logger.info("Using Azure CLI credentials.")
+            lines.append("dns_azure_use_cli_credentials = true")
+        elif self._config.get("use_workload_identity_credentials"):
+            logger.info("Using Azure workflow identity.")
+            lines.append("dns_azure_use_workload_identity_credentials = true")
+        elif self._config.get("use_managed_identity_credentials"):
             logger.info(
-                "Using Azure service principal '%s'", self._config["sp_client_id"]
+                "Using Azure managed identity '%s'",
+                self._config["managed_identity_id"],
             )
-            lines.append("dns_azure_sp_client_id = %s" % self._config["sp_client_id"])
             lines.append(
-                "dns_azure_sp_client_secret = %s" % self._config["sp_client_secret"]
+                f'dns_azure_msi_client_id = {self._config.get("managed_identity_id")}'
             )
-        else:
+        elif self._config.get("use_provided_service_principal_credentials"):
             logger.info(
-                "Using Azure managed identity '%s'", self._config["managed_identity_id"]
-            )
-            lines.append(
-                "dns_azure_msi_client_id = %s" % self._config["managed_identity_id"]
+                "Using Azure service principal '%s'", self._config["sp_client_id"]
             )
-        lines.append("dns_azure_tenant_id = %s" % self._config["tenant_id"])
-        lines.append("dns_azure_environment = %s" % self._config["azure_environment"])
+            lines.append(f'dns_azure_sp_client_id = {self._config.get("sp_client_id")}')
+            if self._config.get("sp_client_secret"):
+                lines.append(
+                    f'dns_azure_sp_client_secret = {self._config.get("sp_client_secret")}'
+                )
+            elif self._config.get("sp_certificate_path"):
+                lines.append(
+                    f'dns_azure_sp_certificate_path = {self._config.get("sp_certificate_path")}'
+                )
+
+        lines.append(f'dns_azure_tenant_id = {self._config.get("tenant_id")}')
+        lines.append(f'dns_azure_environment = {self._config.get("azure_environment")}')
 
+        # validate and build certificate list
         idx = 0
         for certificate in self._config["certificates"]:
             for domain in certificate["domains"]:

acme_dns_azure/config/__init__.py

@@ -23,6 +23,13 @@ def load(config_yaml: str = ""):
         config["sp_client_id"] = os.environ["ARM_CLIENT_ID"]
         config["sp_client_secret"] = os.environ["ARM_CLIENT_SECRET"]
 
+    config, result, message = validate_azure_credentials_use(config)
+
+    if result is False:
+        raise ConfigurationError(message)
+
+    logger.info(message)
+
     if config["keyvault_account_secret_name"] == "":
         config["keyvault_account_secret_name"] = "acme-account-%s" % (
             re.sub("[^-a-zA-Z0-9]+", "-", urlparse(config["server"]).netloc)
@@ -63,3 +70,72 @@ def load_from_file(filename: str = None):
         )
     except Exception as e:
         raise ConfigurationError("Error while loading configuration: %s" % e)
+
+
+def validate_azure_credentials_use(config: dict):
+    """
+    Validates config for valid Azure credentials configuration.
+
+    Args:
+        config (dict): pre validated config.yaml as dict.
+
+    Returns:
+        config (dict): config modified as applicable.
+        result (bool): result of the validation, false if validation failed.
+        message (string): a string describing the result or why the validation failed.
+    """
+    # check if only one identity flag is set to true
+    credential_flags = 0
+    if config.get("use_system_assigned_identity_credentials"):
+        credential_flags += 1
+    if config.get("use_azure_cli_credentials"):
+        credential_flags += 1
+    if config.get("use_workload_identity_credentials"):
+        credential_flags += 1
+    if config.get("use_managed_identity_credentials"):
+        credential_flags += 1
+    if config.get("use_provided_service_principal_credentials"):
+        credential_flags += 1
+
+    # to avoid confusion we only accept one flag to be set to true
+    if credential_flags > 1:
+        message = f'{credential_flags} "use_*_identity" flags set to true.'
+        return config, False, message
+
+    # if no flags are set to true we check if other fields are enough for authentication, this was done for backwards compatibility. The old logic of preferring the sp credentials is preserved here.
+    if credential_flags == 0:
+        if not (
+            config.get("sp_client_id")
+            and (config.get("sp_client_secret") or config.get("sp_certificate_path"))
+        ):
+            if not config.get("managed_identity_id"):
+                message = "Azure credentials not specified or incomplete."
+                return config, False, message
+            else:
+                config["use_managed_identity_credentials"] = True
+        else:
+            config["use_provided_service_principal_credentials"] = True
+    # check if the additional values are provided for Service principal and Managed identity use
+    if (
+        config.get("use_provided_service_principal_credentials")
+        and (
+            not (
+                config.get("sp_client_id")
+                and (
+                    config.get("sp_client_secret") or config.get("sp_certificate_path")
+                )
+            )
+        )
+        or (config.get("sp_certificate_path") and config.get("sp_client_secret"))
+    ):
+        message = "Service Principal credential set is invalid. Id and either password or certificate path must be provided (not both)."
+        return config, False, message
+
+    elif config.get("use_managed_identity_credentials") and not config.get(
+        "managed_identity_id"
+    ):
+        message = "Managed identity id is missing!"
+        return config, False, message
+
+    message = "Azure credentials validation successful!"
+    return config, True, message

acme_dns_azure/config/schema.py

@@ -2,14 +2,26 @@
 
 schema = Map(
     {
+        # Azure identity choice section. Choose credentials to be used to interact with Azure, we only accept one value to be true from this set, for reference see: https://docs.certbot-dns-azure.co.uk/en/latest/index.html#certbot-azure-workload-identity-ini
+        Optional("use_system_assigned_identity_credentials"): Bool(),
+        Optional("use_azure_cli_credentials"): Bool(),
+        Optional("use_workload_identity_credentials"): Bool(),
+        # added to support validation logic when choosing credentials to be used
+        Optional("use_managed_identity_credentials"): Bool(),
+        # added to support validation logic when choosing credentials to be used
+        Optional("use_provided_service_principal_credentials"): Bool(),
+        # managed_identity_id must be provided if use_managed_identity_credentials is true
         Optional("managed_identity_id"): Regex(
             r"^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$"
         ),
+        # sp_client* values must be provided if use_provided_service_principal_credentials is true, user can provide password or certificate path
+        Optional("sp_client_id"): Str(),
+        Optional("sp_client_secret"): Str(),
+        Optional("sp_certificate_path"): Str(),
+        # End of Azure identity choice section
         Optional("azure_environment", default="AzurePublicCloud"): Regex(
             "AzurePublicCloud|AzureUSGovernmentCloud|AzureChinaCloud|AzureGermanCloud"
         ),
-        Optional("sp_client_id"): Str(),
-        Optional("sp_client_secret"): Str(),
         "tenant_id": Regex(
             r"^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$"
         ),
@@ -42,7 +54,8 @@
                     "domains": Seq(
                         Map(
                             {
-                                "name": Str(),  # TODO: Check regex Regex(r"(?=^.{4,253}$)(^((?!-)[*a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}$)")
+                                # TODO: Check regex Regex(r"(?=^.{4,253}$)(^((?!-)[*a-zA-Z0-9-]{1,63}(?<!-)\.)+[a-zA-Z]{2,63}$)")
+                                "name": Str(),
                                 Optional("dns_zone_resource_id", default=""): Str(),
                             }
                         )

tests/app/resources/certbot_init/expected_azure_cli_credentials.ini

@@ -0,0 +1,6 @@
+dns_azure_use_cli_credentials = true
+dns_azure_tenant_id = 56578228-5913-11ee-8c99-0242ac120002
+dns_azure_environment = AzurePublicCloud
+dns_azure_zone1 = xyz.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+dns_azure_zone2 = zyx.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme-challenge.zyx
+dns_azure_zone3 = abc.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme

tests/app/resources/certbot_init/expected_managed_idty.ini

@@ -0,0 +1,6 @@
+dns_azure_msi_client_id = 00000000-0000-0000-0000-000000000000
+dns_azure_tenant_id = 56578228-5913-11ee-8c99-0242ac120002
+dns_azure_environment = AzurePublicCloud
+dns_azure_zone1 = xyz.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+dns_azure_zone2 = zyx.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme-challenge.zyx
+dns_azure_zone3 = abc.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme

tests/app/resources/certbot_init/expected_sp_credentials.ini

@@ -0,0 +1,7 @@
+dns_azure_sp_client_id = 00000000-0000-0000-0000-000000000000
+dns_azure_sp_client_secret = xyz
+dns_azure_tenant_id = 56578228-5913-11ee-8c99-0242ac120002
+dns_azure_environment = AzurePublicCloud
+dns_azure_zone1 = xyz.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+dns_azure_zone2 = zyx.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme-challenge.zyx
+dns_azure_zone3 = abc.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme

tests/app/resources/certbot_init/expected_sp_credentials_cert.ini

@@ -0,0 +1,7 @@
+dns_azure_sp_client_id = 00000000-0000-0000-0000-000000000000
+dns_azure_sp_certificate_path = /path/to/cert.pem
+dns_azure_tenant_id = 56578228-5913-11ee-8c99-0242ac120002
+dns_azure_environment = AzurePublicCloud
+dns_azure_zone1 = xyz.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+dns_azure_zone2 = zyx.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme-challenge.zyx
+dns_azure_zone3 = abc.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme

tests/app/resources/certbot_init/expected_system_assigned_idty_credentials.ini

@@ -0,0 +1,6 @@
+dns_azure_msi_system_assigned = true
+dns_azure_tenant_id = 56578228-5913-11ee-8c99-0242ac120002
+dns_azure_environment = AzurePublicCloud
+dns_azure_zone1 = xyz.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+dns_azure_zone2 = zyx.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme-challenge.zyx
+dns_azure_zone3 = abc.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme

tests/app/resources/certbot_init/expected_workload_idty_credentials.ini

@@ -0,0 +1,6 @@
+dns_azure_use_workload_identity_credentials = true
+dns_azure_tenant_id = 56578228-5913-11ee-8c99-0242ac120002
+dns_azure_environment = AzurePublicCloud
+dns_azure_zone1 = xyz.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+dns_azure_zone2 = zyx.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme-challenge.zyx
+dns_azure_zone3 = abc.example.org:/subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com/TXT/_acme

tests/app/resources/config/credential_tests/accepted_azure_cli_credentials.yaml

@@ -0,0 +1,28 @@
+use_azure_cli_credentials: True
+server: https://acme-staging-v02.api.letsencrypt.org/directory
+
+tenant_id: 56578228-5913-11ee-8c99-0242ac120002
+
+key_vault_id: https://my12keyvaultdev.vault.azure.net/
+# keyvault_account_secret_name: #optional / default: acme-account-${acme_server} #if non-existing or empty, certbot register will be executed, account directory will be stored as base64 tar.gz file
+eab:
+  enabled: false #optional / default: false
+#  kid_secret_name: #optional / default: acme-eab-kid #must be created upfront with information from CIT
+#   hmac_key_secret_name: #optional / default: acme-eab-hmac-key #must be created upfront with information from CIT
+
+certbot.ini: |
+  key-type = rsa
+  rsa-key-size = 2048
+
+certificates:
+  - name: tls-xyz-example-org
+    dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+    domains: # min 1 entry
+      - name: xyz.example.org
+      - name: zyx.example.org
+        dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com
+
+  - name: tls-wildcard-abc-example-org
+    dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com
+    domains:
+      - name: "*.abc.example.org"

tests/app/resources/config/credential_tests/accepted_managed_idty.yaml

@@ -0,0 +1,29 @@
+use_managed_identity_credentials: True
+managed_identity_id: 00000000-0000-0000-0000-000000000000
+server: https://acme-staging-v02.api.letsencrypt.org/directory
+
+tenant_id: 56578228-5913-11ee-8c99-0242ac120002
+
+key_vault_id: https://my12keyvaultdev.vault.azure.net/
+# keyvault_account_secret_name: #optional / default: acme-account-${acme_server} #if non-existing or empty, certbot register will be executed, account directory will be stored as base64 tar.gz file
+eab:
+  enabled: false #optional / default: false
+#  kid_secret_name: #optional / default: acme-eab-kid #must be created upfront with information from CIT
+#   hmac_key_secret_name: #optional / default: acme-eab-hmac-key #must be created upfront with information from CIT
+
+certbot.ini: |
+  key-type = rsa
+  rsa-key-size = 2048
+
+certificates:
+  - name: tls-xyz-example-org
+    dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+    domains: # min 1 entry
+      - name: xyz.example.org
+      - name: zyx.example.org
+        dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com
+
+  - name: tls-wildcard-abc-example-org
+    dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com
+    domains:
+      - name: "*.abc.example.org"

tests/app/resources/config/credential_tests/accepted_managed_idty_no_flag.yaml

@@ -0,0 +1,28 @@
+managed_identity_id: 00000000-0000-0000-0000-000000000000
+server: https://acme-staging-v02.api.letsencrypt.org/directory
+
+tenant_id: 56578228-5913-11ee-8c99-0242ac120002
+
+key_vault_id: https://my12keyvaultdev.vault.azure.net/
+# keyvault_account_secret_name: #optional / default: acme-account-${acme_server} #if non-existing or empty, certbot register will be executed, account directory will be stored as base64 tar.gz file
+eab:
+  enabled: false #optional / default: false
+#  kid_secret_name: #optional / default: acme-eab-kid #must be created upfront with information from CIT
+#   hmac_key_secret_name: #optional / default: acme-eab-hmac-key #must be created upfront with information from CIT
+
+certbot.ini: |
+  key-type = rsa
+  rsa-key-size = 2048
+
+certificates:
+  - name: tls-xyz-example-org
+    dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/example.org
+    domains: # min 1 entry
+      - name: xyz.example.org
+      - name: zyx.example.org
+        dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com
+
+  - name: tls-wildcard-abc-example-org
+    dns_zone_resource_id: /subscriptions/2709c03e-5888-11ee-8c99-0242ac120002/resourceGroups/rg123-my-rg/providers/Microsoft.Network/dnszones/my-dev.domain.com
+    domains:
+      - name: "*.abc.example.org"