feat: Custom keys for apt archives (canonical#5828)

Users with local Ubuntu archive mirrors and Landscape instances have
been unable to specify the corresponding gpg keys. This resulted in
errors such as "NO_PUBKEY" on commands such as "apt update".

This commit adds the functionality to supply keys alongside primary
and security mirror declarations. The key can either be defined using
the "key" mapping, or via a keyid and (optionally) a keyserver. Using
either approach, when a key is supplied alongside the primary or
security mirror it is now added to /etc/apt/trusted.gpg.d/ as
primary.gpg or security.gpg accordingly and the Signed-By field in the
deb822 templates are appropriately populated with this value.

If no primary key is supplied, it defaults to the ubuntu archive keyring. If no
security key is supplied, it falls back on the primary key (to match the
behaviour of the security URI falling back on the primary URI), and in turn
falls back on the ubuntu archive keyring if that is not defined.

Fixes canonicalGH-5473

cloudinit/config/cc_apt_configure.py

@@ -147,8 +147,8 @@ def apply_apt(cfg, cloud, gpg):
     _ensure_dependencies(cfg, matcher, cloud)
 
     if util.is_false(cfg.get("preserve_sources_list", False)):
-        add_mirror_keys(cfg, cloud, gpg)
-        generate_sources_list(cfg, release, mirrors, cloud)
+        keys = add_mirror_keys(cfg, cloud, gpg)
+        generate_sources_list(cfg, release, mirrors, cloud, keys)
         rename_apt_lists(mirrors, arch)
 
     try:
@@ -421,11 +421,15 @@ def disable_suites(disabled, src, release) -> str:
     return retsrc
 
 
-def add_mirror_keys(cfg, cloud, gpg):
+def add_mirror_keys(cfg, cloud, gpg) -> Mapping[str, str]:
     """Adds any keys included in the primary/security mirror clauses"""
+    keys = {}
     for key in ("primary", "security"):
         for mirror in cfg.get(key, []):
-            add_apt_key(mirror, cloud, gpg, file_name=key)
+            resp = add_apt_key(mirror, cloud, gpg, file_name=key)
+            if resp:
+                keys[f"{key}_key"] = resp
+    return keys
 
 
 def is_deb822_sources_format(apt_src_content: str) -> bool:
@@ -515,7 +519,7 @@ def get_apt_cfg() -> Dict[str, str]:
     }
 
 
-def generate_sources_list(cfg, release, mirrors, cloud):
+def generate_sources_list(cfg, release, mirrors, cloud, keys):
     """generate_sources_list
     create a source.list file based on a custom or default template
     by replacing mirrors and release in the template"""
@@ -528,6 +532,7 @@ def generate_sources_list(cfg, release, mirrors, cloud):
         aptsrc_file = apt_sources_list
 
     params = {"RELEASE": release, "codename": release}
+    params.update(keys)
     for k in mirrors:
         params[k] = mirrors[k]
         params[k.lower()] = mirrors[k]

templates/sources.list.debian.deb822.tmpl

@@ -24,11 +24,11 @@ Types: deb deb-src
 URIs: {{mirror}}
 Suites: {{codename}} {{codename}}-updates {{codename}}-backports
 Components: main
-Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+Signed-By: {{primary_key | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}}
 
 ## Major bug fix updates produced after the final release of the distribution.
 Types: deb deb-src
 URIs: {{security}}
 Suites: {{codename}}{% if codename in ('buster', 'stretch') %}/updates{% else %}-security{% endif %}
 Components: main
-Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg
+Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}}

templates/sources.list.ubuntu.deb822.tmpl

@@ -45,12 +45,12 @@ Types: deb
 URIs: {{mirror}}
 Suites: {{codename}} {{codename}}-updates {{codename}}-backports
 Components: main universe restricted multiverse
-Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+Signed-By: {{primary_key | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)}}
 
 ## Ubuntu security updates. Aside from URIs and Suites,
 ## this should mirror your choices in the previous section.
 Types: deb
 URIs: {{security}}
 Suites: {{codename}}-security
 Components: main universe restricted multiverse
-Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg
+Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)}}

tests/unittests/config/test_apt_configure_sources_list_v3.py

@@ -150,6 +150,46 @@
 Components: main restricted
 """
 
+EXAMPLE_CUSTOM_KEY_TMPL_DEB822 = """\
+## template:jinja
+# Generated by cloud-init
+Types: deb deb-src
+URIs: {{mirror}}
+Suites: {{codename}} {{codename}}-updates
+Components: main restricted
+Signed-By: {{
+    primary_key
+    | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)
+}}
+
+# Security section
+Types: deb deb-src
+URIs: {{security}}
+Suites: {{codename}}-security
+Components: main restricted
+Signed-By: {{
+    security_key
+    | default(primary_key, true)
+    | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)
+}}
+"""
+
+EXPECTED_PM_BASE_CUSTOM_KEYS = """\
+# Generated by cloud-init
+Types: deb deb-src
+URIs: http://local.ubuntu.com/
+Suites: fakerel fakerel-updates
+Components: main restricted
+"""
+
+EXPECTED_SM_BASE_CUSTOM_KEYS = """\
+# Security section
+Types: deb deb-src
+URIs: http://local.ubuntu.com/
+Suites: fakerel-security
+Components: main restricted
+"""
+
 
 @pytest.mark.usefixtures("fake_filesystem")
 class TestAptSourceConfigSourceList:
@@ -332,3 +372,94 @@ def test_apt_v3_srcl_custom_deb822_feature_aware(
         sources_file = tmpdir.join(apt_file)
         assert expected == sources_file.read()
         assert 0o644 == stat.S_IMODE(sources_file.stat().mode)
+
+    @pytest.mark.parametrize(
+        "distro,pm,pmkey,sm,smkey",
+        (
+            pytest.param(
+                "ubuntu",
+                "http://local.ubuntu.com/",
+                "fakekey 4321",
+                "http://local.ubuntu.com/",
+                "fakekey 1234",
+            ),
+            pytest.param(
+                "ubuntu",
+                "http://local.ubuntu.com/",
+                "fakekey 4321",
+                None,
+                None,
+            ),
+            pytest.param(
+                "ubuntu", "http://local.ubuntu.com/", None, None, None
+            ),
+            pytest.param(
+                "ubuntu",
+                "http://local.ubuntu.com/",
+                None,
+                "http://local.ubuntu.com/",
+                "fakekey 1234",
+            ),
+        ),
+    )
+    def test_apt_v3_srcl_deb822_custom_psm_keys(
+        self,
+        distro,
+        pm,
+        pmkey,
+        sm,
+        smkey,
+        mocker,
+        tmpdir,
+    ):
+        """test_apt_v3_srcl_deb822_custom_psm_keys - Test the ability to
+        specify raw GPG keys alongside primary and security mirrors such
+        that the keys are both added to the trusted.gpg.d directory
+        also the ubuntu.sources template
+        """
+
+        self.deb822 = mocker.patch.object(
+            cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", True
+        )
+
+        tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl"
+        tmpl_content = EXAMPLE_CUSTOM_KEY_TMPL_DEB822
+        util.write_file(tmpl_file, tmpl_content)
+
+        # Base config
+        cfg = {
+            "preserve_sources_list": False,
+            "primary": [{"arches": ["default"], "uri": pm}],
+        }
+
+        # Add defined variables to the config
+        if pmkey:
+            cfg["primary"][0]["key"] = pmkey
+        if sm:
+            cfg["security"] = [{"arches": ["default"], "uri": sm}]
+            if smkey:
+                cfg["security"][0]["key"] = smkey
+
+        mycloud = get_cloud(distro)
+        cc_apt_configure.handle("test", {"apt": cfg}, mycloud, None)
+
+        apt_file = f"/etc/apt/sources.list.d/{distro}.sources"
+        sources_file = tmpdir.join(apt_file)
+
+        default_keyring = f"/usr/share/keyrings/{distro}-archive-keyring.gpg"
+        trusted_keys_dir = "/etc/apt/trusted.gpg.d/"
+        primary_keyring = f"{trusted_keys_dir}primary.gpg"
+        security_keyring = f"{trusted_keys_dir}security.gpg"
+
+        primary_keyring_path = primary_keyring if pmkey else default_keyring
+        primary_signature = f"Signed-By: {primary_keyring_path}"
+        expected_pm = f"{EXPECTED_PM_BASE_CUSTOM_KEYS}{primary_signature}"
+
+        fallback_keyring = primary_keyring if pmkey else default_keyring
+        security_keyring_path = security_keyring if smkey else fallback_keyring
+
+        security_signature = f"Signed-By: {security_keyring_path}"
+        expected_sm = f"{EXPECTED_SM_BASE_CUSTOM_KEYS}{security_signature}"
+
+        expected = f"{expected_pm}\n\n{expected_sm}\n"
+        assert expected == sources_file.read()

tests/unittests/config/test_cc_apt_configure.py

@@ -328,7 +328,7 @@ def test_remove_source(
 Components: main restricted universe multiverse
 Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg"""
         }
-        cc_apt.generate_sources_list(cfg, "noble", {}, cloud)
+        cc_apt.generate_sources_list(cfg, "noble", {}, cloud, {})
         if expected_content is None:
             assert not sources_file.exists()
             assert f"Removing {sources_file} to favor deb822" in caplog.text

tools/.github-cla-signers

@@ -33,6 +33,7 @@ blackhelicoptersdotnet
 bmhughes
 brianphaley
 BrinKe-dev
+bryanfraschetti
 CalvoM
 candlerb
 CarlosNihelton