From 179c69886be4c8e7e93bf22c34e97bf03e142b22 Mon Sep 17 00:00:00 2001 From: Bryan Fraschetti Date: Thu, 12 Dec 2024 15:18:04 -0500 Subject: [PATCH] feat: Custom keys for apt archives (#5828) Users with local Ubuntu archive mirrors and Landscape instances have been unable to specify the corresponding gpg keys. This resulted in errors such as "NO_PUBKEY" on commands such as "apt update". This commit adds the functionality to supply keys alongside primary and security mirror declarations. The key can either be defined using the "key" mapping, or via a keyid and (optionally) a keyserver. Using either approach, when a key is supplied alongside the primary or security mirror it is now added to /etc/apt/trusted.gpg.d/ as primary.gpg or security.gpg accordingly and the Signed-By field in the deb822 templates are appropriately populated with this value. If no primary key is supplied, it defaults to the ubuntu archive keyring. If no security key is supplied, it falls back on the primary key (to match the behaviour of the security URI falling back on the primary URI), and in turn falls back on the ubuntu archive keyring if that is not defined. Fixes GH-5473 --- cloudinit/config/cc_apt_configure.py | 15 +- templates/sources.list.debian.deb822.tmpl | 4 +- templates/sources.list.ubuntu.deb822.tmpl | 4 +- .../test_apt_configure_sources_list_v3.py | 131 ++++++++++++++++++ .../unittests/config/test_cc_apt_configure.py | 2 +- tools/.github-cla-signers | 1 + 6 files changed, 147 insertions(+), 10 deletions(-) diff --git a/cloudinit/config/cc_apt_configure.py b/cloudinit/config/cc_apt_configure.py index 35445711e60..7392cbff63f 100644 --- a/cloudinit/config/cc_apt_configure.py +++ b/cloudinit/config/cc_apt_configure.py @@ -147,8 +147,8 @@ def apply_apt(cfg, cloud, gpg): _ensure_dependencies(cfg, matcher, cloud) if util.is_false(cfg.get("preserve_sources_list", False)): - add_mirror_keys(cfg, cloud, gpg) - generate_sources_list(cfg, release, mirrors, cloud) + keys = add_mirror_keys(cfg, cloud, gpg) + generate_sources_list(cfg, release, mirrors, cloud, keys) rename_apt_lists(mirrors, arch) try: @@ -421,11 +421,15 @@ def disable_suites(disabled, src, release) -> str: return retsrc -def add_mirror_keys(cfg, cloud, gpg): +def add_mirror_keys(cfg, cloud, gpg) -> Mapping[str, str]: """Adds any keys included in the primary/security mirror clauses""" + keys = {} for key in ("primary", "security"): for mirror in cfg.get(key, []): - add_apt_key(mirror, cloud, gpg, file_name=key) + resp = add_apt_key(mirror, cloud, gpg, file_name=key) + if resp: + keys[f"{key}_key"] = resp + return keys def is_deb822_sources_format(apt_src_content: str) -> bool: @@ -515,7 +519,7 @@ def get_apt_cfg() -> Dict[str, str]: } -def generate_sources_list(cfg, release, mirrors, cloud): +def generate_sources_list(cfg, release, mirrors, cloud, keys): """generate_sources_list create a source.list file based on a custom or default template by replacing mirrors and release in the template""" @@ -528,6 +532,7 @@ def generate_sources_list(cfg, release, mirrors, cloud): aptsrc_file = apt_sources_list params = {"RELEASE": release, "codename": release} + params.update(keys) for k in mirrors: params[k] = mirrors[k] params[k.lower()] = mirrors[k] diff --git a/templates/sources.list.debian.deb822.tmpl b/templates/sources.list.debian.deb822.tmpl index 6d15096c3ab..bb286e6641b 100644 --- a/templates/sources.list.debian.deb822.tmpl +++ b/templates/sources.list.debian.deb822.tmpl @@ -24,11 +24,11 @@ Types: deb deb-src URIs: {{mirror}} Suites: {{codename}} {{codename}}-updates {{codename}}-backports Components: main -Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Signed-By: {{primary_key | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}} ## Major bug fix updates produced after the final release of the distribution. Types: deb deb-src URIs: {{security}} Suites: {{codename}}{% if codename in ('buster', 'stretch') %}/updates{% else %}-security{% endif %} Components: main -Signed-By: /usr/share/keyrings/debian-archive-keyring.gpg +Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/debian-archive-keyring.gpg', true)}} diff --git a/templates/sources.list.ubuntu.deb822.tmpl b/templates/sources.list.ubuntu.deb822.tmpl index 8202dcbe505..0f5a16cf764 100644 --- a/templates/sources.list.ubuntu.deb822.tmpl +++ b/templates/sources.list.ubuntu.deb822.tmpl @@ -45,7 +45,7 @@ Types: deb URIs: {{mirror}} Suites: {{codename}} {{codename}}-updates {{codename}}-backports Components: main universe restricted multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Signed-By: {{primary_key | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)}} ## Ubuntu security updates. Aside from URIs and Suites, ## this should mirror your choices in the previous section. @@ -53,4 +53,4 @@ Types: deb URIs: {{security}} Suites: {{codename}}-security Components: main universe restricted multiverse -Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg +Signed-By: {{security_key | default(primary_key, true) | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true)}} diff --git a/tests/unittests/config/test_apt_configure_sources_list_v3.py b/tests/unittests/config/test_apt_configure_sources_list_v3.py index 5ba6dac86df..cdb26c3bace 100644 --- a/tests/unittests/config/test_apt_configure_sources_list_v3.py +++ b/tests/unittests/config/test_apt_configure_sources_list_v3.py @@ -150,6 +150,46 @@ Components: main restricted """ +EXAMPLE_CUSTOM_KEY_TMPL_DEB822 = """\ +## template:jinja +# Generated by cloud-init +Types: deb deb-src +URIs: {{mirror}} +Suites: {{codename}} {{codename}}-updates +Components: main restricted +Signed-By: {{ + primary_key + | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true) +}} + +# Security section +Types: deb deb-src +URIs: {{security}} +Suites: {{codename}}-security +Components: main restricted +Signed-By: {{ + security_key + | default(primary_key, true) + | default('/usr/share/keyrings/ubuntu-archive-keyring.gpg', true) +}} +""" + +EXPECTED_PM_BASE_CUSTOM_KEYS = """\ +# Generated by cloud-init +Types: deb deb-src +URIs: http://local.ubuntu.com/ +Suites: fakerel fakerel-updates +Components: main restricted +""" + +EXPECTED_SM_BASE_CUSTOM_KEYS = """\ +# Security section +Types: deb deb-src +URIs: http://local.ubuntu.com/ +Suites: fakerel-security +Components: main restricted +""" + @pytest.mark.usefixtures("fake_filesystem") class TestAptSourceConfigSourceList: @@ -332,3 +372,94 @@ def test_apt_v3_srcl_custom_deb822_feature_aware( sources_file = tmpdir.join(apt_file) assert expected == sources_file.read() assert 0o644 == stat.S_IMODE(sources_file.stat().mode) + + @pytest.mark.parametrize( + "distro,pm,pmkey,sm,smkey", + ( + pytest.param( + "ubuntu", + "http://local.ubuntu.com/", + "fakekey 4321", + "http://local.ubuntu.com/", + "fakekey 1234", + ), + pytest.param( + "ubuntu", + "http://local.ubuntu.com/", + "fakekey 4321", + None, + None, + ), + pytest.param( + "ubuntu", "http://local.ubuntu.com/", None, None, None + ), + pytest.param( + "ubuntu", + "http://local.ubuntu.com/", + None, + "http://local.ubuntu.com/", + "fakekey 1234", + ), + ), + ) + def test_apt_v3_srcl_deb822_custom_psm_keys( + self, + distro, + pm, + pmkey, + sm, + smkey, + mocker, + tmpdir, + ): + """test_apt_v3_srcl_deb822_custom_psm_keys - Test the ability to + specify raw GPG keys alongside primary and security mirrors such + that the keys are both added to the trusted.gpg.d directory + also the ubuntu.sources template + """ + + self.deb822 = mocker.patch.object( + cc_apt_configure.features, "APT_DEB822_SOURCE_LIST_FILE", True + ) + + tmpl_file = f"/etc/cloud/templates/sources.list.{distro}.deb822.tmpl" + tmpl_content = EXAMPLE_CUSTOM_KEY_TMPL_DEB822 + util.write_file(tmpl_file, tmpl_content) + + # Base config + cfg = { + "preserve_sources_list": False, + "primary": [{"arches": ["default"], "uri": pm}], + } + + # Add defined variables to the config + if pmkey: + cfg["primary"][0]["key"] = pmkey + if sm: + cfg["security"] = [{"arches": ["default"], "uri": sm}] + if smkey: + cfg["security"][0]["key"] = smkey + + mycloud = get_cloud(distro) + cc_apt_configure.handle("test", {"apt": cfg}, mycloud, None) + + apt_file = f"/etc/apt/sources.list.d/{distro}.sources" + sources_file = tmpdir.join(apt_file) + + default_keyring = f"/usr/share/keyrings/{distro}-archive-keyring.gpg" + trusted_keys_dir = "/etc/apt/trusted.gpg.d/" + primary_keyring = f"{trusted_keys_dir}primary.gpg" + security_keyring = f"{trusted_keys_dir}security.gpg" + + primary_keyring_path = primary_keyring if pmkey else default_keyring + primary_signature = f"Signed-By: {primary_keyring_path}" + expected_pm = f"{EXPECTED_PM_BASE_CUSTOM_KEYS}{primary_signature}" + + fallback_keyring = primary_keyring if pmkey else default_keyring + security_keyring_path = security_keyring if smkey else fallback_keyring + + security_signature = f"Signed-By: {security_keyring_path}" + expected_sm = f"{EXPECTED_SM_BASE_CUSTOM_KEYS}{security_signature}" + + expected = f"{expected_pm}\n\n{expected_sm}\n" + assert expected == sources_file.read() diff --git a/tests/unittests/config/test_cc_apt_configure.py b/tests/unittests/config/test_cc_apt_configure.py index 7b4ce012864..3651355e98f 100644 --- a/tests/unittests/config/test_cc_apt_configure.py +++ b/tests/unittests/config/test_cc_apt_configure.py @@ -328,7 +328,7 @@ def test_remove_source( Components: main restricted universe multiverse Signed-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg""" } - cc_apt.generate_sources_list(cfg, "noble", {}, cloud) + cc_apt.generate_sources_list(cfg, "noble", {}, cloud, {}) if expected_content is None: assert not sources_file.exists() assert f"Removing {sources_file} to favor deb822" in caplog.text diff --git a/tools/.github-cla-signers b/tools/.github-cla-signers index a7dcbb92d63..87631bb0425 100644 --- a/tools/.github-cla-signers +++ b/tools/.github-cla-signers @@ -33,6 +33,7 @@ blackhelicoptersdotnet bmhughes brianphaley BrinKe-dev +bryanfraschetti CalvoM candlerb CarlosNihelton