diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index c946980d8..ce84f0ac1 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -120,7 +120,10 @@ This sample shows most configuration options export_destination: /var/ftp/ export_timer: 7200 nrtm_access_list: generic_nrtm_access - nrtm4_server_private_key: base64privatekey== + nrtm4_server_private_key: | + -----BEGIN PRIVATE KEY----- + key date in base64 PEM here + -----END PRIVATE KEY----- nrtm4_server_local_path: /var/www/nrtmv4/authdatabase/ nrtm4_server_base_url: https://www.example.net/nrtmv4/authdatabase/ MIRROR-FIRST: @@ -155,7 +158,10 @@ This sample shows most configuration options - 'ftp://ftp.example.net/mirror-second.db.route6.gz' - 'ftp://ftp.example.net/mirror-second.db.route-set.gz' nrtm4_client_notification_file_url: https://example.net/nrtmv4/MIRROR-SECOND/update-notification-file.json - nrtm4_client_initial_public_key: base64publickey== + nrtm4_client_initial_public_key: | + -----BEGIN PUBLIC KEY----- + key date in base64 PEM here + -----END PUBLIC KEY----- Loading and reloading @@ -624,10 +630,11 @@ Sources are configured. |br| **Default**: ``false``. |br| **Change takes effect**: after SIGHUP, for all subsequent changes. -* ``sources.{name}.nrtm4_server_private_key``: the private key Ed25519 key - used to sign the Update Notification File for an NRTMv4 server, base64 encoded. +* ``sources.{name}.nrtm4_server_private_key``: the private key + used to sign the Update Notification File for an NRTMv4 server, in PEM format. You may find the ``irrdctl nrtmv4 generate-private-key`` and - ``irrdctl nrtmv4 show-public-key`` convenience commands helpful. + ``irrdctl nrtmv4 server-show-public-key`` convenience commands helpful. + Note the use of the pipe character (``|``) to enter this multi-line data. |br| **Default**: not defined, no NRTMv4 server runs. |br| **Change takes effect**: after SIGHUP, at the next mirror update. * ``sources.{name}.nrtm4_server_private_key_next``: the next private Ed25519 @@ -635,6 +642,7 @@ Sources setting is used for key rotation. Base64 encoded. See the :doc:`mirroring documentation ` for details on key rotation. + Note the use of the pipe character (``|``) to enter this multi-line data. |br| **Default**: not defined, no key rotation signalled. |br| **Change takes effect**: after SIGHUP, at the next mirror update. * ``sources.{name}.nrtm4_server_local_path``: the path where the NRTMv4 server @@ -654,7 +662,8 @@ Sources |br| **Default**: not defined, no NRTMv4 updates attempted. |br| **Change takes effect**: after SIGHUP, at the next mirror update. * ``sources.{name}.nrtm4_client_initial_public_key``: the initial public - Ed25519 key expected to match the Update Notification File signature, base64 encoded. + key expected to match the Update Notification File signature, in PEM format. + Note the use of the pipe character (``|``) to enter this multi-line data. |br| **Default**: not defined, no NRTMv4 updates attempted. |br| **Change takes effect**: after SIGHUP, at the next mirror update. * ``sources.{name}.nrtm_host``: the hostname or IP to connect to for an NRTMv3 stream. diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index 9260ed203..9bbe22bdf 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -244,10 +244,7 @@ def _check_staging_config(self) -> List[str]: Validate the current staging configuration. Returns a list of any errors, or an empty list for a valid config. """ - from irrd.utils.crypto import ( - ed25519_private_key_from_str, - ed25519_public_key_from_str, - ) + from irrd.utils.crypto import eckey_from_str errors = [] config = self.user_config_staging @@ -433,7 +430,7 @@ def _validate_subconfig(key, value): if details.get("nrtm4_client_initial_public_key"): try: - ed25519_public_key_from_str(details["nrtm4_client_initial_public_key"]) + eckey_from_str(details["nrtm4_client_initial_public_key"]) except ValueError as ve: errors.append( f"Invalid value for setting nrtm4_client_initial_public_key for source {name}: {ve}" @@ -469,7 +466,7 @@ def _validate_subconfig(key, value): if details.get("nrtm4_server_private_key"): try: - ed25519_private_key_from_str(details["nrtm4_server_private_key"]) + eckey_from_str(details["nrtm4_server_private_key"], require_private=True) except ValueError as ve: errors.append( f"Invalid value for setting nrtm4_server_private_key for source {name}: {ve}" @@ -477,7 +474,7 @@ def _validate_subconfig(key, value): if details.get("nrtm4_server_private_key_next"): try: - ed25519_private_key_from_str(details["nrtm4_server_private_key_next"]) + eckey_from_str(details["nrtm4_server_private_key_next"], require_private=True) except ValueError as ve: errors.append( f"Invalid value for setting nrtm4_server_private_key_next for source {name}: {ve}" diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index 74eaa1838..b2def71cc 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -6,6 +6,12 @@ import pytest import yaml +from ..mirroring.nrtm4.tests import ( + MOCK_UNF_PRIVATE_KEY, + MOCK_UNF_PRIVATE_KEY_OTHER_STR, + MOCK_UNF_PRIVATE_KEY_STR, +) +from ..utils.crypto import eckey_public_key_as_str from . import ( ConfigurationError, config_init, @@ -119,10 +125,10 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp "TESTDB-NRTM4": { "keep_journal": True, "nrtm4_client_notification_file_url": "https://testhost/", - "nrtm4_client_initial_public_key": "kL7kSk56ASeaHl6Nj0eXC3XCHkCzktoPA3ceKz/cjOo=", + "nrtm4_client_initial_public_key": eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY), "nrtm4_server_base_url": "https://example.com", - "nrtm4_server_private_key": "FalXchs8HIU22Efc3ipNcxVwYwB+Mp0x9TCM9BFtig0=", - "nrtm4_server_private_key_next": "4YDgaXpRDIU8vJbFYeYgPQqEa4YAdHeRF1s6SLdXCsE=", + "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR, + "nrtm4_server_private_key_next": MOCK_UNF_PRIVATE_KEY_OTHER_STR, "nrtm4_server_local_path": str(tmpdir), "nrtm4_server_snapshot_frequency": 3600 * 2, }, @@ -457,14 +463,8 @@ def test_load_invalid_config(self, save_yaml_config, tmpdir): assert "Unknown setting key: log.unknown" in str(ce.value) assert "Unknown key(s) under source TESTDB: unknown" in str(ce.value) - assert ( - "Invalid value for setting nrtm4_server_private_key for source TESTDB: Incorrect padding" - in str(ce.value) - ) - assert ( - "Invalid value for setting nrtm4_server_private_key_next for source TESTDB: Incorrect padding" - in str(ce.value) - ) + assert "Invalid value for setting nrtm4_server_private_key for source TESTDB:" in str(ce.value) + assert "Invalid value for setting nrtm4_server_private_key_next for source TESTDB:" in str(ce.value) assert "Setting nrtm4_server_base_url for source TESTDB is not a valid https or file URL." in str( ce.value ) diff --git a/irrd/integration_tests/run.py b/irrd/integration_tests/run.py index 25f445600..2db745068 100644 --- a/irrd/integration_tests/run.py +++ b/irrd/integration_tests/run.py @@ -15,7 +15,7 @@ import ujson import yaml from alembic import command, config -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from joserfc.rfc7518.ec_key import ECKey from python_graphql_client import GraphqlClient from irrd.conf import PASSWORD_HASH_DUMMY_VALUE, config_init @@ -41,7 +41,8 @@ ) from irrd.utils.whois_client import whois_query, whois_query_irrd -from ..utils.crypto import ed25519_private_key_as_str, ed25519_public_key_as_str +from ..mirroring.nrtm4 import UPDATE_NOTIFICATION_FILENAME +from ..utils.crypto import eckey_private_key_as_str, eckey_public_key_as_str from .constants import ( EMAIL_DISCARD_MSGS_COMMAND, EMAIL_END, @@ -924,7 +925,7 @@ def _start_irrds(self): with open(self.config_path1, "w") as yaml_file: yaml.safe_dump(config1, yaml_file) - self.nrtm4_private_key = Ed25519PrivateKey.generate() + self.nrtm4_private_key = ECKey.generate_key() config2 = base_config.copy() config2["irrd"]["piddir"] = self.piddir2 config2["irrd"]["database_url"] = self.database_url2 @@ -944,7 +945,7 @@ def _start_irrds(self): "nrtm_host": "127.0.0.1", "nrtm_port": str(self.port_whois1), "nrtm_access_list": "localhost", - "nrtm4_server_private_key": ed25519_private_key_as_str(self.nrtm4_private_key), + "nrtm4_server_private_key": eckey_private_key_as_str(self.nrtm4_private_key), "nrtm4_server_local_path": self.nrtm4_dir2, "nrtm4_server_base_url": f"file://{self.nrtm4_dir2}", "nrtm4_server_snapshot_frequency": 3600, @@ -963,8 +964,8 @@ def _start_irrds(self): config3["irrd"]["rpki"]["roa_source"] = None config3["irrd"]["sources"]["TEST"] = { "keep_journal": True, - "nrtm4_client_notification_file_url": f"file://{self.nrtm4_dir2}update-notification-file.json", - "nrtm4_client_initial_public_key": ed25519_public_key_as_str(self.nrtm4_private_key.public_key()), + "nrtm4_client_notification_file_url": f"file://{self.nrtm4_dir2}{UPDATE_NOTIFICATION_FILENAME}", + "nrtm4_client_initial_public_key": eckey_public_key_as_str(self.nrtm4_private_key), } with open(self.config_path3, "w") as yaml_file: yaml.safe_dump(config3, yaml_file) diff --git a/irrd/mirroring/nrtm4/__init__.py b/irrd/mirroring/nrtm4/__init__.py index e69de29bb..2ee4c313f 100644 --- a/irrd/mirroring/nrtm4/__init__.py +++ b/irrd/mirroring/nrtm4/__init__.py @@ -0,0 +1 @@ +UPDATE_NOTIFICATION_FILENAME = "update-notification-file.jose" diff --git a/irrd/mirroring/nrtm4/nrtm4_client.py b/irrd/mirroring/nrtm4/nrtm4_client.py index 333d3a41d..2656e485b 100644 --- a/irrd/mirroring/nrtm4/nrtm4_client.py +++ b/irrd/mirroring/nrtm4/nrtm4_client.py @@ -1,12 +1,10 @@ -import hashlib import logging import os -from base64 import b64decode from typing import Any, Dict, List, Optional, Tuple from urllib.parse import urlparse import pydantic -from cryptography.exceptions import InvalidSignature +from joserfc.rfc7515.model import CompactSignature from irrd.conf import get_setting from irrd.mirroring.nrtm4.jsonseq import jsonseq_decode @@ -29,7 +27,7 @@ NRTM4ClientDatabaseStatus, ) from irrd.storage.queries import DatabaseStatusQuery -from irrd.utils.crypto import ed25519_public_key_from_str +from irrd.utils.crypto import eckey_from_config, eckey_from_str, jws_deserialize from irrd.utils.misc import format_pydantic_errors logger = logging.getLogger(__name__) @@ -107,7 +105,7 @@ def _run_client(self) -> bool: ) return has_loaded_snapshot - def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, str]: + def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, Optional[str]]: """ Retrieve, verify and parse the Update Notification File. Returns the UNF object and the used key in base64 string. @@ -116,24 +114,17 @@ def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, str]: if not notification_file_url: # pragma: no cover raise RuntimeError("NRTM4 client called for a source without a Update Notification File URL") - unf_content, _ = retrieve_file(notification_file_url, return_contents=True) - unf_hash = hashlib.sha256(unf_content.encode("ascii")).hexdigest() - sig_url = notification_file_url.replace( - "update-notification-file.json", f"update-notification-file-signature-{unf_hash}.sig" - ) - legacy_sig_url = notification_file_url + ".sig" + unf_signed, _ = retrieve_file(notification_file_url, return_contents=True) if "nrtm.db.ripe.net" in notification_file_url: # pragma: no cover - logger.warning( - f"Downloading signature from legacy url {legacy_sig_url} instead of expected {sig_url}" - ) - signature, _ = retrieve_file(legacy_sig_url, return_contents=True) + # When removing this, also remove Optional[] from return type + logger.warning("Expecting raw UNF as source is RIPE*, signature not checked") + unf_payload = unf_signed.encode("ascii") + used_key = None else: - signature, _ = retrieve_file(sig_url, return_contents=True) - - used_key = self._validate_unf_signature(unf_content, signature) + unf_payload, used_key = self._deserialize_unf(unf_signed) unf = NRTM4UpdateNotificationFile.model_validate_json( - unf_content, + unf_payload, context={ "update_notification_file_scheme": urlparse(notification_file_url).scheme, "expected_values": { @@ -143,61 +134,64 @@ def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, str]: ) return unf, used_key - def _validate_unf_signature(self, unf_content: str, signature_b64: str) -> str: + def _deserialize_unf(self, unf_content: str) -> Tuple[bytes, str]: """ Verify the Update Notification File signature, - given the content (before JSON parsing) and a base64 signature. - Returns the used key in base64 string. + given the content (before JWS deserialize). + Returns the deserialized payload and used key in PEM string. """ + compact_signature: Optional[CompactSignature] unf_content_bytes = unf_content.encode("utf-8") - signature = b64decode(signature_b64, validate=True) config_key = get_setting(f"sources.{self.source}.nrtm4_client_initial_public_key") if self.last_status.current_key: - keys = [ + keys_pem = [ self.last_status.current_key, self.last_status.next_key, ] else: - keys = [get_setting(f"sources.{self.source}.nrtm4_client_initial_public_key")] - - for key in keys: - if key and self._validate_ed25519_signature(key, unf_content_bytes, signature): - return key - - if self.last_status.current_key and self._validate_ed25519_signature( - config_key, unf_content_bytes, signature - ): - # While technically just a "signature not valid case", it is a rather - # confusing situation for the user, so gets a special message. - msg = ( - f"{self.source}: No valid signature found for the Update Notification File for signature" - f" {signature_b64}. The signature is valid for public key {config_key} set in the" - " nrtm4_client_initial_public_key setting, but that is only used for initial validation." - f" IRRD is currently expecting the public key {self.last_status.current_key}. If you want to" - " clear IRRDs key information and revert to nrtm4_client_initial_public_key, use the" - " 'irrdctl nrtmv4 client-clear-known-keys' command." - ) - if self.last_status.next_key: - msg += f" or {self.last_status.next_key}" - raise NRTM4ClientError(msg) + keys_pem = [get_setting(f"sources.{self.source}.nrtm4_client_initial_public_key")] + + for key_pem in keys_pem: + if not key_pem: # pragma: no cover + continue + pubkey = eckey_from_str(key_pem) + try: + compact_signature = jws_deserialize(unf_content_bytes, pubkey) + return compact_signature.payload, key_pem + except ValueError: + continue + + if self.last_status.current_key: + compact_signature = None + + try: + ec_key = eckey_from_config(f"sources.{self.source}.nrtm4_client_initial_public_key") + if ec_key: + compact_signature = jws_deserialize( + unf_content_bytes, + ec_key, + ) + except ValueError: # pragma: no cover + pass + if compact_signature: + # While technically just a "signature not valid case", it is a rather + # confusing situation for the user, so gets a special message. + msg = ( + f"{self.source}: No valid signature found for the Update Notification File. The signature" + f" is valid for public key {config_key} set in the nrtm4_client_initial_public_key" + " setting, but that is only used for initial validation. IRRD is currently expecting the" + f" public key {self.last_status.current_key}. If you want to clear IRRDs key information" + " and revert to nrtm4_client_initial_public_key, use the 'irrdctl nrtmv4" + " client-clear-known-keys' command." + ) + if self.last_status.next_key: + msg += f" or {self.last_status.next_key}" + raise NRTM4ClientError(msg) raise NRTM4ClientError( - f"{self.source}: No valid signature found for any known keys, signature {signature_b64}," - f" considered public keys: {keys}" + f"{self.source}: No valid signature found for any known keys, considered public keys: {keys_pem}" ) - def _validate_ed25519_signature(self, key_b64: str, content: bytes, signature: bytes) -> bool: - """ - Verify an Ed25519 signature, given the key in base64, and the content - and signature in bytes. Returns True or False for validity, raises other - exceptions for things like an invalid key format. - """ - try: - ed25519_public_key_from_str(key_b64).verify(signature, content) - return True - except InvalidSignature: - return False - def _current_db_status(self) -> Tuple[bool, NRTM4ClientDatabaseStatus]: """Look up the current status of self.source in the database.""" query = DatabaseStatusQuery().source(self.source) diff --git a/irrd/mirroring/nrtm4/nrtm4_server.py b/irrd/mirroring/nrtm4/nrtm4_server.py index 0e5686756..e443ed4e6 100644 --- a/irrd/mirroring/nrtm4/nrtm4_server.py +++ b/irrd/mirroring/nrtm4/nrtm4_server.py @@ -1,7 +1,5 @@ -import base64 import datetime import gzip -import hashlib import logging import os import secrets @@ -24,11 +22,12 @@ RPSLDatabaseJournalStatisticsQuery, RPSLDatabaseQuery, ) -from irrd.utils.crypto import ed25519_private_key_from_config, ed25519_public_key_as_str +from irrd.utils.crypto import eckey_from_config, eckey_public_key_as_str, jws_serialize from irrd.utils.text import remove_auth_hashes from ...utils.process_support import get_lockfile from ..retrieval import file_hash_sha256 +from . import UPDATE_NOTIFICATION_FILENAME from .jsonseq import jsonseq_encode, jsonseq_encode_one from .nrtm4_types import ( NRTM4DeltaHeader, @@ -39,7 +38,7 @@ logger = logging.getLogger(__name__) -DANGLING_SNAPSHOT_UNF_SIGNATURE_EXPIRY_TIME = datetime.timedelta(minutes=5) +DANGLING_SNAPSHOT_EXPIRY_TIME = datetime.timedelta(minutes=5) class NRTM4Server: @@ -213,22 +212,20 @@ def _commit_status(self) -> None: self._write_unf() self.database_handler.record_nrtm4_server_status(self.source, self.status) - self._expire_snapshots_unf_signatures() + self._expire_snapshots() self.database_handler.commit() def _write_unf(self) -> None: """ - Write the Update Notification File and signature. + Write the Update Notification File. This is based on settings and self.status. """ assert self.status - next_signing_private_key = ed25519_private_key_from_config( + next_signing_private_key = eckey_from_config( f"sources.{self.source}.nrtm4_server_private_key_next", permit_empty=True ) next_signing_public_key = ( - ed25519_public_key_as_str(next_signing_private_key.public_key()) - if next_signing_private_key - else None + eckey_public_key_as_str(next_signing_private_key) if next_signing_private_key else None ) unf = NRTM4UpdateNotificationFile( nrtm_version=4, @@ -251,14 +248,11 @@ def _write_unf(self) -> None: ], ) unf_content = unf.model_dump_json(exclude_none=True, include=unf.model_fields_set).encode("ascii") - private_key = ed25519_private_key_from_config(f"sources.{self.source}.nrtm4_server_private_key") + private_key = eckey_from_config(f"sources.{self.source}.nrtm4_server_private_key") assert private_key - signature = private_key.sign(unf_content) - unf_hash = hashlib.sha256(unf_content).hexdigest() - with open(self.path / f"update-notification-file-signature-{unf_hash}.sig", "wb") as sig_file: - sig_file.write(base64.b64encode(signature)) - with open(self.path / "update-notification-file.json", "wb") as unf_file: - unf_file.write(unf_content) + unf_serialized = jws_serialize(unf_content, private_key) + with open(self.path / UPDATE_NOTIFICATION_FILENAME, "w") as unf_file: + unf_file.write(unf_serialized) self.status.last_update_notification_file_update = unf.timestamp def _expire_deltas(self) -> None: @@ -285,22 +279,21 @@ def _expire_deltas(self) -> None: file_path.unlink() logger.debug(f"{self.source}: Removed dangling delta file {file_path.name}") - def _expire_snapshots_unf_signatures(self) -> None: + def _expire_snapshots(self) -> None: """ - Expire old UNF signatures and old snapshots. + Expire old snapshots. To avoid race conditions, these files are kept around after they are no longer referred. This method cleans them up. """ assert self.status - patterns = ["update-notification-file-signature-*.sig", "nrtm-snapshot.*.json.gz"] - for pattern in patterns: - for file_path in self.path.glob(pattern): - modification_time = datetime.datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC) - if ( - self.timestamp - modification_time > DANGLING_SNAPSHOT_UNF_SIGNATURE_EXPIRY_TIME - and file_path.name != self.status.last_snapshot_filename - ): - file_path.unlink() + snapshot_pattern = "nrtm-snapshot.*.json.gz" + for file_path in self.path.glob(snapshot_pattern): + modification_time = datetime.datetime.fromtimestamp(os.path.getmtime(file_path), tz=UTC) + if ( + self.timestamp - modification_time > DANGLING_SNAPSHOT_EXPIRY_TIME + and file_path.name != self.status.last_snapshot_filename + ): + file_path.unlink() def _write_snapshot(self, version: int) -> str: """ diff --git a/irrd/mirroring/nrtm4/nrtm4_types.py b/irrd/mirroring/nrtm4/nrtm4_types.py index de78308d1..5d3c848f4 100644 --- a/irrd/mirroring/nrtm4/nrtm4_types.py +++ b/irrd/mirroring/nrtm4/nrtm4_types.py @@ -4,11 +4,10 @@ from uuid import UUID import pydantic +from joserfc.rfc7518.ec_key import ECKey from pytz import UTC from typing_extensions import Self -from irrd.utils.crypto import ed25519_public_key_from_str - def get_from_pydantic_context(info: pydantic.ValidationInfo, key: str) -> Optional[Any]: """ @@ -144,7 +143,7 @@ def validate_timestamp(cls, timestamp: datetime.datetime): def validate_next_signing_key(cls, next_signing_key: Optional[str]): if next_signing_key: try: - ed25519_public_key_from_str(next_signing_key) + ECKey.import_key(next_signing_key) except ValueError as ve: raise ValueError( f"Update Notification File has invalid next_signing_key {next_signing_key}: {ve}" diff --git a/irrd/mirroring/nrtm4/tests/__init__.py b/irrd/mirroring/nrtm4/tests/__init__.py index e1da62607..1f7fda02a 100644 --- a/irrd/mirroring/nrtm4/tests/__init__.py +++ b/irrd/mirroring/nrtm4/tests/__init__.py @@ -1,13 +1,22 @@ -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from irrd.utils.crypto import eckey_from_str, eckey_public_key_as_str -from irrd.utils.crypto import ed25519_public_key_as_str - -MOCK_UNF_PRIVATE_KEY = Ed25519PrivateKey.from_private_bytes( - b"\x15\xa9Wr\x1b<\x1c\x856\xd8G\xdc\xde*Ms\x15pc\x00~2\x9d1\xf50\x8c\xf4\x11m\x8a\r" +MOCK_UNF_PRIVATE_KEY_STR = ( + "-----BEGIN PRIVATE" + " KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgGQrdALKHTVC4sVav\nmKUjXaPB22CWZP3t5XSkLqKHMO2hRANCAAQ9U/aaZwLV4koey4Jvu9cRaxiXna9k\naQ3YwrPzZlwd5MQSZ59kfT2+LAbQmXbZg0NGzptqHoOK0YD3YVBjv4kc\n-----END" + " PRIVATE KEY-----\n" ) -MOCK_UNF_PUBLIC_KEY = ed25519_public_key_as_str(MOCK_UNF_PRIVATE_KEY.public_key()) -MOCK_UNF_PRIVATE_KEY_OTHER = Ed25519PrivateKey.from_private_bytes( - b"\xe1\x80\xe0izQ\x0c\x85<\xbc\x96\xc5a\xe6 =\n\x84k\x86\x00tw\x91\x17[:H\xb7W\n\xc1" +MOCK_UNF_PRIVATE_KEY = eckey_from_str(MOCK_UNF_PRIVATE_KEY_STR) + +MOCK_UNF_PUBLIC_KEY = eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY) + +MOCK_UNF_PRIVATE_KEY_OTHER_STR = ( + "-----BEGIN PRIVATE" + " KEY-----\nMIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgmHtbXrQ0uEcrzeZK\niaK8UnpD5c/YAmqdUHqoHLz997ShRANCAAQ18hSL1o3ynp1kLsfXgZBtlWSYwKvc\nLT2qRj7QeJPxHA6X3XMk7eD6xbdeyNFnLXiKwNPFMPcwRLC6oLN81Fvb\n-----END" + " PRIVATE KEY-----\n" ) -MOCK_UNF_PUBLIC_KEY_OTHER = ed25519_public_key_as_str(MOCK_UNF_PRIVATE_KEY_OTHER.public_key()) + + +MOCK_UNF_PRIVATE_KEY_OTHER = eckey_from_str(MOCK_UNF_PRIVATE_KEY_OTHER_STR) + +MOCK_UNF_PUBLIC_KEY_OTHER = eckey_public_key_as_str(MOCK_UNF_PRIVATE_KEY_OTHER) diff --git a/irrd/mirroring/nrtm4/tests/test_nrtm4_client.py b/irrd/mirroring/nrtm4/tests/test_nrtm4_client.py index 639ac1f5e..d98440bff 100644 --- a/irrd/mirroring/nrtm4/tests/test_nrtm4_client.py +++ b/irrd/mirroring/nrtm4/tests/test_nrtm4_client.py @@ -1,11 +1,11 @@ -import base64 -import hashlib import json from tempfile import NamedTemporaryFile from uuid import UUID, uuid4 import pytest +from joserfc import jws +from irrd.mirroring.nrtm4 import UPDATE_NOTIFICATION_FILENAME from irrd.mirroring.nrtm4.jsonseq import jsonseq_encode from irrd.mirroring.nrtm4.nrtm4_client import NRTM4Client, NRTM4ClientError from irrd.mirroring.nrtm4.tests import ( @@ -22,8 +22,7 @@ MOCK_SNAPSHOT_URL = "https://example.com/snapshot.2.json" MOCK_DELTA3_URL = "https://example.com/delta.3.json" MOCK_DELTA4_URL = "https://example.com/delta.4.json" -MOCK_UNF_URL = "https://example.com/update-notification-file.json" -MOCK_UNF_SIG_URL = "https://example.com/update-notification-file-signature-hash.json" +MOCK_UNF_URL = "https://example.com/" + UPDATE_NOTIFICATION_FILENAME MOCK_UNF = { "nrtm_version": 4, @@ -88,17 +87,9 @@ def _mock_retrieve_file(tmp_path, mock_responses): def mock_retrieve_file(url, expected_hash=None, return_contents=True): url = str(url) mock_unf_content = json.dumps(mock_responses[MOCK_UNF_URL]) + mock_unf_serialized = jws.serialize_compact({"alg": "ES256"}, mock_unf_content, MOCK_UNF_PRIVATE_KEY) if url == MOCK_UNF_URL and return_contents: - return mock_unf_content, False - elif "update-notification-file-signature" in url and return_contents: - try: - return mock_responses[MOCK_UNF_SIG_URL], False - except KeyError: - unf_bytes = mock_unf_content.encode("utf-8") - unf_hash = hashlib.sha256(unf_bytes).hexdigest() - if unf_hash not in url: # pragma: no cover - raise ValueError(f"Signature URL requested {url}, expected hash {unf_hash}") - return base64.b64encode(MOCK_UNF_PRIVATE_KEY.sign(unf_bytes)), False + return mock_unf_serialized, False elif not return_contents: assert url == expected_hash destination = NamedTemporaryFile(dir=tmp_path, delete=False) @@ -172,17 +163,17 @@ def test_valid_from_delta(self, prepare_nrtm4_test, caplog): self._assert_import_queries(mock_dh, expect_reload=False) assert "Updating from deltas, starting from version 3" in caplog.text - def test_invalid_signature(self, prepare_nrtm4_test, monkeypatch, tmp_path): - mock_responses = { - MOCK_UNF_URL: MOCK_UNF, - MOCK_UNF_SIG_URL: "invalid-base64", - MOCK_SNAPSHOT_URL: MOCK_SNAPSHOT, - # Shorten delta 3 to header only - MOCK_DELTA3_URL: MOCK_DELTA3, - MOCK_DELTA4_URL: MOCK_DELTA4, - } - monkeypatch.setattr( - "irrd.mirroring.nrtm4.nrtm4_client.retrieve_file", _mock_retrieve_file(tmp_path, mock_responses) + def test_invalid_signature(self, prepare_nrtm4_test, monkeypatch, tmp_path, config_override): + config_override( + { + "sources": { + "TEST": { + "nrtm4_client_notification_file_url": MOCK_UNF_URL, + "nrtm4_client_initial_public_key": MOCK_UNF_PUBLIC_KEY_OTHER, + }, + }, + "rpki": {"roa_source": None}, + } ) mock_dh = MockDatabaseHandler() mock_dh.reset_mock() @@ -190,10 +181,6 @@ def test_invalid_signature(self, prepare_nrtm4_test, monkeypatch, tmp_path): with pytest.raises(NRTM4ClientError): NRTM4Client("TEST", mock_dh).run_client() - mock_responses[MOCK_UNF_SIG_URL] = base64.b64encode(b"invalid-key") - with pytest.raises(NRTM4ClientError): - NRTM4Client("TEST", mock_dh).run_client() - def test_invalid_empty_delta(self, prepare_nrtm4_test, tmp_path, monkeypatch): mock_responses = { MOCK_UNF_URL: MOCK_UNF, @@ -434,7 +421,7 @@ def test_invalid_current_db_key_with_valid_config_key(self, prepare_nrtm4_test, ) with pytest.raises(NRTM4ClientError) as exc: NRTM4Client("TEST", mock_dh).run_client() - assert "use the 'irrdctl" in str(exc) + assert "is valid for" in str(exc) def test_uses_current_db_key(self, prepare_nrtm4_test, config_override): config_override( diff --git a/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py b/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py index c3a724304..45820ff5e 100644 --- a/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py +++ b/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py @@ -1,7 +1,5 @@ -import base64 import dataclasses import gzip -import hashlib import json import os import time @@ -9,9 +7,10 @@ from unittest.mock import create_autospec from irrd.conf import NRTM4_SERVER_DELTA_EXPIRY_TIME, PASSWORD_HASH_DUMMY_VALUE +from irrd.mirroring.nrtm4 import UPDATE_NOTIFICATION_FILENAME from irrd.mirroring.nrtm4.jsonseq import jsonseq_decode from irrd.mirroring.nrtm4.nrtm4_server import NRTM4Server, NRTM4ServerWriter -from irrd.mirroring.nrtm4.tests import MOCK_UNF_PRIVATE_KEY, MOCK_UNF_PUBLIC_KEY +from irrd.mirroring.nrtm4.tests import MOCK_UNF_PRIVATE_KEY, MOCK_UNF_PRIVATE_KEY_STR from irrd.mirroring.retrieval import check_file_hash_sha256 from irrd.storage.models import DatabaseOperation, NRTM4ServerDatabaseStatus from irrd.storage.queries import ( @@ -20,7 +19,7 @@ RPSLDatabaseJournalStatisticsQuery, RPSLDatabaseQuery, ) -from irrd.utils.crypto import ed25519_private_key_as_str, ed25519_public_key_from_str +from irrd.utils.crypto import jws_deserialize from irrd.utils.rpsl_samples import SAMPLE_MNTNER from irrd.utils.test_utils import MockDatabaseHandler from irrd.utils.text import remove_auth_hashes @@ -64,7 +63,7 @@ def test_nrtm4_server(self, tmpdir, config_override): "piddir": pid_path, "sources": { "TEST": { - "nrtm4_server_private_key": ed25519_private_key_as_str(MOCK_UNF_PRIVATE_KEY), + "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR, "nrtm4_server_local_path": str(nrtm_path), "nrtm4_server_base_url": BASE_URL, # "nrtm4_server_snapshot_frequency": 0, @@ -76,8 +75,7 @@ def test_nrtm4_server(self, tmpdir, config_override): delta_dangling_path = nrtm_path / "nrtm-delta.aaaaa.json.gz" snapshot_outdated_path = nrtm_path / "nrtm-snapshot.aaaaa.json.gz" - unf_signature_outdated_path = nrtm_path / "update-notification-file-signature-aaaaa.sig" - for path in delta_dangling_path, snapshot_outdated_path, unf_signature_outdated_path: + for path in delta_dangling_path, snapshot_outdated_path: path.touch() os.utime(path, (time.time() - 3600, time.time() - 3600)) @@ -251,18 +249,13 @@ def test_nrtm4_server(self, tmpdir, config_override): assert not mock_dh.other_calls def _load_unf(self, nrtm_path): - with open(nrtm_path / "update-notification-file.json", "rb") as f: + with open(nrtm_path / UPDATE_NOTIFICATION_FILENAME, "rb") as f: unf_content = f.read() - unf = json.loads(unf_content) + unf_payload = jws_deserialize(unf_content, MOCK_UNF_PRIVATE_KEY) + unf = json.loads(unf_payload.payload) assert unf["nrtm_version"] == 4 assert unf["source"] == "TEST" assert unf["type"] == "notification" - - unf_hash = hashlib.sha256(unf_content).hexdigest() - with open(nrtm_path / f"update-notification-file-signature-{unf_hash}.sig", "r") as sig_file: - sig_content = base64.b64decode(sig_file.read()) - public_key = ed25519_public_key_from_str(MOCK_UNF_PUBLIC_KEY) - public_key.verify(sig_content, unf_content) return unf def _status_to_dict(self, status: NRTM4ServerDatabaseStatus, force_reload=False): diff --git a/irrd/scripts/irrd_control.py b/irrd/scripts/irrd_control.py index 5dd320bcb..8300adf1e 100755 --- a/irrd/scripts/irrd_control.py +++ b/irrd/scripts/irrd_control.py @@ -7,12 +7,12 @@ from pathlib import Path import click -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from joserfc.rfc7518.ec_key import ECKey from irrd.utils.crypto import ( - ed25519_private_key_as_str, - ed25519_private_key_from_config, - ed25519_public_key_as_str, + eckey_from_config, + eckey_private_key_as_str, + eckey_public_key_as_str, ) from irrd.webui.helpers import send_authentication_change_mail @@ -180,13 +180,13 @@ def generate_private_key(): """ Generate a new private key for an NRTMv4 server. """ - private_key = Ed25519PrivateKey.generate() - private_key_str = ed25519_private_key_as_str(private_key) - public_key_str = ed25519_public_key_as_str(private_key.public_key()) + private_key = ECKey.generate_key() + private_key_str = eckey_private_key_as_str(private_key) + public_key_str = eckey_public_key_as_str(private_key) click.echo( - f"Private key: {private_key_str}\nCorresponding public key: {public_key_str}\nNote: this key has not" - " been configured in IRRD, that is a manual step. This is only a convenience command." + f"Private key:\n\n{private_key_str}\nCorresponding public key:\n\n{public_key_str}\nNote: this key" + " has not been configured in IRRD, that is a manual step. This is only a convenience command." ) @@ -196,24 +196,19 @@ def server_show_public_key(source: str): """ Show the public key(s) matching the currently configured private keys. """ - private_key = ed25519_private_key_from_config( - f"sources.{source}.nrtm4_server_private_key", permit_empty=True - ) + private_key = eckey_from_config(f"sources.{source}.nrtm4_server_private_key", permit_empty=True) if not private_key: raise click.ClickException(f"Source {source} is not configured as an NRTMv4 server") - public_key_str = ed25519_public_key_as_str(private_key.public_key()) + public_key_str = eckey_public_key_as_str(private_key) click.echo( - f"Source {source} NRTMv4 server public keys (base64):\n" - f"Current public key (from nrtm4_server_private_key): {public_key_str}" + f"Current public key for {source} NRTMv4 server (from nrtm4_server_private_key):\n\n{public_key_str}" ) - next_private_key = ed25519_private_key_from_config( - f"sources.{source}.nrtm4_server_private_key_next", permit_empty=True - ) + next_private_key = eckey_from_config(f"sources.{source}.nrtm4_server_private_key_next", permit_empty=True) if next_private_key: - next_public_key_str = ed25519_public_key_as_str(next_private_key.public_key()) - click.echo(f"Next key (from nrtm4_server_private_key_next): {next_public_key_str}") + next_public_key_str = eckey_public_key_as_str(next_private_key) + click.echo(f"Next key (from nrtm4_server_private_key_next):\n{next_public_key_str}") def find_user(session_provider: ORMSessionProvider, email: str) -> AuthUser: diff --git a/irrd/scripts/tests/test_irrd_control.py b/irrd/scripts/tests/test_irrd_control.py index d79cf8616..f4574a5b4 100644 --- a/irrd/scripts/tests/test_irrd_control.py +++ b/irrd/scripts/tests/test_irrd_control.py @@ -1,6 +1,6 @@ import pytest from click.testing import CliRunner -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +from joserfc.rfc7518.ec_key import ECKey from irrd.scripts.irrd_control import ( cli, @@ -11,11 +11,7 @@ user_mfa_clear, ) from irrd.storage.models import AuthWebAuthn, RPSLDatabaseStatus -from irrd.utils.crypto import ( - ed25519_private_key_as_str, - ed25519_private_key_from_str, - ed25519_public_key_as_str, -) +from irrd.utils.crypto import eckey_private_key_as_str, eckey_public_key_as_str from irrd.utils.factories import AuthWebAuthnFactory @@ -229,15 +225,14 @@ def test_call(self): runner = CliRunner() result = runner.invoke(generate_private_key) assert result.exit_code == 0 - private_str, public_str = [line.split(":")[1].strip() for line in result.output.splitlines()[:2]] - assert ed25519_public_key_as_str(ed25519_private_key_from_str(private_str).public_key()) == public_str + assert "BEGIN PRIVATE KEY" in result.output class TestNRTM4ServerShowPublicKey: def test_valid(self, config_override): - private_key = Ed25519PrivateKey.generate() - private_key_str = ed25519_private_key_as_str(private_key) - public_key_str = ed25519_public_key_as_str(private_key.public_key()) + private_key = ECKey.generate_key() + private_key_str = eckey_private_key_as_str(private_key) + public_key_str = eckey_public_key_as_str(private_key) config_override( { @@ -245,6 +240,7 @@ def test_valid(self, config_override): "TEST": { "nrtm4_server_private_key": private_key_str, "nrtm4_server_private_key_next": private_key_str, + "nrtm4_server_base_url": "https://url/", } } } diff --git a/irrd/utils/crypto.py b/irrd/utils/crypto.py index 989a6e410..f0b2432d1 100644 --- a/irrd/utils/crypto.py +++ b/irrd/utils/crypto.py @@ -1,37 +1,49 @@ -import base64 -from typing import Optional +from typing import Optional, Union -from cryptography.hazmat.primitives.asymmetric.ed25519 import ( - Ed25519PrivateKey, - Ed25519PublicKey, -) +from joserfc import jws +from joserfc.errors import JoseError +from joserfc.rfc7515.model import CompactSignature +from joserfc.rfc7518.ec_key import ECKey from irrd.conf import get_setting """ -Convenience functions for handling Ed25519 private keys, +Convenience functions for handling ECKey private keys, and decode/encode them for (at this time) NRTMv4 usage. """ -def ed25519_private_key_from_config(setting: str, permit_empty=False) -> Optional[Ed25519PrivateKey]: +def eckey_from_config(setting: str, permit_empty=False) -> Optional[ECKey]: value = get_setting(setting) if not value and permit_empty: return None - return ed25519_private_key_from_str(value) + return eckey_from_str(value) -def ed25519_public_key_from_str(encoded: str) -> Ed25519PublicKey: - return Ed25519PublicKey.from_public_bytes(base64.b64decode(encoded)) +def eckey_from_str(encoded: str, require_private=False) -> ECKey: + key = ECKey.import_key(encoded) + if require_private and not key.is_private: + raise ValueError("ECKey is a public key, but must be a private key") + return key -def ed25519_private_key_from_str(encoded: str) -> Ed25519PrivateKey: - return Ed25519PrivateKey.from_private_bytes(base64.b64decode(encoded)) +def eckey_public_key_as_str(key: ECKey) -> str: + return key.as_pem(private=False).decode("ascii") -def ed25519_public_key_as_str(public_key: Ed25519PublicKey) -> str: - return base64.b64encode(public_key.public_bytes_raw()).decode("ascii") +def eckey_private_key_as_str(key: ECKey) -> str: + return key.as_pem(private=True).decode("ascii") -def ed25519_private_key_as_str(private_key: Ed25519PrivateKey) -> str: - return base64.b64encode(private_key.private_bytes_raw()).decode("ascii") +def jws_deserialize(value: Union[bytes, str], public_key: ECKey) -> CompactSignature: + try: + return jws.deserialize_compact(value, public_key) + except JoseError as error: + raise ValueError(error) + + +def jws_serialize(value: Union[bytes, str], private_key: ECKey) -> str: + try: + return jws.serialize_compact({"alg": "ES256"}, value, private_key) + except JoseError as error: + raise ValueError(error) diff --git a/irrd/utils/tests/test_crypto.py b/irrd/utils/tests/test_crypto.py index 17aa66c73..fd9bbc116 100644 --- a/irrd/utils/tests/test_crypto.py +++ b/irrd/utils/tests/test_crypto.py @@ -1,31 +1,48 @@ -from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PrivateKey +import pytest +from joserfc.rfc7518.ec_key import ECKey from irrd.utils.crypto import ( - ed25519_private_key_as_str, - ed25519_private_key_from_config, - ed25519_private_key_from_str, - ed25519_public_key_as_str, - ed25519_public_key_from_str, + eckey_from_config, + eckey_from_str, + eckey_private_key_as_str, + eckey_public_key_as_str, + jws_deserialize, + jws_serialize, ) -def test_crypto_ed25519(config_override): - private_key = Ed25519PrivateKey.generate() +def test_crypto_eckey(config_override): + private_key = ECKey.generate_key() config_override( - {"sources": {"TEST": {"nrtm4_server_private_key": ed25519_private_key_as_str(private_key)}}} - ) - assert ( - ed25519_private_key_from_config("sources.TEST.nrtm4_server_private_key").private_bytes_raw() - == private_key.private_bytes_raw() - ) - assert ( - ed25519_public_key_from_str( - ed25519_public_key_as_str( - ed25519_private_key_from_str(ed25519_private_key_as_str(private_key)).public_key() - ) - ).public_bytes_raw() - == private_key.public_key().public_bytes_raw() + { + "sources": {"TEST": {"nrtm4_server_private_key": eckey_private_key_as_str(private_key)}}, + "invalid": "invalid", + } ) + assert eckey_from_config("sources.TEST.nrtm4_server_private_key").as_pem() == private_key.as_pem() + assert eckey_from_str( + eckey_public_key_as_str(eckey_from_str(eckey_private_key_as_str(private_key))) + ).as_pem() == private_key.as_pem(private=False) + assert eckey_from_config("sources.OTHER.nrtm4_server_private_key", permit_empty=True) is None + + with pytest.raises(ValueError): + eckey_from_config("invalid") + + eckey_from_str(eckey_public_key_as_str(private_key), require_private=False) + with pytest.raises(ValueError): + eckey_from_str(eckey_public_key_as_str(private_key), require_private=True) + with pytest.raises(ValueError): + eckey_from_str(eckey_public_key_as_str(private_key)[:20]) + + payload = b"test" assert ( - ed25519_private_key_from_config("sources.OTHER.nrtm4_server_private_key", permit_empty=True) is None + jws_deserialize( + jws_serialize(payload, private_key), eckey_from_str(eckey_public_key_as_str(private_key)) + ).payload + == payload ) + with pytest.raises(ValueError): + jws_deserialize(jws_serialize(payload, private_key), ECKey.generate_key()) + + with pytest.raises(ValueError): + jws_serialize(payload, "invalid") diff --git a/poetry.lock b/poetry.lock index c2f426cf4..ccd434783 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1000,8 +1000,11 @@ name = "docutils" version = "0.19" description = "Docutils -- Python Documentation Utilities" optional = false -python-versions = "*" -files = [] +python-versions = ">=3.7" +files = [ + {file = "docutils-0.19-py3-none-any.whl", hash = "sha256:5e1de4d849fee02c63b040a4a3fd567f4ab104defd8a5511fbbc24a8a017efbc"}, + {file = "docutils-0.19.tar.gz", hash = "sha256:33995a6753c30b7f577febfc2c50411fec6aac7f7ffeb7c4cfe5991072dcf9e6"}, +] [[package]] name = "email-validator" @@ -1561,6 +1564,23 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "joserfc" +version = "1.0.0" +description = "The ultimate Python library for JOSE RFCs, including JWS, JWE, JWK, JWA, JWT" +optional = false +python-versions = ">=3.8" +files = [ + {file = "joserfc-1.0.0-py3-none-any.whl", hash = "sha256:1de2c3ac203db8fceb2e84c1e78ba357030b195c21af046a1411711927654a09"}, + {file = "joserfc-1.0.0.tar.gz", hash = "sha256:298a9820c76576f8ca63375d1851cc092f3f225508305c7a36c4632cec38f7bc"}, +] + +[package.dependencies] +cryptography = "*" + +[package.extras] +drafts = ["pycryptodome"] + [[package]] name = "limits" version = "3.13.0" @@ -3881,4 +3901,4 @@ files = [ [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "123ae7911637a1059c530bc3dfb618bbdc6f9114980ae5c82cfb682682b926dc" +content-hash = "36bc4f48122512ec21dca41e480e018f098d9b5a78ae7cf06283c25b03cbd4d3" diff --git a/pyproject.toml b/pyproject.toml index efd5477a0..01d6e93ad 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -63,6 +63,7 @@ zxcvbn = "4.4.28" wtforms-bootstrap5 = "0.3.0" email-validator = "2.2.0" asgi-logger = "0.1.0" +joserfc = "^1.0.0" [tool.poetry.group.dev.dependencies] pytest = "^7.2.1"