From f494509444f6db70c79581c5aa5b68003ef450b4 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 8 Jul 2020 22:05:10 +0200 Subject: [PATCH] Fix #290 - Add last-modified to authoritative objects --- docs/admins/object-validation.rst | 38 ++++++++++++ docs/releases/4.1.0.rst | 7 +++ docs/spelling_wordlist.txt | 3 + irrd/rpki/importer.py | 2 +- irrd/rpsl/parser.py | 17 +++++- irrd/rpsl/tests/test_rpsl_objects.py | 20 +++++++ irrd/scripts/set_last_modified_auth.py | 59 +++++++++++++++++++ .../tests/test_set_last_modified_auth.py | 48 +++++++++++++++ irrd/storage/database_handler.py | 5 +- irrd/storage/tests/test_database.py | 24 ++++---- irrd/updates/tests/test_handler.py | 16 +---- setup.py | 1 + 12 files changed, 209 insertions(+), 31 deletions(-) create mode 100755 irrd/scripts/set_last_modified_auth.py create mode 100644 irrd/scripts/tests/test_set_last_modified_auth.py diff --git a/docs/admins/object-validation.rst b/docs/admins/object-validation.rst index a187cc9e1..4c59e7bb4 100644 --- a/docs/admins/object-validation.rst +++ b/docs/admins/object-validation.rst @@ -78,3 +78,41 @@ Other changes from RFCs: components. * IRRd does not accept prefixes with host bits set. RFCs are unclear on whether these are allowed. + + +Modifications to objects +------------------------ +There are a few cases where IRRd makes changes to the object text. + +rpki-ov-state +^^^^^^^^^^^^^ +The ``rpki-ov-state`` attribute, which is used to indicate the +:doc:`RPKI validation status `, is always discarded from all +incoming objects. Where relevant, it is added to the output of queries. +This applies to authoritative and non-authoritative sources. + +key-cert objects +^^^^^^^^^^^^^^^^ +In `key-cert` objects, the ``fingerpr`` and ``owner`` attributes are +updated to values extracted from the PGP key. The ``method`` attribute is +always set to PGP. This applies to objects from authoritative sources and +sources for which ``strict_import_keycert_objects`` is set. + +.. _last-modified: + +last-modified +^^^^^^^^^^^^^ +For authoritative objects, the ``last-modified`` attribute is set every +time the object is created updated. Any existing ``last-modified`` values are +discarded. This timestamp is not updated for changes in RPKI validation +status. + +By default, this attribute is only added when an object is changed or +created. If you have upgraded to IRRd 4.1, you can use the +``irrd_set_last_modified_auth`` command to set it on all existing +authoritative objects. This may take in the order of 10 minutes, depending +on the number of objects to be updated. This only needs to be done once. +It is safe to execute while other IRRd processes are running. +Journal entries are not created when running this command, i.e. the bulk +updates to ``last-modified`` are not visible over NRTM until the object +is updated for a different reason. diff --git a/docs/releases/4.1.0.rst b/docs/releases/4.1.0.rst index d3c539799..0d49fca53 100644 --- a/docs/releases/4.1.0.rst +++ b/docs/releases/4.1.0.rst @@ -83,6 +83,10 @@ Other changes mirror is. For more extensive status information, like the local serials in the journal, :doc:`use the new !J command `. +* The ``last-modified`` attribute is set every time the object is created or + updated in an authoritative source. You can apply this to all existing + authoritative objects with the + :ref:`new irrd_set_last_modified_auth command `. * IRRd starts a maximum of three mirror processes at the same time, to reduce peak loads. A further three, if needed, are started 15 seconds later, regardless of whether the previous ones have finished. @@ -129,6 +133,9 @@ whether RPKI-aware mode is enabled or not. * Ensure that RPKI-aware mode is configured as desired. By default it is **enabled**. * Start IRRd and re-enable the cron / e-mail triggered tasks. +* If you would like to set ``last-modified`` for existing authoritative + objects, use the + :ref:`new irrd_set_last_modified_auth command `. Downgrading from 4.1 to 4.0.x diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 1ac116635..4da176ed1 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -40,6 +40,9 @@ Redistributions reStructuredText RPSL rpsl +rpki-ov-state +ov +rpki rr setcap snapshotting diff --git a/irrd/rpki/importer.py b/irrd/rpki/importer.py index a0a6b7c32..2928c7bd9 100644 --- a/irrd/rpki/importer.py +++ b/irrd/rpki/importer.py @@ -212,7 +212,7 @@ def source(self): def pk(self): return f'{self.prefix_str}AS{self.asn}/ML{self.max_length}' - def render_rpsl_text(self): + def render_rpsl_text(self, last_modified=None): object_class_display = f'{self.rpsl_object_class}:'.ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) remarks_fill = RPSL_ATTRIBUTE_TEXT_WIDTH * ' ' remarks = get_setting('rpki.pseudo_irr_remarks').replace('\n', '\n' + remarks_fill).strip() diff --git a/irrd/rpsl/parser.py b/irrd/rpsl/parser.py index 57282d4a7..7a176beb1 100644 --- a/irrd/rpsl/parser.py +++ b/irrd/rpsl/parser.py @@ -10,6 +10,7 @@ from irrd.rpki.status import RPKIStatus from irrd.utils.text import splitline_unicodesafe from .fields import RPSLTextField +from ..conf import get_setting RPSL_ATTRIBUTE_TEXT_WIDTH = 16 TypeRPSLObjectData = List[Tuple[str, str, List[str]]] @@ -154,10 +155,18 @@ def references_strong_inbound(self) -> Set[str]: result.add(field_name) return result - def render_rpsl_text(self) -> str: - """Render the RPSL object as an RPSL string.""" + def render_rpsl_text(self, last_modified: datetime.datetime=None) -> str: + """ + Render the RPSL object as an RPSL string. + If last_modified is provided, removes existing last-modified: + attributes and adds a new one with that timestamp, if self.source() + is authoritative. + """ output = "" + authoritative = get_setting(f'sources.{self.source()}.authoritative') for attr, value, continuation_chars in self._object_data: + if authoritative and last_modified and attr == 'last-modified': + continue attr_display = f'{attr}:'.ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) value_lines = list(splitline_unicodesafe(value)) if not value_lines: @@ -172,6 +181,10 @@ def render_rpsl_text(self) -> str: continuation_char = '+' output += continuation_char + (RPSL_ATTRIBUTE_TEXT_WIDTH - 1) * ' ' + line output += '\n' + if authoritative and last_modified: + output += 'last-modified:'.ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) + output += last_modified.replace(microsecond=0).isoformat().replace('+00:00', 'Z') + output += '\n' return output def generate_template(self): diff --git a/irrd/rpsl/tests/test_rpsl_objects.py b/irrd/rpsl/tests/test_rpsl_objects.py index 13c04ced2..6249ed407 100644 --- a/irrd/rpsl/tests/test_rpsl_objects.py +++ b/irrd/rpsl/tests/test_rpsl_objects.py @@ -3,6 +3,7 @@ import pytest from IPy import IP from pytest import raises +from pytz import timezone from irrd.conf import PASSWORD_HASH_DUMMY_VALUE from irrd.utils.rpsl_samples import (object_sample_mapping, SAMPLE_MALFORMED_EMPTY_LINE, @@ -497,3 +498,22 @@ def test_parse(self): ] assert obj.references_strong_inbound() == set() assert obj.render_rpsl_text() == rpsl_text + + +class TestLastModified: + def test_authoritative(self, config_override): + config_override({ + 'sources': {'TEST': {'authoritative': True}} + }) + rpsl_text = object_sample_mapping[RPSLRtrSet().rpsl_object_class] + obj = rpsl_object_from_text(rpsl_text + 'last-modified: old-value\n') + last_modified = datetime.datetime(2020, 1, 1, tzinfo=timezone('UTC')) + expected_text = rpsl_text + 'last-modified: 2020-01-01T00:00:00Z\n' + assert obj.render_rpsl_text(last_modified=last_modified) == expected_text + + def test_not_authoritative(self): + rpsl_text = object_sample_mapping[RPSLRtrSet().rpsl_object_class] + obj = rpsl_object_from_text(rpsl_text + 'last-modified: old-value\n') + last_modified = datetime.datetime(2020, 1, 1, tzinfo=timezone('UTC')) + expected_text = rpsl_text + 'last-modified: old-value\n' + assert obj.render_rpsl_text(last_modified=last_modified) == expected_text diff --git a/irrd/scripts/set_last_modified_auth.py b/irrd/scripts/set_last_modified_auth.py new file mode 100755 index 000000000..f44e02d04 --- /dev/null +++ b/irrd/scripts/set_last_modified_auth.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python +# flake8: noqa: E402 +import argparse +import logging +import sys +from pathlib import Path + +from irrd.rpsl.rpsl_objects import rpsl_object_from_text +from irrd.storage.models import RPSLDatabaseObject +from irrd.storage.queries import RPSLDatabaseQuery + +""" +Set last-modified attribute on all authoritative objects. +""" + +logger = logging.getLogger(__name__) +sys.path.append(str(Path(__file__).resolve().parents[2])) + +from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting +from irrd.storage.database_handler import DatabaseHandler + + +def set_last_modified(): + dh = DatabaseHandler() + auth_sources = [k for k, v in get_setting('sources').items() if v.get('authoritative')] + q = RPSLDatabaseQuery(column_names=['pk', 'object_text', 'updated'], enable_ordering=False) + q = q.sources(auth_sources) + + results = list(dh.execute_query(q)) + print(f'Updating {len(results)} objects in sources {auth_sources}') + for result in results: + rpsl_obj = rpsl_object_from_text(result['object_text'], strict_validation=False) + if rpsl_obj.messages.errors(): # pragma: no cover + print(f'Failed to process {rpsl_obj}: {rpsl_obj.messages.errors()}') + continue + new_text = rpsl_obj.render_rpsl_text(result['updated']) + stmt = RPSLDatabaseObject.__table__.update().where( + RPSLDatabaseObject.__table__.c.pk == result['pk']).values( + object_text=new_text, + ) + dh.execute_statement(stmt) + dh.commit() + dh.close() + + +def main(): # pragma: no cover + description = """Set last-modified attribute on all authoritative objects.""" + parser = argparse.ArgumentParser(description=description) + parser.add_argument('--config', dest='config_file_path', type=str, + help=f'use a different IRRd config file (default: {CONFIG_PATH_DEFAULT})') + args = parser.parse_args() + + config_init(args.config_file_path) + + sys.exit(set_last_modified()) + + +if __name__ == '__main__': # pragma: no cover + main() diff --git a/irrd/scripts/tests/test_set_last_modified_auth.py b/irrd/scripts/tests/test_set_last_modified_auth.py new file mode 100644 index 000000000..c0028c198 --- /dev/null +++ b/irrd/scripts/tests/test_set_last_modified_auth.py @@ -0,0 +1,48 @@ +import datetime +import uuid +from unittest.mock import Mock + +from pytz import timezone + +from irrd.utils.rpsl_samples import SAMPLE_RTR_SET +from irrd.utils.test_utils import flatten_mock_calls +from ..set_last_modified_auth import set_last_modified + + +def test_set_force_reload(capsys, monkeypatch, config_override): + config_override({ + 'sources': { + 'TEST': {'authoritative': True}, + 'TEST2': {}, + } + }) + mock_dh = Mock() + monkeypatch.setattr('irrd.scripts.set_last_modified_auth.DatabaseHandler', lambda: mock_dh) + mock_dq = Mock() + monkeypatch.setattr('irrd.scripts.set_last_modified_auth.RPSLDatabaseQuery', lambda column_names, enable_ordering: mock_dq) + + object_pk = uuid.uuid4() + mock_query_result = [ + { + 'pk': object_pk, + 'object_text': SAMPLE_RTR_SET + 'last-modified: old\n', + 'updated': datetime.datetime(2020, 1, 1, tzinfo=timezone('UTC')), + }, + ] + mock_dh.execute_query = lambda query: mock_query_result + + set_last_modified() + + assert flatten_mock_calls(mock_dq) == [ + ['sources', (['TEST'],), {}] + ] + assert mock_dh.mock_calls[0][0] == 'execute_statement' + statement = mock_dh.mock_calls[0][1][0] + new_text = statement.parameters['object_text'] + assert new_text == SAMPLE_RTR_SET + 'last-modified: 2020-01-01T00:00:00Z\n' + + assert flatten_mock_calls(mock_dh)[1:] == [ + ['commit', (), {}], + ['close', (), {}] + ] + assert capsys.readouterr().out == "Updating 1 objects in sources ['TEST']\n" diff --git a/irrd/storage/database_handler.py b/irrd/storage/database_handler.py index 9bc676965..9c8b0c146 100644 --- a/irrd/storage/database_handler.py +++ b/irrd/storage/database_handler.py @@ -165,12 +165,13 @@ def upsert_rpsl_object(self, rpsl_object: RPSLObject, origin: JournalEntryOrigin if rpsl_pk_source in self._rpsl_pk_source_seen: self._flush_rpsl_object_writing_buffer() + update_time = datetime.now(timezone.utc) object_dict = { 'rpsl_pk': rpsl_object.pk(), 'source': source, 'object_class': rpsl_object.rpsl_object_class, 'parsed_data': rpsl_object.parsed_data, - 'object_text': rpsl_object.render_rpsl_text(), + 'object_text': rpsl_object.render_rpsl_text(last_modified=update_time), 'ip_version': rpsl_object.ip_version(), 'ip_first': ip_first, 'ip_last': ip_last, @@ -179,7 +180,7 @@ def upsert_rpsl_object(self, rpsl_object: RPSLObject, origin: JournalEntryOrigin 'asn_first': rpsl_object.asn_first, 'asn_last': rpsl_object.asn_last, 'rpki_status': rpsl_object.rpki_status, - 'updated': datetime.now(timezone.utc), + 'updated': update_time, } self._rpsl_upsert_buffer.append((object_dict, origin)) diff --git a/irrd/storage/tests/test_database.py b/irrd/storage/tests/test_database.py index 6b42b7b33..5bfcbb6e5 100644 --- a/irrd/storage/tests/test_database.py +++ b/irrd/storage/tests/test_database.py @@ -57,7 +57,7 @@ def database_handler_with_route(): pk=lambda: '192.0.2.0/24,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': ['MNT-TEST', 'MNT-TEST2'], 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.0'), ip_last=IP('192.0.2.255'), @@ -88,7 +88,7 @@ def test_object_writing_and_status_checking(self, monkeypatch, irrd_database): pk=lambda: '192.0.2.0/24,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': 'MNT-WRONG', 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.0'), ip_last=IP('192.0.2.255'), @@ -111,7 +111,7 @@ def test_object_writing_and_status_checking(self, monkeypatch, irrd_database): pk=lambda: '2001:db8::/64,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': 'MNT-CORRECT', 'source': 'TEST2'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 6, ip_first=IP('2001:db8::'), ip_last=IP('2001:db8::ffff:ffff:ffff:ffff'), @@ -145,7 +145,7 @@ def test_object_writing_and_status_checking(self, monkeypatch, irrd_database): pk=lambda: 'AS2914', rpsl_object_class='aut-num', parsed_data={'mnt-by': 'MNT-CORRECT', 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: None, ip_first=None, ip_last=None, @@ -279,7 +279,7 @@ def test_disable_journaling(self, monkeypatch, irrd_database): pk=lambda: '192.0.2.0/24,AS65537', rpsl_object_class='route', parsed_data={'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.0'), ip_last=IP('192.0.2.255'), @@ -462,7 +462,7 @@ def test_ordering_sources(self, irrd_database, database_handler_with_route): pk=lambda: '192.0.2.1/32,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': ['MNT-TEST', 'MNT-TEST2'], 'source': 'AAA-TST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.1'), ip_last=IP('192.0.2.1'), @@ -475,7 +475,7 @@ def test_ordering_sources(self, irrd_database, database_handler_with_route): pk=lambda: '192.0.2.2/32,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': ['MNT-TEST', 'MNT-TEST2'], 'source': 'OTHER-SOURCE'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.2'), ip_last=IP('192.0.2.2'), @@ -512,7 +512,7 @@ def test_text_search_person_role(self, irrd_database): pk=lambda: 'PERSON', rpsl_object_class='person', parsed_data={'person': 'my person-name', 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: None, ip_first=None, ip_last=None, @@ -525,7 +525,7 @@ def test_text_search_person_role(self, irrd_database): pk=lambda: 'ROLE', rpsl_object_class='person', parsed_data={'person': 'my role-name', 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: None, ip_first=None, ip_last=None, @@ -549,7 +549,7 @@ def test_more_less_specific_filters(self, irrd_database, database_handler_with_r pk=lambda: '192.0.2.0/25,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': ['MNT-TEST', 'MNT-TEST2'], 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.0'), ip_last=IP('192.0.2.127'), @@ -562,7 +562,7 @@ def test_more_less_specific_filters(self, irrd_database, database_handler_with_r pk=lambda: '192.0.2.128/25,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': ['MNT-TEST', 'MNT-TEST2'], 'source': 'TEST'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.128'), ip_last=IP('192.0.2.255'), @@ -575,7 +575,7 @@ def test_more_less_specific_filters(self, irrd_database, database_handler_with_r pk=lambda: '192.0.2.0/26,AS65537', rpsl_object_class='route', parsed_data={'mnt-by': ['MNT-TEST', 'MNT-TEST2'], 'source': 'TEST2'}, - render_rpsl_text=lambda: 'object-text', + render_rpsl_text=lambda last_modified: 'object-text', ip_version=lambda: 4, ip_first=IP('192.0.2.0'), ip_last=IP('192.0.2.63'), diff --git a/irrd/updates/tests/test_handler.py b/irrd/updates/tests/test_handler.py index fd3d47a8f..67cbd9437 100644 --- a/irrd/updates/tests/test_handler.py +++ b/irrd/updates/tests/test_handler.py @@ -29,7 +29,6 @@ def prepare_mocks(monkeypatch, config_override): class TestChangeSubmissionHandler: # NOTE: the scope of this test also includes ChangeRequest, ReferenceValidator and AuthValidator - # this is more of an update handler integration test. - expected_changed_date = datetime.datetime.now().strftime('%Y%m%d') def test_parse_valid_new_objects_with_override(self, prepare_mocks): mock_dq, mock_dh, mock_email = prepare_mocks @@ -207,18 +206,7 @@ def test_parse_valid_new_person_existing_mntner_pgp_key(self, prepare_mocks): ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ --- Create succeeded: [person] PERSON-TEST - - person: Placeholder Person Object - address: The Netherlands - phone: +31 20 000 0000 - nic-hdl: PERSON-TEST - mnt-by: TEST-MNT - e-mail: email@example.com - changed: changed@example.com {self.expected_changed_date} # comment - source: TEST - - INFO: Set date in changed line "changed@example.com 20190701 # comment" to today. - + --- Modify succeeded: [mntner] TEST-MNT @@ -254,7 +242,7 @@ def test_parse_valid_new_person_existing_mntner_pgp_key(self, prepare_mocks): nic-hdl: PERSON-TEST mnt-by: TEST-MNT e-mail: email@example.com - changed: changed@example.com {self.expected_changed_date} # comment + changed: changed@example.com 20190701 # comment source: TEST --- diff --git a/setup.py b/setup.py index e136f9ad6..0409180a7 100755 --- a/setup.py +++ b/setup.py @@ -55,6 +55,7 @@ 'irrd_database_downgrade = irrd.scripts.database_downgrade:main', 'irrd_load_database = irrd.scripts.load_database:main', 'irrd_update_database = irrd.scripts.update_database:main', + 'irrd_set_last_modified_auth = irrd.scripts.set_last_modified_auth:main', 'irrd_mirror_force_reload = irrd.scripts.mirror_force_reload:main', ], },