From a91bee29791163e1cc7b6e915d509b52147e0ce4 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Fri, 8 Nov 2024 16:23:35 +0100 Subject: [PATCH] Add rpsl_data_updated to database status, seperate from existing updated The updated field was updated for any internal state change, which causes it to update even when an export is made. The new rpsl_data_updated field updates only when RPSL objects are inserted, removed or updated for the source. --- docs/admins/prometheus-metrics.rst | 36 ++++++++++++++-- docs/admins/status_page.rst | 8 +++- docs/releases/4.5.0.rst | 13 ++++++ docs/users/queries/graphql.rst | 9 ++-- irrd/mirroring/nrtm4/nrtm4_client.py | 18 ++++---- irrd/mirroring/nrtm4/nrtm4_server.py | 6 ++- irrd/server/graphql/schema_generator.py | 1 + .../graphql/tests/test_schema_generator.py | 1 + irrd/server/http/metrics_generator.py | 42 ++++++++++++++++++- irrd/server/http/status_generator.py | 3 +- .../http/tests/test_metrics_generator.py | 16 ++++++- .../http/tests/test_status_generator.py | 20 ++++++--- irrd/server/query_resolver.py | 3 ++ irrd/server/tests/test_query_resolver.py | 4 ++ ...dd_rpsl_data_updated_field_to_database_.py | 28 +++++++++++++ irrd/storage/database_handler.py | 22 ++++++++-- irrd/storage/models.py | 2 + irrd/storage/queries.py | 1 + irrd/storage/tests/test_database.py | 9 +++- 19 files changed, 209 insertions(+), 33 deletions(-) create mode 100644 irrd/storage/alembic/versions/a635d2217a48_add_rpsl_data_updated_field_to_database_.py diff --git a/docs/admins/prometheus-metrics.rst b/docs/admins/prometheus-metrics.rst index b78e510d5..354fc1f2b 100644 --- a/docs/admins/prometheus-metrics.rst +++ b/docs/admins/prometheus-metrics.rst @@ -66,21 +66,49 @@ Source updates and errors The next part exposes information on when the source was last updated and when the last error occurred. -* `irrd_last_update_seconds`: seconds since the last update +The ``irrd_last_update_*`` and ``irrd_last_rpsl_data_update_*`` fields +sounds similar, but have significant differences: the former updates when +anything is changed in the internal state of the source. That includes RPSL +data changes, but also recording an error, making an export, etc. +The ``irrd_last_rpsl_data_update_*`` fields only update after a modification +to the RPSL data. This includes changes in visibility due to object +suppression status. + +* `irrd_last_update_seconds`: seconds since the last update to RPSL data + + .. code-block:: + + # HELP irrd_last_rpsl_data_update_seconds Seconds since the last update to RPSL data + # TYPE irrd_last_rpsl_data_update_seconds gauge + irrd_last_rpsl_data_update_seconds{source="SOURCE1"} 2289 + irrd_last_rpsl_data_update_seconds{source="SOURCE2"} 4301 + irrd_last_rpsl_data_update_seconds{source="RPKI"} 10 + +* `irrd_last_update_timestamp`: UNIX timestamp of the last update to RPSL data + + .. code-block:: + + # HELP irrd_last_rpsl_data_update_timestamp Timestamp of the last update to RPSL data in seconds since UNIX epoch + # TYPE irrd_last_rpsl_data_update_timestamp gauge + irrd_last_rpsl_data_update_timestamp{source="SOURCE1"} 1699965265 + irrd_last_rpsl_data_update_timestamp{source="SOURCE2"} 1699963253 + irrd_last_rpsl_data_update_timestamp{source="RPKI"} 1699967543 + +* `irrd_last_update_seconds`: seconds since the last internal status change .. code-block:: - # HELP irrd_last_update_seconds Seconds since the last update + # HELP irrd_last_update_seconds Seconds since the last internal status change # TYPE irrd_last_update_seconds gauge irrd_last_update_seconds{source="SOURCE1"} 2289 irrd_last_update_seconds{source="SOURCE2"} 4301 irrd_last_update_seconds{source="RPKI"} 10 -* `irrd_last_update_timestamp`: UNIX timestamp of the last update +* `irrd_last_update_timestamp`: UNIX timestamp of the last internal status change .. code-block:: - # HELP irrd_last_update_timestamp Timestamp of the last update in seconds since UNIX epoch + # HELP irrd_last_update_timestamp Timestamp of the last internal status change in seconds since UNIX epoch # TYPE irrd_last_update_timestamp gauge irrd_last_update_timestamp{source="SOURCE1"} 1699965265 irrd_last_update_timestamp{source="SOURCE2"} 1699963253 diff --git a/docs/admins/status_page.rst b/docs/admins/status_page.rst index a97635918..c06b95aad 100644 --- a/docs/admins/status_page.rst +++ b/docs/admins/status_page.rst @@ -45,8 +45,12 @@ The local information shows: or is missing due to temporary disabling of journal keeping. * `Last export at serial number`: serial at which this IRRd instance last created an export for this source, if any. -* `Last update`: the last time when a change was processed for this source, - either by user submitted changes, NRTMv3 operations, or a full import. +* `Last change to RPSL data`: the last time when the RPSL data for this source + changed, due to adding, updating or removing an RPSL object. + This includes changes in visibility due to object suppression status. +* `Last internal status update`: the last time when the internal state of this + source was changed, either by an NRTM import or export, full import or export, + recording an error, RPKI status change, or user submitted changes. * `Local journal kept`: whether local journal keeping is enabled. * `Last import error occurred at`: when the most recent import error occurred, for mirrored sources. An import error means an object diff --git a/docs/releases/4.5.0.rst b/docs/releases/4.5.0.rst index 9676f7fbd..3e72349d8 100644 --- a/docs/releases/4.5.0.rst +++ b/docs/releases/4.5.0.rst @@ -37,6 +37,19 @@ was added to the HTTP server in IRRD, on ``/metrics/``. This page is only accessible to IPs :doc:`configured ` in the access list set in the ``server.http.status_access_list`` setting. +New "RPSL data updated" status timestamp +---------------------------------------- +Various status overviews of IRRD would show a "last update" per source. +While there are uses for this, many users checked this to ensure mirroring +from a remote source was still active. However, that is not what this +indicates. This timestamp updates for any internal change to database +status, including any exports. + +To cover the common use, a new timestamp was added for the last time +the RPSL data for a source changed. This updates when objects are added, +modified or deleted through any method, including a change in visibility +due to object suppression status. + Other changes ------------- * The ``sources.{name}.object_class_filter`` setting can now also be used diff --git a/docs/users/queries/graphql.rst b/docs/users/queries/graphql.rst index 475884a7e..2fb895189 100644 --- a/docs/users/queries/graphql.rst +++ b/docs/users/queries/graphql.rst @@ -276,9 +276,12 @@ are an alias. Other sources have the the following fields for each valid source: This number can be compared to the serials reported by the mirror directly, to see whether IRRd is up to date. This number is independent from the range in the local journal. -* ``lastUpdate``: the time of the last change to this source. This may be - an authoritative change, an update from a mirror, a re-import, a change - in the RPKI status of an object, or something else. +* ``rpslDataUpdated``: the last time when the RPSL data for this source + changed, due to adding, updating or removing an RPSL object. + This includes changes in visibility due to object suppression status. +* ``lastUpdate``: the last time when the internal state of this + source was changed, either by an NRTM import or export, full import or export, + recording an error, RPKI status change, or user submitted changes. * ``synchronisedSerials``: whether or not a mirrored source is running with :ref:`synchronised serials `. diff --git a/irrd/mirroring/nrtm4/nrtm4_client.py b/irrd/mirroring/nrtm4/nrtm4_client.py index e2de00938..fb23dede0 100644 --- a/irrd/mirroring/nrtm4/nrtm4_client.py +++ b/irrd/mirroring/nrtm4/nrtm4_client.py @@ -94,15 +94,17 @@ def _run_client(self) -> bool: f" {self.last_status.current_key} to {used_key}" ) - self.database_handler.record_nrtm4_client_status( - self.source, - NRTM4ClientDatabaseStatus( - session_id=unf.session_id, - version=unf.version, - current_key=used_key, - next_key=unf.next_signing_key, - ), + new_status = NRTM4ClientDatabaseStatus( + session_id=unf.session_id, + version=unf.version, + current_key=used_key, + next_key=unf.next_signing_key, ) + if self.last_status != new_status: + self.database_handler.record_nrtm4_client_status( + self.source, + new_status, + ) return has_loaded_snapshot def _retrieve_unf(self) -> Tuple[NRTM4UpdateNotificationFile, Optional[str]]: diff --git a/irrd/mirroring/nrtm4/nrtm4_server.py b/irrd/mirroring/nrtm4/nrtm4_server.py index 91ee4cd70..e8acf580b 100644 --- a/irrd/mirroring/nrtm4/nrtm4_server.py +++ b/irrd/mirroring/nrtm4/nrtm4_server.py @@ -1,3 +1,4 @@ +import copy import datetime import gzip import logging @@ -84,6 +85,7 @@ def __init__( def _update_status(self): self.status = None + self.original_status = None try: database_status = next( self.database_handler.execute_query(DatabaseStatusQuery().source(self.source)) @@ -95,6 +97,7 @@ def _update_status(self): return self.force_reload = database_status["force_reload"] self.status = NRTM4ServerDatabaseStatus.from_dict(database_status) + self.original_status = copy.deepcopy(self.status) def run(self): status_lockfile = get_lockfile(self.status_lockfile_path, blocking=False) @@ -210,7 +213,8 @@ def _commit_status(self) -> None: self._expire_deltas() self._write_unf() - self.database_handler.record_nrtm4_server_status(self.source, self.status) + if self.status != self.original_status: + self.database_handler.record_nrtm4_server_status(self.source, self.status) self._expire_snapshots() self.database_handler.commit() diff --git a/irrd/server/graphql/schema_generator.py b/irrd/server/graphql/schema_generator.py index ebf29f048..41d39b431 100644 --- a/irrd/server/graphql/schema_generator.py +++ b/irrd/server/graphql/schema_generator.py @@ -83,6 +83,7 @@ def __init__(self): nrtm4ServerVersion: Int nrtm4ServerLastUpdateNotificationFileUpdate: String nrtm4ServerLastSnapshotVersion: Int + rpslDataUpdated: String lastUpdate: String synchronisedSerials: Boolean! aliased_sources: [String!] diff --git a/irrd/server/graphql/tests/test_schema_generator.py b/irrd/server/graphql/tests/test_schema_generator.py index fbf05e105..6ec0cb8b2 100644 --- a/irrd/server/graphql/tests/test_schema_generator.py +++ b/irrd/server/graphql/tests/test_schema_generator.py @@ -59,6 +59,7 @@ def test_schema_generator(): nrtm4ServerVersion: Int nrtm4ServerLastUpdateNotificationFileUpdate: String nrtm4ServerLastSnapshotVersion: Int + rpslDataUpdated: String lastUpdate: String synchronisedSerials: Boolean! aliased_sources: [String!] diff --git a/irrd/server/http/metrics_generator.py b/irrd/server/http/metrics_generator.py index 241ab132a..f424589d3 100644 --- a/irrd/server/http/metrics_generator.py +++ b/irrd/server/http/metrics_generator.py @@ -30,6 +30,7 @@ def generate(self) -> str: results = [ self._generate_header(), self._generate_object_counts(statistics), + self._generate_rpsl_data_updated(status), self._generate_updated(status), self._generate_last_error(status), self._generate_field( @@ -91,13 +92,47 @@ def _generate_object_counts(self, statistics: Iterable[Dict[str, Any]]) -> str: # TYPE irrd_object_class_total gauge """).lstrip() + "\n".join(lines) + "\n" + def _generate_rpsl_data_updated(self, status: Iterable[Dict[str, Any]]) -> str: + """ + Generate statistics about the time since last update + """ + now = datetime.datetime.now(tz=datetime.timezone.utc) + lines = [ + "# HELP irrd_last_rpsl_data_update_seconds Seconds since the last update to RPSL data", + "# TYPE irrd_last_rpsl_data_update_seconds gauge", + ] + for stat in status: + if stat.get("rpsl_data_updated"): + diff = now - stat["rpsl_data_updated"] + lines.append( + f"""irrd_last_rpsl_data_update_seconds{{source="{stat['source']}"}} {int(diff.total_seconds())}""" + ) + + lines += [ + "", + ( + "# HELP irrd_last_rpsl_data_update_timestamp Timestamp of the last update to RPSL data in" + " seconds since UNIX epoch" + ), + "# TYPE irrd_last_rpsl_data_update_timestamp gauge", + ] + + for stat in status: + if stat.get("rpsl_data_updated"): + lines.append( + f"""irrd_last_rpsl_data_update_timestamp{{source="{stat['source']}"}} """ + f"""{int(stat['rpsl_data_updated'].timestamp())}""" + ) + + return "\n".join(lines) + "\n" + def _generate_updated(self, status: Iterable[Dict[str, Any]]) -> str: """ Generate statistics about the time since last update """ now = datetime.datetime.now(tz=datetime.timezone.utc) lines = [ - "# HELP irrd_last_update_seconds Seconds since the last update", + "# HELP irrd_last_update_seconds Seconds since the last internal status change", "# TYPE irrd_last_update_seconds gauge", ] for stat in status: @@ -109,7 +144,10 @@ def _generate_updated(self, status: Iterable[Dict[str, Any]]) -> str: lines += [ "", - "# HELP irrd_last_update_timestamp Timestamp of the last update in seconds since UNIX epoch", + ( + "# HELP irrd_last_update_timestamp Timestamp of the last internal status change in seconds" + " since UNIX epoch" + ), "# TYPE irrd_last_update_timestamp gauge", ] diff --git a/irrd/server/http/status_generator.py b/irrd/server/http/status_generator.py index 46efaf76a..e7b411260 100644 --- a/irrd/server/http/status_generator.py +++ b/irrd/server/http/status_generator.py @@ -142,7 +142,8 @@ def _generate_source_detail(self, database_handler: DatabaseHandler) -> str: NRTMv4 server: last snapshot version: {status_result['nrtm4_server_last_snapshot_version']} NRTMv4 server: number of deltas: {len(status_result['nrtm4_server_previous_deltas'] or [])} Synchronised NRTM serials: {synchronised_serials_str} - Last update: {status_result['updated']} + Last change to RPSL data: {status_result['rpsl_data_updated']} + Last internal status update: {status_result['updated']} Local journal kept: {keep_journal} Last import error occurred at: {status_result['last_error_timestamp']} RPKI validation enabled: {rpki_enabled_str} diff --git a/irrd/server/http/tests/test_metrics_generator.py b/irrd/server/http/tests/test_metrics_generator.py index b8c947070..9439ba80b 100644 --- a/irrd/server/http/tests/test_metrics_generator.py +++ b/irrd/server/http/tests/test_metrics_generator.py @@ -49,6 +49,7 @@ def test_request(self, monkeypatch): "nrtm4_client_session_id": None, "nrtm4_client_version": None, "last_error_timestamp": datetime.fromtimestamp(10, UTC), + "rpsl_data_updated": datetime.fromtimestamp(17, UTC), "updated": datetime.fromtimestamp(18, UTC), }, { @@ -62,6 +63,7 @@ def test_request(self, monkeypatch): "nrtm4_client_session_id": nrtm4_client_session_id, "nrtm4_client_version": 14, "last_error_timestamp": None, + "rpsl_data_updated": datetime.fromtimestamp(14, UTC), "updated": datetime.fromtimestamp(15, UTC), }, ], @@ -93,12 +95,22 @@ def test_request(self, monkeypatch): irrd_object_class_total{source="TEST1", object_class="route"} 10 irrd_object_class_total{source="TEST2", object_class="route"} 42 - # HELP irrd_last_update_seconds Seconds since the last update + # HELP irrd_last_rpsl_data_update_seconds Seconds since the last update to RPSL data + # TYPE irrd_last_rpsl_data_update_seconds gauge + irrd_last_rpsl_data_update_seconds{source="TEST1"} 33 + irrd_last_rpsl_data_update_seconds{source="TEST2"} 36 + + # HELP irrd_last_rpsl_data_update_timestamp Timestamp of the last update to RPSL data in seconds since UNIX epoch + # TYPE irrd_last_rpsl_data_update_timestamp gauge + irrd_last_rpsl_data_update_timestamp{source="TEST1"} 17 + irrd_last_rpsl_data_update_timestamp{source="TEST2"} 14 + + # HELP irrd_last_update_seconds Seconds since the last internal status change # TYPE irrd_last_update_seconds gauge irrd_last_update_seconds{source="TEST1"} 32 irrd_last_update_seconds{source="TEST2"} 35 - # HELP irrd_last_update_timestamp Timestamp of the last update in seconds since UNIX epoch + # HELP irrd_last_update_timestamp Timestamp of the last internal status change in seconds since UNIX epoch # TYPE irrd_last_update_timestamp gauge irrd_last_update_timestamp{source="TEST1"} 18 irrd_last_update_timestamp{source="TEST2"} 15 diff --git a/irrd/server/http/tests/test_status_generator.py b/irrd/server/http/tests/test_status_generator.py index 2afbf800a..f977a33cb 100644 --- a/irrd/server/http/tests/test_status_generator.py +++ b/irrd/server/http/tests/test_status_generator.py @@ -106,6 +106,7 @@ def mock_whois_query(nrtm_host, nrtm_port, source): "nrtm4_server_last_snapshot_version": 21, "nrtm4_server_previous_deltas": ["d1", "d2"], "last_error_timestamp": datetime(2018, 1, 1, tzinfo=timezone.utc), + "rpsl_data_updated": datetime(2017, 6, 1, tzinfo=timezone.utc), "updated": datetime(2018, 6, 1, tzinfo=timezone.utc), }, { @@ -124,6 +125,7 @@ def mock_whois_query(nrtm_host, nrtm_port, source): "nrtm4_server_last_snapshot_version": None, "nrtm4_server_previous_deltas": None, "last_error_timestamp": datetime(2019, 1, 1, tzinfo=timezone.utc), + "rpsl_data_updated": datetime(2018, 6, 1, tzinfo=timezone.utc), "updated": datetime(2019, 6, 1, tzinfo=timezone.utc), }, { @@ -142,6 +144,7 @@ def mock_whois_query(nrtm_host, nrtm_port, source): "nrtm4_server_last_snapshot_version": None, "nrtm4_server_previous_deltas": None, "last_error_timestamp": None, + "rpsl_data_updated": None, "updated": None, }, { @@ -160,6 +163,7 @@ def mock_whois_query(nrtm_host, nrtm_port, source): "nrtm4_server_last_snapshot_version": None, "nrtm4_server_previous_deltas": None, "last_error_timestamp": None, + "rpsl_data_updated": None, "updated": None, }, { @@ -178,6 +182,7 @@ def mock_whois_query(nrtm_host, nrtm_port, source): "nrtm4_server_last_snapshot_version": None, "nrtm4_server_previous_deltas": None, "last_error_timestamp": None, + "rpsl_data_updated": None, "updated": None, }, ], @@ -221,7 +226,8 @@ def mock_whois_query(nrtm_host, nrtm_port, source): NRTMv4 server: last snapshot version: 21 NRTMv4 server: number of deltas: 2 Synchronised NRTM serials: No - Last update: 2018-06-01 00:00:00+00:00 + Last change to RPSL data: 2017-06-01 00:00:00+00:00 + Last internal status update: 2018-06-01 00:00:00+00:00 Local journal kept: Yes Last import error occurred at: 2018-01-01 00:00:00+00:00 RPKI validation enabled: No @@ -255,7 +261,8 @@ def mock_whois_query(nrtm_host, nrtm_port, source): NRTMv4 server: last snapshot version: None NRTMv4 server: number of deltas: 0 Synchronised NRTM serials: No - Last update: 2019-06-01 00:00:00+00:00 + Last change to RPSL data: 2018-06-01 00:00:00+00:00 + Last internal status update: 2019-06-01 00:00:00+00:00 Local journal kept: No Last import error occurred at: 2019-01-01 00:00:00+00:00 RPKI validation enabled: Yes @@ -286,7 +293,8 @@ def mock_whois_query(nrtm_host, nrtm_port, source): NRTMv4 server: last snapshot version: None NRTMv4 server: number of deltas: 0 Synchronised NRTM serials: No - Last update: None + Last change to RPSL data: None + Last internal status update: None Local journal kept: No Last import error occurred at: None RPKI validation enabled: Yes @@ -317,7 +325,8 @@ def mock_whois_query(nrtm_host, nrtm_port, source): NRTMv4 server: last snapshot version: None NRTMv4 server: number of deltas: 0 Synchronised NRTM serials: No - Last update: None + Last change to RPSL data: None + Last internal status update: None Local journal kept: No Last import error occurred at: None RPKI validation enabled: Yes @@ -347,7 +356,8 @@ def mock_whois_query(nrtm_host, nrtm_port, source): NRTMv4 server: last snapshot version: None NRTMv4 server: number of deltas: 0 Synchronised NRTM serials: No - Last update: None + Last change to RPSL data: None + Last internal status update: None Local journal kept: No Last import error occurred at: None RPKI validation enabled: Yes diff --git a/irrd/server/query_resolver.py b/irrd/server/query_resolver.py index 8637f3b87..787f66c29 100644 --- a/irrd/server/query_resolver.py +++ b/irrd/server/query_resolver.py @@ -405,6 +405,9 @@ def database_status( results[source]["nrtm4_server_last_snapshot_version"] = query_result[ "nrtm4_server_last_snapshot_version" ] + results[source]["rpsl_data_updated"] = ( + query_result["rpsl_data_updated"].astimezone(timezone("UTC")).isoformat() + ) results[source]["last_update"] = query_result["updated"].astimezone(timezone("UTC")).isoformat() results[source]["synchronised_serials"] = is_serial_synchronised(self.database_handler, source) diff --git a/irrd/server/tests/test_query_resolver.py b/irrd/server/tests/test_query_resolver.py index e30ca6ed3..d652c3019 100644 --- a/irrd/server/tests/test_query_resolver.py +++ b/irrd/server/tests/test_query_resolver.py @@ -589,6 +589,7 @@ def test_database_status(self, monkeypatch, prepare_resolver, config_override): 2020, 1, 1, tzinfo=datetime.timezone.utc ), "nrtm4_server_last_snapshot_version": 21, + "rpsl_data_updated": datetime.datetime(2019, 1, 1, tzinfo=timezone("UTC")), "updated": datetime.datetime(2020, 1, 1, tzinfo=timezone("UTC")), }, { @@ -603,6 +604,7 @@ def test_database_status(self, monkeypatch, prepare_resolver, config_override): "nrtm4_server_version": None, "nrtm4_server_last_update_notification_file_update": None, "nrtm4_server_last_snapshot_version": None, + "rpsl_data_updated": datetime.datetime(2019, 1, 1, tzinfo=timezone("UTC")), "updated": datetime.datetime(2020, 1, 1, tzinfo=timezone("UTC")), }, ] @@ -630,6 +632,7 @@ def test_database_status(self, monkeypatch, prepare_resolver, config_override): ("nrtm4_server_version", 22), ("nrtm4_server_last_update_notification_file_update", "2020-01-01T00:00:00+00:00"), ("nrtm4_server_last_snapshot_version", 21), + ("rpsl_data_updated", "2019-01-01T00:00:00+00:00"), ("last_update", "2020-01-01T00:00:00+00:00"), ("synchronised_serials", False), ] @@ -659,6 +662,7 @@ def test_database_status(self, monkeypatch, prepare_resolver, config_override): ("nrtm4_server_version", None), ("nrtm4_server_last_update_notification_file_update", None), ("nrtm4_server_last_snapshot_version", None), + ("rpsl_data_updated", "2019-01-01T00:00:00+00:00"), ("last_update", "2020-01-01T00:00:00+00:00"), ("synchronised_serials", False), ] diff --git a/irrd/storage/alembic/versions/a635d2217a48_add_rpsl_data_updated_field_to_database_.py b/irrd/storage/alembic/versions/a635d2217a48_add_rpsl_data_updated_field_to_database_.py new file mode 100644 index 000000000..77c6c5892 --- /dev/null +++ b/irrd/storage/alembic/versions/a635d2217a48_add_rpsl_data_updated_field_to_database_.py @@ -0,0 +1,28 @@ +"""Add rpsl_data_updated field to database_status + +Revision ID: a635d2217a48 +Revises: b7b3c367f9ba +Create Date: 2024-11-08 16:11:21.101990 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "a635d2217a48" +down_revision = "b7b3c367f9ba" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "database_status", + sa.Column( + "rpsl_data_updated", sa.DateTime(timezone=True), server_default=sa.text("now()"), nullable=False + ), + ) + + +def downgrade(): + op.drop_column("database_status", "rpsl_data_updated") diff --git a/irrd/storage/database_handler.py b/irrd/storage/database_handler.py index aa244c4d9..cfef2050a 100644 --- a/irrd/storage/database_handler.py +++ b/irrd/storage/database_handler.py @@ -877,6 +877,7 @@ def record_serial_exported(self, source: str, serial: int) -> None: def record_nrtm4_client_status(self, source: str, status: NRTM4ClientDatabaseStatus) -> None: """ Record the status of NRTMv4 mirroring for clients. + Only call this if the data has actually changed, as it updates rpsl_data_updated. """ self._check_write_permitted() self.status_tracker.record_nrtm4_client_status(source, status) @@ -939,6 +940,7 @@ class DatabaseStatusTracker: journaling_enabled: bool _new_serials_per_source: Dict[str, Set[int]] _sources_seen: Set[str] + _sources_rpsl_data_updated: Set[str] _newest_mirror_serials: Dict[str, int] _mirroring_error: Dict[str, str] _exported_serials: Dict[str, int] @@ -1031,6 +1033,7 @@ def record_operation( gapless set of NRTM serials. """ self._sources_seen.add(source) + self._sources_rpsl_data_updated.add(source) if self.journaling_enabled and get_setting(f"sources.{source}.keep_journal"): serial_nrtm: Union[int, sa.sql.expression.Select] journal_tablename = RPSLDatabaseJournal.__tablename__ @@ -1092,7 +1095,7 @@ def finalise_transaction(self): source=source, force_reload=False, synchronised_serials=self._is_serial_synchronised(source), - updated=datetime.now(timezone.utc), + rpsl_data_updated=datetime.now(timezone.utc), ) stmt = stmt.on_conflict_do_update( index_elements=["source"], @@ -1103,6 +1106,16 @@ def finalise_transaction(self): ) self.database_handler.execute_statement(stmt) + for source in self._sources_rpsl_data_updated: + stmt = ( + RPSLDatabaseStatus.__table__.update() + .where(self.c_status.source == source) + .values( + rpsl_data_updated=datetime.now(timezone.utc), + ) + ) + self.database_handler.execute_statement(stmt) + for source, serials in self._new_serials_per_source.items(): serial_oldest_journal_q = sa.select([sa.func.min(self.c_journal.serial_nrtm)]).where( self.c_journal.source == source @@ -1139,7 +1152,7 @@ def finalise_transaction(self): serial_newest_seen=serial_newest_seen, serial_oldest_journal=serial_oldest_journal, serial_newest_journal=serial_newest_journal, - updated=datetime.now(timezone.utc), + rpsl_data_updated=datetime.now(timezone.utc), ) ) self.database_handler.execute_statement(stmt) @@ -1151,7 +1164,6 @@ def finalise_transaction(self): .values( last_error=error, last_error_timestamp=datetime.now(timezone.utc), - updated=datetime.now(timezone.utc), ) ) self.database_handler.execute_statement(stmt) @@ -1162,7 +1174,7 @@ def finalise_transaction(self): .where(self.c_status.source == source) .values( serial_newest_mirror=serial, - updated=datetime.now(timezone.utc), + rpsl_data_updated=datetime.now(timezone.utc), ) ) self.database_handler.execute_statement(stmt) @@ -1186,6 +1198,7 @@ def finalise_transaction(self): nrtm4_client_version=status.version, nrtm4_client_current_key=status.current_key, nrtm4_client_next_key=status.next_key, + rpsl_data_updated=datetime.now(timezone.utc), ) ) self.database_handler.execute_statement(stmt) @@ -1229,6 +1242,7 @@ def reset(self): self._journal_table_locked = False self._new_serials_per_source = defaultdict(set) self._sources_seen = set() + self._sources_rpsl_data_updated = set() self._newest_mirror_serials = dict() self._mirroring_error = dict() self._exported_serials = dict() diff --git a/irrd/storage/models.py b/irrd/storage/models.py index 97a6f5f3b..10e22e637 100644 --- a/irrd/storage/models.py +++ b/irrd/storage/models.py @@ -302,6 +302,8 @@ class RPSLDatabaseStatus(Base): # type: ignore last_error = sa.Column(sa.Text) last_error_timestamp = sa.Column(sa.DateTime(timezone=True)) + rpsl_data_updated = sa.Column(sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False) + created = sa.Column(sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False) updated = sa.Column( sa.DateTime(timezone=True), server_default=sa.func.now(), onupdate=sa.func.now(), nullable=False diff --git a/irrd/storage/queries.py b/irrd/storage/queries.py index f1837d26b..334c2758c 100644 --- a/irrd/storage/queries.py +++ b/irrd/storage/queries.py @@ -575,6 +575,7 @@ def __init__(self): self.columns.synchronised_serials, self.columns.last_error, self.columns.last_error_timestamp, + self.columns.rpsl_data_updated, self.columns.created, self.columns.updated, ] diff --git a/irrd/storage/tests/test_database.py b/irrd/storage/tests/test_database.py index 39c8f141e..552abe81d 100644 --- a/irrd/storage/tests/test_database.py +++ b/irrd/storage/tests/test_database.py @@ -1021,7 +1021,14 @@ def test_route_preference_status_storage( assert len(list(dh.execute_query(RPSLDatabaseJournalQuery()))) == 2 # no new entry since last test def _clean_result(self, results): - variable_fields = ["pk", "timestamp", "created", "updated", "last_error_timestamp"] + variable_fields = [ + "pk", + "timestamp", + "created", + "updated", + "last_error_timestamp", + "rpsl_data_updated", + ] return [{k: v for k, v in result.items() if k not in variable_fields} for result in list(results)] def test_suspension(