Fix #233 - Add irrd_mirror_force_reload command. (#235)

docs/users/mirroring.rst

@@ -123,8 +123,23 @@ changes in the journal with the same serial as the original source provided.
 
 Updates will be retrieved every `import_timer`, and IRRd will automatically
 perform a full import the first time, and then use NRTM for updates.
-A new full import can be forced by setting `force_reload` in the SQL database,
-which will also discard the entire local journal.
+
+Even in sources that normally use NRTM, IRRd can run a full new import of the
+database. This may be needed if the NRTM stream has gotten so far behind that
+the updates IRRd needs are no longer available. To start a full reload,
+use the ``irrd_mirror_force_reload`` command. For example, to force a full
+reload for the ``MIRROR-EXAMPLE`` source::
+
+    irrd_mirror_force_reload --config /etc/irrd.yaml MIRROR-EXAMPLE
+
+The config parameter is optional. The reload will start the next time
+`import_timer` expires. After the reload, IRRd will resume mirroring from
+the NRTM stream.
+
+Note that any instances mirroring from your instance (i.e. your IRRd is
+mirroring a source, a third party mirrors this from your instance), will also
+have to do a full reload, as the journal for NRTM queries is purged when
+doing a full reload.
 
 Periodic full imports
 ~~~~~~~~~~~~~~~~~~~~~
@@ -163,6 +178,13 @@ A third option is to manually load data. This can be useful while testing,
 or when generating data files from scripts, as it provides direct feedback
 on whether loading data was successful.
 
+.. caution::
+    This process is intended for data sources such as produced by scripts.
+    The validation is quite strict, as in script output, an error in script
+    execution is a likely cause for any issues in the data.
+    To force a reload of a regular mirror that normally uses NRTM,
+    use the ``irrd_mirror_force_reload`` command instead.
+
 Manual loading uses the ``irrd_load_database`` command:
 
 * The command can be called, providing a name of a source and a path to

irrd/scripts/mirror_force_reload.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python
+# flake8: noqa: E402
+import argparse
+import logging
+import sys
+
+from pathlib import Path
+
+from irrd.storage.preload import send_reload_signal
+
+"""
+Load an RPSL file into the database.
+"""
+
+logger = logging.getLogger(__name__)
+sys.path.append(str(Path(__file__).resolve().parents[2]))
+
+from irrd.conf import config_init, CONFIG_PATH_DEFAULT
+from irrd.storage.database_handler import DatabaseHandler
+
+
+def set_force_reload(source) -> None:
+    dh = DatabaseHandler(enable_preload_update=False)
+    dh.set_force_reload(source)
+    dh.commit()
+    dh.close()
+
+
+def main():  # pragma: no cover
+    description = """Force a full reload for a mirror."""
+    parser = argparse.ArgumentParser(description=description)
+    parser.add_argument('--config', dest='config_file_path', type=str,
+                        help=f'use a different IRRd config file (default: {CONFIG_PATH_DEFAULT})')
+    parser.add_argument('source', type=str,
+                        help='the name of the source to reload')
+    args = parser.parse_args()
+
+    config_init(args.config_file_path)
+
+    set_force_reload(args.source)
+
+
+if __name__ == '__main__':  # pragma: no cover
+    main()

irrd/scripts/tests/test_mirror_force_reload.py

@@ -0,0 +1,18 @@
+from unittest.mock import Mock
+
+from irrd.utils.test_utils import flatten_mock_calls
+from ..mirror_force_reload import set_force_reload
+
+
+def test_set_force_reload(capsys, monkeypatch):
+    mock_dh = Mock()
+    monkeypatch.setattr('irrd.scripts.mirror_force_reload.DatabaseHandler',
+                        lambda enable_preload_update: mock_dh)
+
+    set_force_reload('TEST')
+    assert flatten_mock_calls(mock_dh) == [
+        ['set_force_reload', ('TEST', ), {}],
+        ['commit', (), {}],
+        ['close', (), {}]
+    ]
+    assert not capsys.readouterr().out

irrd/storage/database_handler.py

@@ -259,6 +259,16 @@ def delete_all_rpsl_objects_with_journal(self, source):
         # All objects are presumed to have been changed.
         self._object_classes_modified.update(OBJECT_CLASS_MAPPING.keys())
 
+    def set_force_reload(self, source):
+        """
+        Set the force_reload flag for a source.
+        Upon the next mirror update, all data for the source will be
+        discarded and reloaded from the source.
+        """
+        table = RPSLDatabaseStatus.__table__
+        stmt = table.update().where(table.c.source == source).values(force_reload=True)
+        self._connection.execute(stmt)
+
     def force_record_serial_seen(self, source: str, serial: int) -> None:
         """
         Forcibly record that a serial was seen for a source.

irrd/storage/tests/test_database.py

@@ -284,13 +284,16 @@ def test_updates_database_status_forced_serials(self, monkeypatch, irrd_database
             asn_last=65537,
         )
         self.dh.upsert_rpsl_object(rpsl_object_route_v6)
+        self.dh.set_force_reload('TEST2')  # Should be ignored, as source is new
+        self.dh.commit()
+        self.dh.set_force_reload('TEST')
         self.dh.commit()
 
         status = self._clean_result(self.dh.execute_query(DatabaseStatusQuery().source('TEST')))
         assert status == [
             {'source': 'TEST', 'serial_oldest_seen': 42, 'serial_newest_seen': 4242,
              'serial_oldest_journal': None, 'serial_newest_journal': None,
-             'serial_last_export': None, 'force_reload': False, 'last_error': None},
+             'serial_last_export': None, 'force_reload': True, 'last_error': None},
         ]
 
         # The insert of rpsl_object_route_v6 had no forced serial and no
@@ -303,7 +306,7 @@ def test_updates_database_status_forced_serials(self, monkeypatch, irrd_database
         ]
 
         self.dh.force_record_serial_seen('TEST', 424242)
-        self.dh.commit()
+        self.dh.commit()  # should also reset force_reload
 
         status = self._clean_result(self.dh.execute_query(DatabaseStatusQuery().source('TEST')))
         assert status == [

setup.py

@@ -42,6 +42,7 @@
             'irrd_submit_email = irrd.scripts.submit_email:main',
             'irrd_database_upgrade = irrd.scripts.database_upgrade:main',
             'irrd_load_database = irrd.scripts.load_database:main',
+            'irrd_mirror_force_reload = irrd.scripts.mirror_force_reload:main',
         ],
     },
     classifiers=(