Add config for returning dummy object in the NRTMv3 response

docs/admins/configuration.rst

@@ -717,7 +717,7 @@ Sources
 * ``sources.{name}.export_destination_unfiltered``: a path to save full exports,
   including a serial file, of this source. This is identical to
   ``export_destination``, except that the files saved here contain full unfiltered
-  password hashes from mntner objects.
+  password hashes from mntner objects and objects are never dummified.
   Sharing password hashes externally is a security risk, the unfiltered data
   is intended only to support
   :doc:`availability and data migration </admins/availability-and-migration>`.
@@ -741,7 +741,7 @@ Sources
 * ``sources.{name}.nrtm_access_list_unfiltered``: a reference to an access list
   in the configuration, where IPs in the access list are permitted unfiltered
   access to the NRTMv3 stream for this particular source (``-g`` queries).
-  Unfiltered means full password hashes are included.
+  Unfiltered means full password hashes are included and objects are never dummified.
   Sharing password hashes externally is a security risk, the unfiltered data
   is intended only to support
   :doc:`availability and data migration </admins/availability-and-migration>`.
@@ -775,6 +775,28 @@ Sources
   use the `|` style to preserve newlines.
   |br| **Default**: not defined, no additional header added.
   |br| **Change takes effect**: after SIGHUP, upon next request.
+* ``sources.{name}.nrtm_dummified_object_classes``: a list of object classes
+  that will be dummified in exports and NRTM, i.e. any values
+  of attributes listed in ``nrtm_dummified_attributes`` are removed.
+  This affects exports, NRTMv3 responses, and NRTMv4.
+  |br| **Default**: not defined, no objects dummified.
+  |br| **Change takes effect**: after SIGHUP, upon next request/export.
+* ``sources.{name}.nrtm_dummified_attributes``: a dictionary where keys are names
+  of attributes to dummify, and values are the dummy data.
+  If the attribute value contains ``%s``, IRRD will replace it by the object primary key,
+  e.g. key ``person``, value ``Dummy name for person %s``.
+  This is only applied when the RPSL object class is listed in
+  ``nrtm_dummified_object_classes``.
+  This affects exports, NRTMv3 responses, and NRTMv4.
+  |br| **Default**: not defined. no objects dummified.
+  |br| **Change takes effect**: after SIGHUP, upon next request/export.
+* ``sources.{name}.nrtm_dummified_remarks``: additional remarks to add
+  when an object is dummified, e.g. to indicate this is not the
+  original object.
+  This can have multiple lines. When adding this to the configuration,
+  use the `|` style to preserve newlines.
+  |br| **Default**: not defined, no additional remarks added.
+  |br| **Change takes effect**: after SIGHUP, upon next request.
 * ``sources.{name}.strict_import_keycert_objects``: a setting used when
   migrating authoritative data that may contain `key-cert` objects.
   See the :doc:`data migration guide </admins/availability-and-migration>`

docs/releases/4.5.0.rst

@@ -37,6 +37,17 @@ was added to the HTTP server in IRRD, on ``/metrics/``.
 This page is only accessible to IPs :doc:`configured </admins/configuration>`
 in the access list set in the ``server.http.status_access_list`` setting.
 
+RPSL object text dummification for exports and NRTM
+---------------------------------------------------
+IRRD can now dummify RPSL object text in exports and NRTM, to remove
+sensitive or personal information. This is controlled by the new
+``sources.{name}.nrtm_dummified_object_classes`` and
+``sources.{name}.nrtm_dummified_attributes`` settings.
+
+Dummification applies to exports, NRTMv3 responses, and NRTMv4. It does
+not apply to interactive query responses like whois or GraphQL.
+By default, dummification is disabled.
+
 New "RPSL data updated" status timestamp
 ----------------------------------------
 Various status overviews of IRRD would show a "last update" per source.

docs/spelling_wordlist.txt

@@ -107,3 +107,6 @@ validator
 validators
 virtualenv
 whois
+dummified
+dummify
+dummification

irrd/conf/known_keys.py

@@ -101,6 +101,9 @@
     "nrtm_query_serial_days_limit",
     "nrtm_query_serial_range_limit",
     "nrtm_response_header",
+    "nrtm_dummified_attributes",
+    "nrtm_dummified_object_classes",
+    "nrtm_dummified_remarks",
     "nrtm4_client_notification_file_url",
     "nrtm4_client_initial_public_key",
     "nrtm4_server_private_key",

irrd/conf/test_conf.py

@@ -108,6 +108,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp
                         "suspension_enabled": True,
                         "nrtm_query_serial_range_limit": 10,
                         "object_class_filter": "route",
+                        "nrtm_dummified_object_classes": "person",
                     },
                     "TESTDB2": {
                         "nrtm_host": "192.0.2.1",
@@ -116,6 +117,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp
                         "keep_journal": True,
                         "route_object_preference": 200,
                         "object_class_filter": ["ROUTE"],
+                        "nrtm_dummified_object_classes": ["PERSON"],
                     },
                     "TESTDB3": {
                         "export_destination_unfiltered": "/tmp",

irrd/mirroring/mirror_runners_export.py

@@ -8,6 +8,7 @@
 from irrd.conf import get_setting
 from irrd.storage.database_handler import DatabaseHandler
 from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseQuery
+from irrd.utils.text import dummify_object_text as dummify_object_text_func
 from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func
 
 EXPORT_PERMISSIONS = 0o644
@@ -46,7 +47,7 @@ def run(self) -> None:
                     f"Starting an unfiltered source export for {self.source} "
                     f"to {export_destination_unfiltered}"
                 )
-                self._export(export_destination_unfiltered, remove_auth_hashes=False)
+                self._export(export_destination_unfiltered, remove_auth_hashes=False, dummify_object=False)
 
             self.database_handler.commit()
         except Exception as exc:
@@ -57,7 +58,7 @@ def run(self) -> None:
         finally:
             self.database_handler.close()
 
-    def _export(self, export_destination, remove_auth_hashes=True):
+    def _export(self, export_destination, remove_auth_hashes=True, dummify_object=True):
         filename_export = Path(export_destination) / f"{self.source.lower()}.db.gz"
         export_tmpfile = NamedTemporaryFile(delete=False)
         filename_serial = Path(export_destination) / f"{self.source.upper()}.CURRENTSERIAL"
@@ -75,6 +76,11 @@ def _export(self, export_destination, remove_auth_hashes=True):
                 object_text = obj["object_text"]
                 if remove_auth_hashes:
                     object_text = remove_auth_hashes_func(object_text)
+
+                if dummify_object:
+                    object_text = dummify_object_text_func(
+                        object_text, obj["object_class"], self.source, obj["rpsl_pk"]
+                    )
                 object_bytes = object_text.encode("utf-8")
                 fh.write(object_bytes + b"\n")
             fh.write(b"# EOF\n")

irrd/mirroring/nrtm4/nrtm4_server.py

@@ -24,7 +24,7 @@
     RPSLDatabaseQuery,
 )
 from irrd.utils.crypto import eckey_from_config, eckey_public_key_as_str, jws_serialize
-from irrd.utils.text import remove_auth_hashes
+from irrd.utils.text import dummify_object_text, remove_auth_hashes
 
 from ...utils.process_support import get_lockfile
 from ..retrieval import file_hash_sha256
@@ -318,7 +318,17 @@ def _write_snapshot(self, version: int) -> str:
                 header.model_dump(mode="json", include=header.model_fields_set),
                 outstream,
             )
-            objs_filtered = ({"object": remove_auth_hashes(obj["object_text"])} for obj in objs)
+            objs_filtered = (
+                {
+                    "object": dummify_object_text(
+                        remove_auth_hashes(obj["object_text"]),
+                        obj["object_class"],
+                        self.source,
+                        obj["rpsl_pk"],
+                    )
+                }
+                for obj in objs
+            )
             jsonseq_encode(objs_filtered, outstream)
         return filename
 
@@ -351,9 +361,13 @@ def _write_delta(self, version: int, serial_global_start: int) -> Optional[str]:
             )
             for journal_entry in journal_entries:
                 if journal_entry["operation"] == DatabaseOperation.add_or_update:
+                    object_text = remove_auth_hashes(journal_entry["object_text"])
+                    object_text = dummify_object_text(
+                        object_text, journal_entry["object_class"], self.source, journal_entry["rpsl_pk"]
+                    )
                     entry_encoded = {
                         "action": "add_modify",
-                        "object": remove_auth_hashes(journal_entry["object_text"]),
+                        "object": object_text,
                     }
                 elif journal_entry["operation"] == DatabaseOperation.delete:
                     entry_encoded = {

irrd/mirroring/nrtm4/tests/test_nrtm4_server.py

@@ -22,7 +22,7 @@
 from irrd.utils.crypto import jws_deserialize
 from irrd.utils.rpsl_samples import SAMPLE_MNTNER
 from irrd.utils.test_utils import MockDatabaseHandler
-from irrd.utils.text import remove_auth_hashes
+from irrd.utils.text import dummify_object_text, remove_auth_hashes
 
 
 class TestNRTM4Server:
@@ -64,6 +64,12 @@ def test_nrtm4_server(self, tmpdir, config_override):
                         "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR,
                         "nrtm4_server_local_path": str(nrtm_path),
                         # "nrtm4_server_snapshot_frequency": 0,
+                        "nrtm_dummified_object_classes": "mntner",
+                        "nrtm_dummified_attributes": {
+                            "descr": "Dummy description for %s",
+                            "upd-to": "unread@ripe.net",
+                        },
+                        "nrtm_dummified_remarks": "Invalid object",
                     }
                 },
             }
@@ -107,7 +113,9 @@ def test_nrtm4_server(self, tmpdir, config_override):
         assert snapshot[0]["type"] == "snapshot"
         assert snapshot[0]["session_id"] == unf["session_id"]
         assert snapshot[0]["version"] == unf["snapshot"]["version"]
-        assert snapshot[1]["object"] == remove_auth_hashes(SAMPLE_MNTNER)
+        assert snapshot[1]["object"] == dummify_object_text(
+            remove_auth_hashes(SAMPLE_MNTNER), "mntner", "TEST", "TEST-MNT"
+        )
         assert PASSWORD_HASH_DUMMY_VALUE in snapshot[1]["object"]
 
         assert len(mock_dh.other_calls) == 2
@@ -131,6 +139,8 @@ def test_nrtm4_server(self, tmpdir, config_override):
                 {
                     "operation": DatabaseOperation.add_or_update,
                     "object_text": SAMPLE_MNTNER,
+                    "object_class": "mntner",
+                    "rpsl_pk": "TEST-MNT",
                 }
             ],
         )
@@ -156,7 +166,9 @@ def test_nrtm4_server(self, tmpdir, config_override):
         assert delta[0]["type"] == "delta"
         assert delta[0]["session_id"] == unf["session_id"]
         assert delta[0]["version"] == new_unf["version"]
-        assert delta[1]["object"] == remove_auth_hashes(SAMPLE_MNTNER)
+        assert delta[1]["object"] == dummify_object_text(
+            remove_auth_hashes(SAMPLE_MNTNER), "mntner", "TEST", "TEST-MNT"
+        )
         assert delta[1]["action"] == "add_modify"
         assert PASSWORD_HASH_DUMMY_VALUE in delta[1]["object"]
 
@@ -268,6 +280,8 @@ def _run_writer(self, mock_dh, status, journal=None):
             [
                 {
                     "object_text": SAMPLE_MNTNER,
+                    "object_class": "mntner",
+                    "rpsl_pk": "TEST-MNT",
                 }
             ]
         )

irrd/mirroring/nrtm_generator.py

@@ -5,6 +5,7 @@
 from irrd.conf import get_setting
 from irrd.storage.database_handler import DatabaseHandler
 from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery
+from irrd.utils.text import dummify_object_text as dummify_object_text_func
 from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func
 
 
@@ -21,6 +22,7 @@ def generate(
         serial_end_requested: Optional[int],
         database_handler: DatabaseHandler,
         remove_auth_hashes=True,
+        client_is_dummifying_exempt=False,
     ) -> str:
         """
         Generate an NRTM response for a particular source, serial range and
@@ -119,8 +121,13 @@ def generate(
             if version == "3":
                 operation_str += " " + str(operation["serial_nrtm"])
             text = operation["object_text"]
+
             if remove_auth_hashes:
                 text = remove_auth_hashes_func(text)
+
+            if not client_is_dummifying_exempt:
+                text = dummify_object_text_func(text, operation["object_class"], source, operation["rpsl_pk"])
+
             operation_str += "\n\n" + text
             output.append(operation_str)
 

irrd/mirroring/tests/test_mirror_runners_export.py

@@ -16,6 +16,10 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog):
                 "sources": {
                     "TEST": {
                         "export_destination": str(tmpdir),
+                        "nrtm_dummified_object_classes": "mntner",
+                        "nrtm_dummified_attributes": {
+                            "descr": "Dummy description for %s",
+                        },
                     }
                 }
             }
@@ -34,8 +38,12 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog):
                 repeat({"serial_newest_seen": "424242"}),
                 [
                     # The CRYPT-PW hash must not appear in the output
-                    {"object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n"},
-                    {"object_text": "object 2 🌈\n"},
+                    {
+                        "object_text": "object 1 🦄\ndescr: description\nauth: CRYPT-PW foobar\n",
+                        "object_class": "mntner",
+                        "rpsl_pk": "TEST-MNT",
+                    },
+                    {"object_text": "object 2 🌈\n", "object_class": "person", "rpsl_pk": "PERSON-TEST"},
                 ],
             ]
         )
@@ -55,7 +63,8 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog):
         with gzip.open(export_filename) as fh:
             assert (
                 fh.read().decode("utf-8")
-                == "object 1 🦄\nauth: CRYPT-PW DummyValue  # Filtered for security\n\nobject 2 🌈\n\n# EOF\n"
+                == "object 1 🦄\ndescr:          Dummy description for TEST-MNT\nauth: CRYPT-PW DummyValue  #"
+                " Filtered for security\n\nobject 2 🌈\n\n# EOF\n"
             )
 
         assert flatten_mock_calls(mock_dh) == [
@@ -99,8 +108,12 @@ def test_export_unfiltered(self, tmpdir, config_override, monkeypatch, caplog):
                 repeat({"serial_newest_seen": "424242"}),
                 [
                     # The CRYPT-PW hash should appear in the output
-                    {"object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n"},
-                    {"object_text": "object 2 🌈\n"},
+                    {
+                        "object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n",
+                        "object_class": "mntner",
+                        "rpsl_pk": "TEST-MNT",
+                    },
+                    {"object_text": "object 2 🌈\n", "object_class": "person", "rpsl_pk": "PERSON-TEST"},
                 ],
             ]
         )
@@ -166,8 +179,12 @@ def test_export_no_serial(self, tmpdir, config_override, monkeypatch, caplog):
                 iter([]),
                 [
                     # The CRYPT-PW hash must not appear in the output
-                    {"object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n"},
-                    {"object_text": "object 2 🌈\n"},
+                    {
+                        "object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n",
+                        "object_class": "mntner",
+                        "rpsl_pk": "TEST-MNT",
+                    },
+                    {"object_text": "object 2 🌈\n", "object_class": "person", "rpsl_pk": "PERSON-TEST"},
                 ],
             ]
         )