From 88b3c5cb28788ccd120d16933852693ab02d8c82 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Fri, 15 Mar 2024 10:27:42 +1000 Subject: [PATCH 01/22] Add config for returning dummy object in the NRTMv3 response --- docs/admins/configuration.rst | 24 ++++++++ irrd/conf/__init__.py | 13 +++++ irrd/conf/known_keys.py | 4 +- irrd/conf/test_conf.py | 5 ++ irrd/mirroring/nrtm_generator.py | 28 ++++++++- irrd/mirroring/tests/test_nrtm_generator.py | 64 ++++++++++++++++++--- irrd/utils/text.py | 33 ++++++++++- 7 files changed, 160 insertions(+), 11 deletions(-) diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index ab78950e2..61005d514 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -775,6 +775,30 @@ Sources use the `|` style to preserve newlines. |br| **Default**: not defined, no additional header added. |br| **Change takes effect**: after SIGHUP, upon next request. +* ``sources.{name}.nrtm_response_dummy_object_class``: a list of object classes + that will contain the dummy data within the NRTMv3 responses. + IRRD will dummy an object class only if the ``nrtm_response_dummy_attributes`` + is defined and the object class attribute keys are in the dummy attributes. + |br| **Default**: not defined, no objects dummyfied. + |br| **Change takes effect**: after SIGHUP, upon next request. +* ``sources.{name}.nrtm_response_dummy_attributes``: object attributes that contain + dummy data. This is a dictionary with the attribute keys and corresponding dummy + attribute string values. + The attributes will be replaced only if ``nrtm_response_dummy_object_class`` + is defined and the attributes are in the defined object class. + If the attibute value has the ``%s``, IRRD will replace it by the object primary key, + e.g. ``person: Dummy name for %s``. + + |br| **Default**: not defined. no objects dummyfied. + |br| **Change takes effect**: after SIGHUP, upon next request. +* ``sources.{name}.nrtm_response_dummy_remarks``: an additional remarks to be added + to the end of the dummyfied object in the NRTMv3 response. + IRRD will only add the remarks to the object classes defined in the + ``nrtm_response_dummy_object_class``. + This can have multiple lines. When adding this to the configuration, + use the `|` style to preserve newlines. + |br| **Default**: not defined, no additional remarks added. + |br| **Change takes effect**: after SIGHUP, upon next request. * ``sources.{name}.strict_import_keycert_objects``: a setting used when migrating authoritative data that may contain `key-cert` objects. See the :doc:`data migration guide ` diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index 076e74a6e..0116739dc 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -662,6 +662,19 @@ def get_object_class_filter_for_source(source: str) -> Optional[list[str]]: return None +def get_nrtm_response_dummy_object_class_for_source(source: str) -> Optional[List[str]]: + """ + Helper method to get the cleaned dummy object class in NRTMv3 reponse for a source, if any. + """ + dummy_object_class = get_setting(f"sources.{source}.nrtm_response_dummy_object_class") + if dummy_object_class: + if isinstance(dummy_object_class, str): + dummy_object_class = [dummy_object_class] + return [c.strip().lower() for c in dummy_object_class] + else: + return None + + def sighup_handler(signum, frame) -> None: """ Reload the settings when a SIGHUP is received. diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py index 7621aeaaf..141ee4d68 100644 --- a/irrd/conf/known_keys.py +++ b/irrd/conf/known_keys.py @@ -100,7 +100,9 @@ "nrtm_access_list_unfiltered", "nrtm_query_serial_days_limit", "nrtm_query_serial_range_limit", - "nrtm_response_header", + "nrtm_response_dummy_attributes", + "nrtm_response_dummy_object_class", + "nrtm_response_dummy_remarks", "nrtm4_client_notification_file_url", "nrtm4_client_initial_public_key", "nrtm4_server_private_key", diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index 1609c8d75..d58355fff 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -15,6 +15,7 @@ ConfigurationError, config_init, get_configuration, + get_nrtm_response_dummy_object_class_for_source, get_object_class_filter_for_source, get_setting, is_config_initialised, @@ -108,6 +109,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp "suspension_enabled": True, "nrtm_query_serial_range_limit": 10, "object_class_filter": "route", + "nrtm_response_dummy_object_class": "person", }, "TESTDB2": { "nrtm_host": "192.0.2.1", @@ -116,6 +118,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp "keep_journal": True, "route_object_preference": 200, "object_class_filter": ["ROUTE"], + "nrtm_response_dummy_object_class": ["PERSON"], }, "TESTDB3": { "export_destination_unfiltered": "/tmp", @@ -143,6 +146,8 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp assert is_config_initialised() assert get_object_class_filter_for_source("TESTDB") == ["route"] assert get_object_class_filter_for_source("TESTDB2") == ["route"] + assert get_nrtm_response_dummy_object_class_for_source("TESTDB") == ["person"] + assert get_nrtm_response_dummy_object_class_for_source("TESTDB2") == ["person"] config["irrd"]["sources_default"] = ["TESTDB2"] save_yaml_config(config, run_init=False) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index dfd2d78fa..4a442daf3 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -2,10 +2,14 @@ from datetime import datetime, timedelta, timezone from typing import Optional -from irrd.conf import get_setting +from irrd.conf import ( + get_setting, + get_nrtm_response_dummy_object_class_for_source, +) +from irrd.rpsl.rpsl_objects import rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery -from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func +from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func, dummy_rpsl_object class NRTMGeneratorException(Exception): # noqa: N818 @@ -119,6 +123,26 @@ def generate( if version == "3": operation_str += " " + str(operation["serial_nrtm"]) text = operation["object_text"] + + nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source) + if nrtm_response_dummy_object_class: + object_class = operation["object_class"] + + if object_class in nrtm_response_dummy_object_class: + obj = rpsl_object_from_text(text.strip(), strict_validation=False) + + dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") + + if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): + dummy_remarks = textwrap.indent( + get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), "remarks:".ljust(16) + ) + else: + dummy_remarks = None + + if dummy_attributes: + text = dummy_rpsl_object(text, dummy_attributes, obj.pk(), dummy_remarks) + if remove_auth_hashes: text = remove_auth_hashes_func(text) operation_str += "\n\n" + text diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index 431b12a61..962a71a2f 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -36,9 +36,13 @@ def prepare_generator(monkeypatch, config_override): [ { # The CRYPT-PW hash must not appear in the output - "object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n", + "object_text": ( + "object 1 🦄\ndescr: description\nnotify: notify@example.com\nauth: " + " CRYPT-PW foobar\n" + ), "operation": DatabaseOperation.add_or_update, "serial_nrtm": 120, + "object_class": "mntner", }, { "object_text": "object 2 🌈\n", @@ -64,7 +68,9 @@ def test_generate_serial_range_v3(self, prepare_generator): ADD 120 object 1 🦄 - auth: CRYPT-PW DummyValue # Filtered for security + descr: description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security DEL 180 @@ -82,7 +88,9 @@ def test_generate_serial_range_v1(self, prepare_generator): ADD object 1 🦄 - auth: CRYPT-PW DummyValue # Filtered for security + descr: description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security DEL @@ -100,7 +108,9 @@ def test_generate_until_last(self, prepare_generator, config_override): ADD 120 object 1 🦄 - auth: CRYPT-PW DummyValue # Filtered for security + descr: description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security DEL 180 @@ -200,7 +210,9 @@ def test_v3_range_limit_not_set(self, prepare_generator, config_override): ADD 120 object 1 🦄 - auth: CRYPT-PW DummyValue # Filtered for security + descr: description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security DEL 180 @@ -235,7 +247,9 @@ def test_include_auth_hash(self, prepare_generator): ADD 120 object 1 🦄 - auth: CRYPT-PW foobar + descr: description + notify: notify@example.com + auth: CRYPT-PW foobar DEL 180 @@ -267,7 +281,9 @@ def test_nrtm_response_header(self, prepare_generator, config_override): ADD 120 object 1 🦄 - auth: CRYPT-PW DummyValue # Filtered for security + descr: description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security DEL 180 @@ -332,3 +348,37 @@ def test_days_limit_exceeded(self, prepare_generator, config_override): with pytest.raises(NRTMGeneratorException) as nge: generator.generate("TEST", "3", 110, 190, mock_dh) assert "Requesting serials older than 14 days will be rejected" in str(nge.value) + + def test_nrtm_response_dummy_object(self, prepare_generator, config_override): + generator, mock_dh = prepare_generator + config_override( + { + "sources": { + "TEST": { + "keep_journal": True, + "nrtm_response_dummy_object_class": "mntner", + "nrtm_response_dummy_attributes": {"descr": "Dummy description"}, + "nrtm_response_dummy_remarks": "THIS OBJECT IS NOT VALID", + } + } + } + ) + + result = generator.generate("TEST", "3", 110, 190, mock_dh) + + assert result == textwrap.dedent(""" + %START Version: 3 TEST 110-190 + + ADD 120 + + object 1 🦄 + descr: Dummy description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security + remarks: THIS OBJECT IS NOT VALID + + DEL 180 + + object 2 🌈 + + %END TEST""").strip() diff --git a/irrd/utils/text.py b/irrd/utils/text.py index 24fd5f037..ac387b213 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -1,12 +1,13 @@ import re from collections.abc import Iterator -from typing import Optional, TextIO, Union +from typing import Optional, TextIO, Union, Dict from irrd.conf import PASSWORD_HASH_DUMMY_VALUE from irrd.rpsl.auth import PASSWORD_HASHERS_ALL re_remove_passwords = re.compile(r"(%s)[^\n]+" % "|".join(PASSWORD_HASHERS_ALL.keys()), flags=re.IGNORECASE) re_remove_last_modified = re.compile(r"^last-modified: [^\n]+\n", flags=re.MULTILINE) +RPSL_ATTRIBUTE_TEXT_WIDTH = 16 def remove_auth_hashes(input: Optional[str]): @@ -98,3 +99,33 @@ def _str_to_camel_case(snake_str: str): def clean_ip_value_error(value_error): return re.sub(re_clean_ip_error, "", str(value_error)) + + +def dummy_rpsl_object(rpsl_text: str, dummy_attributes: Dict[str, str], pk: str, remarks: Optional[str]): + """ + Modify the value of attributes in an RPSL object. + """ + + if not rpsl_text: + return + + lines = rpsl_text.splitlines() + + for index, line in enumerate(lines): + for key, value in dummy_attributes.items(): + if "%s" in str(value): + value = str(value).replace("%s", pk) + + if line.startswith(f"{key}:"): + format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) + if not isinstance(value, str): + value = str(value) + lines[index] = format_key + value + + dummyfied_rpsl_object = "\n".join(lines) + + if rpsl_text != dummyfied_rpsl_object: + if remarks: + dummyfied_rpsl_object += "\n" + remarks + + return dummyfied_rpsl_object From 28a5573f9bc67981d47bd29dc88c29060713656a Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Fri, 15 Mar 2024 10:39:33 +1000 Subject: [PATCH 02/22] lint --- irrd/mirroring/nrtm_generator.py | 8 +++----- irrd/utils/text.py | 3 +-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 4a442daf3..e88ca6df9 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -2,14 +2,12 @@ from datetime import datetime, timedelta, timezone from typing import Optional -from irrd.conf import ( - get_setting, - get_nrtm_response_dummy_object_class_for_source, -) +from irrd.conf import get_nrtm_response_dummy_object_class_for_source, get_setting from irrd.rpsl.rpsl_objects import rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery -from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func, dummy_rpsl_object +from irrd.utils.text import dummy_rpsl_object +from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func class NRTMGeneratorException(Exception): # noqa: N818 diff --git a/irrd/utils/text.py b/irrd/utils/text.py index ac387b213..db63c5c5f 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -1,6 +1,5 @@ import re -from collections.abc import Iterator -from typing import Optional, TextIO, Union, Dict +from typing import Dict, Iterator, Optional, TextIO, Union from irrd.conf import PASSWORD_HASH_DUMMY_VALUE from irrd.rpsl.auth import PASSWORD_HASHERS_ALL From 0eaba1ae5e8abb6eddc9a4eb2ad9eeaa154df298 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Fri, 15 Mar 2024 12:38:12 +1000 Subject: [PATCH 03/22] Fix failed tests --- irrd/conf/known_keys.py | 1 + irrd/mirroring/tests/test_nrtm_generator.py | 35 +++++++++++---------- irrd/utils/text.py | 7 +++-- 3 files changed, 23 insertions(+), 20 deletions(-) diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py index 141ee4d68..5c22624e1 100644 --- a/irrd/conf/known_keys.py +++ b/irrd/conf/known_keys.py @@ -100,6 +100,7 @@ "nrtm_access_list_unfiltered", "nrtm_query_serial_days_limit", "nrtm_query_serial_range_limit", + "nrtm_response_header", "nrtm_response_dummy_attributes", "nrtm_response_dummy_object_class", "nrtm_response_dummy_remarks", diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index 962a71a2f..1bedd1227 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -37,7 +37,7 @@ def prepare_generator(monkeypatch, config_override): { # The CRYPT-PW hash must not appear in the output "object_text": ( - "object 1 🦄\ndescr: description\nnotify: notify@example.com\nauth: " + "mntner: TEST-MNT\ndescr: description\nnotify: notify@example.com\nauth: " " CRYPT-PW foobar\n" ), "operation": DatabaseOperation.add_or_update, @@ -45,9 +45,10 @@ def prepare_generator(monkeypatch, config_override): "object_class": "mntner", }, { - "object_text": "object 2 🌈\n", + "object_text": "mntner: TEST-MNT\n", "operation": DatabaseOperation.delete, "serial_nrtm": 180, + "object_class": "mntner", }, ], ] @@ -67,14 +68,14 @@ def test_generate_serial_range_v3(self, prepare_generator): ADD 120 - object 1 🦄 + mntner: TEST-MNT descr: description notify: notify@example.com auth: CRYPT-PW DummyValue # Filtered for security DEL 180 - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() @@ -87,14 +88,14 @@ def test_generate_serial_range_v1(self, prepare_generator): ADD - object 1 🦄 + mntner: TEST-MNT descr: description notify: notify@example.com auth: CRYPT-PW DummyValue # Filtered for security DEL - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() @@ -107,14 +108,14 @@ def test_generate_until_last(self, prepare_generator, config_override): ADD 120 - object 1 🦄 + mntner: TEST-MNT descr: description notify: notify@example.com - auth: CRYPT-PW DummyValue # Filtered for security + auth: CRYPT-PW DummyValue # Filtered for security DEL 180 - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() @@ -209,14 +210,14 @@ def test_v3_range_limit_not_set(self, prepare_generator, config_override): ADD 120 - object 1 🦄 + mntner: TEST-MNT descr: description notify: notify@example.com auth: CRYPT-PW DummyValue # Filtered for security DEL 180 - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() @@ -246,14 +247,14 @@ def test_include_auth_hash(self, prepare_generator): ADD 120 - object 1 🦄 + mntner: TEST-MNT descr: description notify: notify@example.com auth: CRYPT-PW foobar DEL 180 - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() @@ -280,14 +281,14 @@ def test_nrtm_response_header(self, prepare_generator, config_override): ADD 120 - object 1 🦄 + mntner: TEST-MNT descr: description notify: notify@example.com auth: CRYPT-PW DummyValue # Filtered for security DEL 180 - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() @@ -371,7 +372,7 @@ def test_nrtm_response_dummy_object(self, prepare_generator, config_override): ADD 120 - object 1 🦄 + mntner: TEST-MNT descr: Dummy description notify: notify@example.com auth: CRYPT-PW DummyValue # Filtered for security @@ -379,6 +380,6 @@ def test_nrtm_response_dummy_object(self, prepare_generator, config_override): DEL 180 - object 2 🌈 + mntner: TEST-MNT %END TEST""").strip() diff --git a/irrd/utils/text.py b/irrd/utils/text.py index db63c5c5f..45b66778f 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -121,10 +121,11 @@ def dummy_rpsl_object(rpsl_text: str, dummy_attributes: Dict[str, str], pk: str, value = str(value) lines[index] = format_key + value - dummyfied_rpsl_object = "\n".join(lines) + dummyfied_rpsl_object = "\n".join(lines) + "\n" if rpsl_text != dummyfied_rpsl_object: if remarks: - dummyfied_rpsl_object += "\n" + remarks + dummyfied_rpsl_object += remarks.strip() + "\n" + return dummyfied_rpsl_object - return dummyfied_rpsl_object + return rpsl_text From 301b8c60eb98becd422ea182850063ac0acdb70d Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Fri, 15 Mar 2024 13:30:34 +1000 Subject: [PATCH 04/22] Lint --- irrd/mirroring/tests/test_nrtm_generator.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index 1bedd1227..dd2cef1da 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -37,8 +37,8 @@ def prepare_generator(monkeypatch, config_override): { # The CRYPT-PW hash must not appear in the output "object_text": ( - "mntner: TEST-MNT\ndescr: description\nnotify: notify@example.com\nauth: " - " CRYPT-PW foobar\n" + "mntner: TEST-MNT\ndescr: description\nnotify: " + " notify@example.com\nauth: CRYPT-PW foobar\n" ), "operation": DatabaseOperation.add_or_update, "serial_nrtm": 120, From d569a856c936b815b01c4949ce0b0e1c800a3d6c Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Fri, 15 Mar 2024 14:18:00 +1000 Subject: [PATCH 05/22] Add more tests --- irrd/mirroring/tests/test_nrtm_generator.py | 32 +++++++++++++++++++++ irrd/utils/tests/test_text.py | 15 ++++++++++ irrd/utils/text.py | 2 +- 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index dd2cef1da..ed85f05b8 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -383,3 +383,35 @@ def test_nrtm_response_dummy_object(self, prepare_generator, config_override): mntner: TEST-MNT %END TEST""").strip() + + def test_nrtm_response_dummy_object_without_remarks(self, prepare_generator, config_override): + generator, mock_dh = prepare_generator + config_override( + { + "sources": { + "TEST": { + "keep_journal": True, + "nrtm_response_dummy_object_class": "mntner", + "nrtm_response_dummy_attributes": {"descr": "Dummy description"}, + } + } + } + ) + + result = generator.generate("TEST", "3", 110, 190, mock_dh) + + assert result == textwrap.dedent(""" + %START Version: 3 TEST 110-190 + + ADD 120 + + mntner: TEST-MNT + descr: Dummy description + notify: notify@example.com + auth: CRYPT-PW DummyValue # Filtered for security + + DEL 180 + + mntner: TEST-MNT + + %END TEST""").strip() diff --git a/irrd/utils/tests/test_text.py b/irrd/utils/tests/test_text.py index 3bfd11d21..4ab576314 100644 --- a/irrd/utils/tests/test_text.py +++ b/irrd/utils/tests/test_text.py @@ -11,6 +11,7 @@ snake_to_camel_case, split_paragraphs_rpsl, splitline_unicodesafe, + dummy_rpsl_object, ) @@ -64,3 +65,17 @@ def test_split_paragraphs_rpsl(): def test_snake_to_camel_case(): assert snake_to_camel_case("foo1_bar") == "foo1Bar" assert snake_to_camel_case(["foo1_bar", "second_item"]) == ["foo1Bar", "secondItem"] + + +def test_dummy_rpsl_object(): + assert dummy_rpsl_object("", {}, "", None) == "" + assert ( + dummy_rpsl_object( + "person: Test person\nnic-hdl: PERSON-TEST\nphone: +31 20 000 0000", + {"person": "Dummy person for %s", "phone": 1234}, + "PERSON-TEST", + "remarks: Invalid object", + ) + == "person: Dummy person for PERSON-TEST\nnic-hdl: PERSON-TEST\nphone: " + " 1234\nremarks: Invalid object\n" + ) diff --git a/irrd/utils/text.py b/irrd/utils/text.py index 45b66778f..2367c9bc9 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -106,7 +106,7 @@ def dummy_rpsl_object(rpsl_text: str, dummy_attributes: Dict[str, str], pk: str, """ if not rpsl_text: - return + return rpsl_text lines = rpsl_text.splitlines() From 04dfbdc7786b516dbeefe4dda6e1834ab6effe7e Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Fri, 15 Mar 2024 14:21:50 +1000 Subject: [PATCH 06/22] Lint --- irrd/utils/tests/test_text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irrd/utils/tests/test_text.py b/irrd/utils/tests/test_text.py index 4ab576314..b3817709d 100644 --- a/irrd/utils/tests/test_text.py +++ b/irrd/utils/tests/test_text.py @@ -6,12 +6,12 @@ from irrd.utils.rpsl_samples import SAMPLE_MNTNER from ..text import ( + dummy_rpsl_object, remove_auth_hashes, remove_last_modified, snake_to_camel_case, split_paragraphs_rpsl, splitline_unicodesafe, - dummy_rpsl_object, ) From fb28ed63c2b883d594e6f22f9857909f474ee421 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Thu, 28 Mar 2024 12:57:14 +1000 Subject: [PATCH 07/22] Use the pk stored in the db --- irrd/mirroring/nrtm_generator.py | 3 ++- irrd/mirroring/tests/test_nrtm_generator.py | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index e88ca6df9..43ca8fe74 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -125,6 +125,7 @@ def generate( nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source) if nrtm_response_dummy_object_class: object_class = operation["object_class"] + pk = operation["rpsl_pk"] if object_class in nrtm_response_dummy_object_class: obj = rpsl_object_from_text(text.strip(), strict_validation=False) @@ -139,7 +140,7 @@ def generate( dummy_remarks = None if dummy_attributes: - text = dummy_rpsl_object(text, dummy_attributes, obj.pk(), dummy_remarks) + text = dummy_rpsl_object(text, dummy_attributes, pk, dummy_remarks) if remove_auth_hashes: text = remove_auth_hashes_func(text) diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index ed85f05b8..ceaa9f167 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -43,12 +43,14 @@ def prepare_generator(monkeypatch, config_override): "operation": DatabaseOperation.add_or_update, "serial_nrtm": 120, "object_class": "mntner", + "rpsl_pk": "TEST-MNT", }, { "object_text": "mntner: TEST-MNT\n", "operation": DatabaseOperation.delete, "serial_nrtm": 180, "object_class": "mntner", + "rpsl_pk": "TEST-MNT", }, ], ] From 5daf7902048515dc71f87537bb2756dca748ff44 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Thu, 28 Mar 2024 14:15:33 +1000 Subject: [PATCH 08/22] Lint --- irrd/mirroring/nrtm_generator.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 43ca8fe74..622b79b5d 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -3,7 +3,6 @@ from typing import Optional from irrd.conf import get_nrtm_response_dummy_object_class_for_source, get_setting -from irrd.rpsl.rpsl_objects import rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery from irrd.utils.text import dummy_rpsl_object @@ -128,8 +127,6 @@ def generate( pk = operation["rpsl_pk"] if object_class in nrtm_response_dummy_object_class: - obj = rpsl_object_from_text(text.strip(), strict_validation=False) - dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): From 386506c51fdf3a5ea4539c6974862c3db96e5c22 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Wed, 17 Apr 2024 14:07:20 +1000 Subject: [PATCH 09/22] Codes refactoring --- irrd/mirroring/nrtm_generator.py | 24 ++------- irrd/utils/tests/test_text.py | 85 ++++++++++++++++++++++++++++---- irrd/utils/text.py | 63 ++++++++++++++--------- 3 files changed, 121 insertions(+), 51 deletions(-) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 622b79b5d..903c168be 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -2,10 +2,10 @@ from datetime import datetime, timedelta, timezone from typing import Optional -from irrd.conf import get_nrtm_response_dummy_object_class_for_source, get_setting +from irrd.conf import get_setting from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery -from irrd.utils.text import dummy_rpsl_object +from irrd.utils.text import dummify_object_text from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func @@ -121,23 +121,9 @@ def generate( operation_str += " " + str(operation["serial_nrtm"]) text = operation["object_text"] - nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source) - if nrtm_response_dummy_object_class: - object_class = operation["object_class"] - pk = operation["rpsl_pk"] - - if object_class in nrtm_response_dummy_object_class: - dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") - - if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): - dummy_remarks = textwrap.indent( - get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), "remarks:".ljust(16) - ) - else: - dummy_remarks = None - - if dummy_attributes: - text = dummy_rpsl_object(text, dummy_attributes, pk, dummy_remarks) + object_class = operation["object_class"] + pk = operation["rpsl_pk"] + text = dummify_object_text(text, object_class, source, pk) if remove_auth_hashes: text = remove_auth_hashes_func(text) diff --git a/irrd/utils/tests/test_text.py b/irrd/utils/tests/test_text.py index b3817709d..af2618c48 100644 --- a/irrd/utils/tests/test_text.py +++ b/irrd/utils/tests/test_text.py @@ -6,7 +6,7 @@ from irrd.utils.rpsl_samples import SAMPLE_MNTNER from ..text import ( - dummy_rpsl_object, + dummify_object_text, remove_auth_hashes, remove_last_modified, snake_to_camel_case, @@ -67,15 +67,82 @@ def test_snake_to_camel_case(): assert snake_to_camel_case(["foo1_bar", "second_item"]) == ["foo1Bar", "secondItem"] -def test_dummy_rpsl_object(): - assert dummy_rpsl_object("", {}, "", None) == "" +def test_dummify_object_text(config_override): + assert dummify_object_text("", "person", "TEST", "PERSON-TEST") == "" + + config_override( + { + "sources": { + "TEST": { + "keep_journal": True, + "nrtm_response_dummy_object_class": "role", + } + } + } + ) + assert ( + dummify_object_text( + "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " + " +31 20 000 0000", + "person", + "TEST", + "PERSON-TEST", + ) + == "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " + " +31 20 000 0000", + ) + + config_override( + { + "sources": { + "TEST": { + "keep_journal": True, + "nrtm_response_dummy_object_class": "person", + "nrtm_response_dummy_attributes": { + "person": "Dummy person for %s", + "address": "Dummy address", + "phone": "1234", + }, + } + } + } + ) + assert ( + dummify_object_text( + "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " + " +31 20 000 0000", + "person", + "TEST", + "PERSON-TEST", + ) + == "person: Dummy person for PERSON-TEST\naddress: Dummy address\nnic-hdl: " + " PERSON-TEST\nphone: 1234\n" + ) + + config_override( + { + "sources": { + "TEST": { + "keep_journal": True, + "nrtm_response_dummy_object_class": "person", + "nrtm_response_dummy_attributes": { + "person": "Dummy person for %s", + "address": "Dummy address", + "phone": "1234", + }, + "nrtm_response_dummy_remarks": "Invalid object", + } + } + } + ) assert ( - dummy_rpsl_object( - "person: Test person\nnic-hdl: PERSON-TEST\nphone: +31 20 000 0000", - {"person": "Dummy person for %s", "phone": 1234}, + dummify_object_text( + "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " + " +31 20 000 0000", + "person", + "TEST", "PERSON-TEST", - "remarks: Invalid object", ) - == "person: Dummy person for PERSON-TEST\nnic-hdl: PERSON-TEST\nphone: " - " 1234\nremarks: Invalid object\n" + == "person: Dummy person for PERSON-TEST\naddress: Dummy address\nnic-hdl: " + " PERSON-TEST\nphone: 1234\nremarks: Invalid object\n" ) diff --git a/irrd/utils/text.py b/irrd/utils/text.py index 2367c9bc9..bf65bd9e6 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -1,7 +1,12 @@ import re -from typing import Dict, Iterator, Optional, TextIO, Union - -from irrd.conf import PASSWORD_HASH_DUMMY_VALUE +import textwrap +from typing import Iterator, Optional, TextIO, Union + +from irrd.conf import ( + PASSWORD_HASH_DUMMY_VALUE, + get_nrtm_response_dummy_object_class_for_source, + get_setting, +) from irrd.rpsl.auth import PASSWORD_HASHERS_ALL re_remove_passwords = re.compile(r"(%s)[^\n]+" % "|".join(PASSWORD_HASHERS_ALL.keys()), flags=re.IGNORECASE) @@ -100,7 +105,7 @@ def clean_ip_value_error(value_error): return re.sub(re_clean_ip_error, "", str(value_error)) -def dummy_rpsl_object(rpsl_text: str, dummy_attributes: Dict[str, str], pk: str, remarks: Optional[str]): +def dummify_object_text(rpsl_text: str, object_class: str, source: str, pk: str): """ Modify the value of attributes in an RPSL object. """ @@ -108,24 +113,36 @@ def dummy_rpsl_object(rpsl_text: str, dummy_attributes: Dict[str, str], pk: str, if not rpsl_text: return rpsl_text - lines = rpsl_text.splitlines() - - for index, line in enumerate(lines): - for key, value in dummy_attributes.items(): - if "%s" in str(value): - value = str(value).replace("%s", pk) - - if line.startswith(f"{key}:"): - format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) - if not isinstance(value, str): - value = str(value) - lines[index] = format_key + value - - dummyfied_rpsl_object = "\n".join(lines) + "\n" - - if rpsl_text != dummyfied_rpsl_object: - if remarks: - dummyfied_rpsl_object += remarks.strip() + "\n" - return dummyfied_rpsl_object + nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source) + if nrtm_response_dummy_object_class: + if object_class in nrtm_response_dummy_object_class: + dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") + if dummy_attributes: + if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): + dummy_remarks = textwrap.indent( + get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), "remarks:".ljust(16) + ) + else: + dummy_remarks = None + + lines = rpsl_text.splitlines() + + for index, line in enumerate(lines): + for key, value in dummy_attributes.items(): + if "%s" in str(value): + value = str(value).replace("%s", pk) + + if line.startswith(f"{key}:"): + format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) + if not isinstance(value, str): + value = str(value) + lines[index] = format_key + value + + dummyfied_rpsl_object = "\n".join(lines) + "\n" + + if rpsl_text != dummyfied_rpsl_object: + if dummy_remarks: + dummyfied_rpsl_object += dummy_remarks.strip() + "\n" + return dummyfied_rpsl_object return rpsl_text From ffde3f145472f5827a8bd5a64b9cf1a936656662 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Wed, 17 Apr 2024 14:23:04 +1000 Subject: [PATCH 10/22] Lint and fix test coverage error --- irrd/utils/tests/test_text.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/irrd/utils/tests/test_text.py b/irrd/utils/tests/test_text.py index af2618c48..b460ce22c 100644 --- a/irrd/utils/tests/test_text.py +++ b/irrd/utils/tests/test_text.py @@ -89,7 +89,7 @@ def test_dummify_object_text(config_override): "PERSON-TEST", ) == "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " - " +31 20 000 0000", + " +31 20 000 0000" ) config_override( @@ -101,7 +101,7 @@ def test_dummify_object_text(config_override): "nrtm_response_dummy_attributes": { "person": "Dummy person for %s", "address": "Dummy address", - "phone": "1234", + "phone": 1234, }, } } @@ -128,7 +128,7 @@ def test_dummify_object_text(config_override): "nrtm_response_dummy_attributes": { "person": "Dummy person for %s", "address": "Dummy address", - "phone": "1234", + "phone": 1234, }, "nrtm_response_dummy_remarks": "Invalid object", } From 15b7410d3d790008387bf16d50f4a88805e3250e Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Tue, 7 May 2024 13:03:41 +1000 Subject: [PATCH 11/22] Add config for keeping original data in NRTM stream for specific source --- docs/admins/configuration.rst | 6 ++++++ irrd/conf/__init__.py | 2 ++ irrd/conf/known_keys.py | 1 + irrd/conf/test_conf.py | 1 + irrd/mirroring/nrtm_generator.py | 10 ++++++---- irrd/server/whois/query_parser.py | 4 ++++ 6 files changed, 20 insertions(+), 4 deletions(-) diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index 61005d514..0d01d6e5c 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -775,6 +775,12 @@ Sources use the `|` style to preserve newlines. |br| **Default**: not defined, no additional header added. |br| **Change takes effect**: after SIGHUP, upon next request. +* ``sources.{name}.nrtm_original_data_access_list``: a reference to an access + list in the configuration, where only IPs in the list are permitted access + to the original data in the NRTMv3 stream for this particular source (``-g`` queries), + regardless of any dummy data defined. + IPs not in the list will get the dummy data if defined. + |br| **Default**: not defined, all NRTMv3 clients get the dummy data if defined. * ``sources.{name}.nrtm_response_dummy_object_class``: a list of object classes that will contain the dummy data within the NRTMv3 responses. IRRD will dummy an object class only if the ``nrtm_response_dummy_attributes`` diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index 0116739dc..6d569ae81 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -531,6 +531,8 @@ def _validate_subconfig(key, value): expected_access_lists.add(details.get("nrtm_access_list")) if details.get("nrtm_access_list_unfiltered"): expected_access_lists.add(details.get("nrtm_access_list_unfiltered")) + if details.get("nrtm_original_data_access_list"): + expected_access_lists.add(details.get("nrtm_original_data_access_list")) source_keys_no_duplicates = ["nrtm4_server_local_path"] for key in source_keys_no_duplicates: diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py index 5c22624e1..93864e633 100644 --- a/irrd/conf/known_keys.py +++ b/irrd/conf/known_keys.py @@ -98,6 +98,7 @@ "export_timer", "nrtm_access_list", "nrtm_access_list_unfiltered", + "nrtm_original_data_access_list", "nrtm_query_serial_days_limit", "nrtm_query_serial_range_limit", "nrtm_response_header", diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index d58355fff..841b88b95 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -109,6 +109,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp "suspension_enabled": True, "nrtm_query_serial_range_limit": 10, "object_class_filter": "route", + "nrtm_original_data_access_list": "valid-list", "nrtm_response_dummy_object_class": "person", }, "TESTDB2": { diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 903c168be..45382d1b5 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -5,7 +5,7 @@ from irrd.conf import get_setting from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseJournalQuery -from irrd.utils.text import dummify_object_text +from irrd.utils.text import dummify_object_text as dummify_object_text_func from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func @@ -22,6 +22,7 @@ def generate( serial_end_requested: Optional[int], database_handler: DatabaseHandler, remove_auth_hashes=True, + dummify_object_text=True, ) -> str: """ Generate an NRTM response for a particular source, serial range and @@ -121,9 +122,10 @@ def generate( operation_str += " " + str(operation["serial_nrtm"]) text = operation["object_text"] - object_class = operation["object_class"] - pk = operation["rpsl_pk"] - text = dummify_object_text(text, object_class, source, pk) + if dummify_object_text: + object_class = operation["object_class"] + pk = operation["rpsl_pk"] + text = dummify_object_text_func(text, object_class, source, pk) if remove_auth_hashes: text = remove_auth_hashes_func(text) diff --git a/irrd/server/whois/query_parser.py b/irrd/server/whois/query_parser.py index 45e537465..a9fc0dd45 100644 --- a/irrd/server/whois/query_parser.py +++ b/irrd/server/whois/query_parser.py @@ -577,6 +577,9 @@ def handle_nrtm_request(self, param): if not in_access_list and not in_unfiltered_access_list: raise InvalidQueryException("Access denied") + in_nrtm_original_data_access_list = is_client_permitted( + self.client_ip, f"sources.{source}.nrtm_original_data_access_list", log=False + ) try: return NRTMGenerator().generate( source, @@ -585,6 +588,7 @@ def handle_nrtm_request(self, param): serial_end, self.database_handler, remove_auth_hashes=not in_unfiltered_access_list, + dummify_object_text=not in_nrtm_original_data_access_list, ) except NRTMGeneratorException as nge: raise InvalidQueryException(str(nge)) From 407d984ded653702d9c3a5aa65e7852dbb3c166a Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Tue, 7 May 2024 13:24:30 +1000 Subject: [PATCH 12/22] Fix spelling error and failed test --- docs/admins/configuration.rst | 8 ++++---- irrd/server/whois/tests/test_query_parser.py | 12 ++++++------ 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index 0d01d6e5c..8d1b4275b 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -785,20 +785,20 @@ Sources that will contain the dummy data within the NRTMv3 responses. IRRD will dummy an object class only if the ``nrtm_response_dummy_attributes`` is defined and the object class attribute keys are in the dummy attributes. - |br| **Default**: not defined, no objects dummyfied. + |br| **Default**: not defined, no objects dummied. |br| **Change takes effect**: after SIGHUP, upon next request. * ``sources.{name}.nrtm_response_dummy_attributes``: object attributes that contain dummy data. This is a dictionary with the attribute keys and corresponding dummy attribute string values. The attributes will be replaced only if ``nrtm_response_dummy_object_class`` is defined and the attributes are in the defined object class. - If the attibute value has the ``%s``, IRRD will replace it by the object primary key, + If the attribute value has the ``%s``, IRRD will replace it by the object primary key, e.g. ``person: Dummy name for %s``. - |br| **Default**: not defined. no objects dummyfied. + |br| **Default**: not defined. no objects dummied. |br| **Change takes effect**: after SIGHUP, upon next request. * ``sources.{name}.nrtm_response_dummy_remarks``: an additional remarks to be added - to the end of the dummyfied object in the NRTMv3 response. + to the end of the dummied object in the NRTMv3 response. IRRD will only add the remarks to the object classes defined in the ``nrtm_response_dummy_object_class``. This can have multiple lines. When adding this to the configuration, diff --git a/irrd/server/whois/tests/test_query_parser.py b/irrd/server/whois/tests/test_query_parser.py index 31a8eccff..3ceef8ede 100644 --- a/irrd/server/whois/tests/test_query_parser.py +++ b/irrd/server/whois/tests/test_query_parser.py @@ -320,8 +320,8 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): mock_nrg = Mock() monkeypatch.setattr("irrd.server.whois.query_parser.NRTMGenerator", lambda: mock_nrg) mock_nrg.generate = ( - lambda source, version, serial_start, serial_end, dh, remove_auth_hashes: ( - f"{source}/{version}/{serial_start}/{serial_end}/{remove_auth_hashes}" + lambda source, version, serial_start, serial_end, dh, remove_auth_hashes, dummify_object_text: ( + f"{source}/{version}/{serial_start}/{serial_end}/{remove_auth_hashes}/{dummify_object_text}" ) ) @@ -345,13 +345,13 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-5") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/5/True" + assert response.result == "TEST1/3/1/5/True/True" assert not response.remove_auth_hashes response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/True" + assert response.result == "TEST1/3/1/None/True/True" assert not response.remove_auth_hashes config_override( @@ -368,7 +368,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/False" + assert response.result == "TEST1/3/1/None/False/True" assert not response.remove_auth_hashes config_override( @@ -388,7 +388,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/False" + assert response.result == "TEST1/3/1/None/False/True" assert not response.remove_auth_hashes response = parser.handle_query("-g TEST1:9:1-LAST") From 0c0c0fe2f5c43debcc63ee5112dea131261925d4 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Tue, 7 May 2024 13:28:27 +1000 Subject: [PATCH 13/22] Add new word to spelling word list --- docs/spelling_wordlist.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 77e6eccdd..e05ac75d1 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -107,3 +107,4 @@ validator validators virtualenv whois +dummied From 2d7951d0f6daab4ddc6d5bc67737497bcc835402 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Wed, 29 May 2024 10:27:51 +1000 Subject: [PATCH 14/22] Codes refactoring --- irrd/conf/__init__.py | 13 --- irrd/conf/test_conf.py | 3 - irrd/mirroring/nrtm_generator.py | 4 +- irrd/server/whois/query_parser.py | 2 +- irrd/server/whois/tests/test_query_parser.py | 12 ++- irrd/utils/text.py | 86 ++++++++++++-------- test-reports/junit.xml | 1 + 7 files changed, 59 insertions(+), 62 deletions(-) create mode 100644 test-reports/junit.xml diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index 6d569ae81..e7857b0d4 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -664,19 +664,6 @@ def get_object_class_filter_for_source(source: str) -> Optional[list[str]]: return None -def get_nrtm_response_dummy_object_class_for_source(source: str) -> Optional[List[str]]: - """ - Helper method to get the cleaned dummy object class in NRTMv3 reponse for a source, if any. - """ - dummy_object_class = get_setting(f"sources.{source}.nrtm_response_dummy_object_class") - if dummy_object_class: - if isinstance(dummy_object_class, str): - dummy_object_class = [dummy_object_class] - return [c.strip().lower() for c in dummy_object_class] - else: - return None - - def sighup_handler(signum, frame) -> None: """ Reload the settings when a SIGHUP is received. diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index 841b88b95..679050480 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -15,7 +15,6 @@ ConfigurationError, config_init, get_configuration, - get_nrtm_response_dummy_object_class_for_source, get_object_class_filter_for_source, get_setting, is_config_initialised, @@ -147,8 +146,6 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp assert is_config_initialised() assert get_object_class_filter_for_source("TESTDB") == ["route"] assert get_object_class_filter_for_source("TESTDB2") == ["route"] - assert get_nrtm_response_dummy_object_class_for_source("TESTDB") == ["person"] - assert get_nrtm_response_dummy_object_class_for_source("TESTDB2") == ["person"] config["irrd"]["sources_default"] = ["TESTDB2"] save_yaml_config(config, run_init=False) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 45382d1b5..7abd3eea1 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -22,7 +22,7 @@ def generate( serial_end_requested: Optional[int], database_handler: DatabaseHandler, remove_auth_hashes=True, - dummify_object_text=True, + client_is_dummifying_exempt=False, ) -> str: """ Generate an NRTM response for a particular source, serial range and @@ -122,7 +122,7 @@ def generate( operation_str += " " + str(operation["serial_nrtm"]) text = operation["object_text"] - if dummify_object_text: + if not client_is_dummifying_exempt: object_class = operation["object_class"] pk = operation["rpsl_pk"] text = dummify_object_text_func(text, object_class, source, pk) diff --git a/irrd/server/whois/query_parser.py b/irrd/server/whois/query_parser.py index a9fc0dd45..5bf7b081a 100644 --- a/irrd/server/whois/query_parser.py +++ b/irrd/server/whois/query_parser.py @@ -588,7 +588,7 @@ def handle_nrtm_request(self, param): serial_end, self.database_handler, remove_auth_hashes=not in_unfiltered_access_list, - dummify_object_text=not in_nrtm_original_data_access_list, + client_is_dummifying_exempt=in_nrtm_original_data_access_list, ) except NRTMGeneratorException as nge: raise InvalidQueryException(str(nge)) diff --git a/irrd/server/whois/tests/test_query_parser.py b/irrd/server/whois/tests/test_query_parser.py index 3ceef8ede..b815c3955 100644 --- a/irrd/server/whois/tests/test_query_parser.py +++ b/irrd/server/whois/tests/test_query_parser.py @@ -320,9 +320,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): mock_nrg = Mock() monkeypatch.setattr("irrd.server.whois.query_parser.NRTMGenerator", lambda: mock_nrg) mock_nrg.generate = ( - lambda source, version, serial_start, serial_end, dh, remove_auth_hashes, dummify_object_text: ( - f"{source}/{version}/{serial_start}/{serial_end}/{remove_auth_hashes}/{dummify_object_text}" - ) + lambda source, version, serial_start, serial_end, dh, remove_auth_hashes, client_is_dummifying_exempt: f"{source}/{version}/{serial_start}/{serial_end}/{remove_auth_hashes}/{client_is_dummifying_exempt}" ) response = parser.handle_query("-g TEST1:3:1-5") @@ -345,13 +343,13 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-5") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/5/True/True" + assert response.result == "TEST1/3/1/5/True/False" assert not response.remove_auth_hashes response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/True/True" + assert response.result == "TEST1/3/1/None/True/False" assert not response.remove_auth_hashes config_override( @@ -368,7 +366,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/False/True" + assert response.result == "TEST1/3/1/None/False/False" assert not response.remove_auth_hashes config_override( @@ -388,7 +386,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/False/True" + assert response.result == "TEST1/3/1/None/False/False" assert not response.remove_auth_hashes response = parser.handle_query("-g TEST1:9:1-LAST") diff --git a/irrd/utils/text.py b/irrd/utils/text.py index bf65bd9e6..27b8eb63d 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -2,11 +2,7 @@ import textwrap from typing import Iterator, Optional, TextIO, Union -from irrd.conf import ( - PASSWORD_HASH_DUMMY_VALUE, - get_nrtm_response_dummy_object_class_for_source, - get_setting, -) +from irrd.conf import PASSWORD_HASH_DUMMY_VALUE, get_setting from irrd.rpsl.auth import PASSWORD_HASHERS_ALL re_remove_passwords = re.compile(r"(%s)[^\n]+" % "|".join(PASSWORD_HASHERS_ALL.keys()), flags=re.IGNORECASE) @@ -105,44 +101,62 @@ def clean_ip_value_error(value_error): return re.sub(re_clean_ip_error, "", str(value_error)) +def get_nrtm_response_dummy_object_class_for_source(source: str) -> List[str]: + """ + Helper method to get the cleaned dummy object class in NRTMv3 reponse for a source, if any. + """ + dummy_object_class = get_setting(f"sources.{source}.nrtm_response_dummy_object_class") + if dummy_object_class: + if isinstance(dummy_object_class, str): + dummy_object_class = [dummy_object_class] + return [c.strip().lower() for c in dummy_object_class] + else: + return [] + + def dummify_object_text(rpsl_text: str, object_class: str, source: str, pk: str): """ - Modify the value of attributes in an RPSL object. + Dummifies the provided RPSL text by replacing certain attributes with dummy values, + based on the configuration defined for the given source. + + This function retrieves the configuration for dummy object class, dummy attributes + and remarks from the settings corresponding to the provided source. If dummy object class + and attributes are configured for the provided object class, attributes will be replaced + in the RPSL text with dummy values. Additionally, if dummy remarks are configured, + they will be appended to the end of the dummied object. """ if not rpsl_text: return rpsl_text nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source) - if nrtm_response_dummy_object_class: - if object_class in nrtm_response_dummy_object_class: - dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") - if dummy_attributes: - if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): - dummy_remarks = textwrap.indent( - get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), "remarks:".ljust(16) - ) - else: - dummy_remarks = None - - lines = rpsl_text.splitlines() - - for index, line in enumerate(lines): - for key, value in dummy_attributes.items(): - if "%s" in str(value): - value = str(value).replace("%s", pk) - - if line.startswith(f"{key}:"): - format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) - if not isinstance(value, str): - value = str(value) - lines[index] = format_key + value - - dummyfied_rpsl_object = "\n".join(lines) + "\n" - - if rpsl_text != dummyfied_rpsl_object: - if dummy_remarks: - dummyfied_rpsl_object += dummy_remarks.strip() + "\n" - return dummyfied_rpsl_object + if object_class in nrtm_response_dummy_object_class: + dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") + if dummy_attributes: + if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): + dummy_remarks = textwrap.indent( + get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), + "remarks:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH), + ) + else: + dummy_remarks = None + + lines = rpsl_text.splitlines() + + for index, line in enumerate(lines): + for key, value in dummy_attributes.items(): + if "%s" in str(value): + value = str(value).replace("%s", pk) + + if line.startswith(f"{key}:"): + format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) + lines[index] = format_key + str(value) + + dummyfied_rpsl_object = "\n".join(lines) + "\n" + + if rpsl_text != dummyfied_rpsl_object: + if dummy_remarks: + dummyfied_rpsl_object += dummy_remarks.strip() + "\n" + return dummyfied_rpsl_object return rpsl_text diff --git a/test-reports/junit.xml b/test-reports/junit.xml new file mode 100644 index 000000000..2fd636d06 --- /dev/null +++ b/test-reports/junit.xml @@ -0,0 +1 @@ + \ No newline at end of file From a8fa88c613ae723aaab20747dbf395a7b1bff8cc Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Wed, 29 May 2024 12:01:17 +1000 Subject: [PATCH 15/22] Delete local test reports --- test-reports/junit.xml | 1 - 1 file changed, 1 deletion(-) delete mode 100644 test-reports/junit.xml diff --git a/test-reports/junit.xml b/test-reports/junit.xml deleted file mode 100644 index 2fd636d06..000000000 --- a/test-reports/junit.xml +++ /dev/null @@ -1 +0,0 @@ - \ No newline at end of file From c818a8310bbe8ff19a280cb601a0852041b1e92d Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Wed, 5 Jun 2024 08:27:16 +1000 Subject: [PATCH 16/22] Overwrite the auth attribute if there is a dummy one --- irrd/mirroring/nrtm_generator.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 7abd3eea1..5ca1d47b1 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -122,13 +122,14 @@ def generate( operation_str += " " + str(operation["serial_nrtm"]) text = operation["object_text"] + if remove_auth_hashes: + text = remove_auth_hashes_func(text) + if not client_is_dummifying_exempt: object_class = operation["object_class"] pk = operation["rpsl_pk"] text = dummify_object_text_func(text, object_class, source, pk) - if remove_auth_hashes: - text = remove_auth_hashes_func(text) operation_str += "\n\n" + text output.append(operation_str) From 229d3ef5ec8b97ae03701ae39b8b700709b7918e Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Mon, 10 Jun 2024 08:30:36 +1000 Subject: [PATCH 17/22] lint --- irrd/mirroring/nrtm_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index 5ca1d47b1..fe40d7a33 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -124,7 +124,7 @@ def generate( if remove_auth_hashes: text = remove_auth_hashes_func(text) - + if not client_is_dummifying_exempt: object_class = operation["object_class"] pk = operation["rpsl_pk"] From 094a93b87903a779aa7b8d6952f3bba8816fb12b Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Wed, 22 Jan 2025 14:06:45 +0100 Subject: [PATCH 18/22] cleanup after rebase --- irrd/utils/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/irrd/utils/text.py b/irrd/utils/text.py index 27b8eb63d..93d82ec83 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -101,7 +101,7 @@ def clean_ip_value_error(value_error): return re.sub(re_clean_ip_error, "", str(value_error)) -def get_nrtm_response_dummy_object_class_for_source(source: str) -> List[str]: +def get_nrtm_response_dummy_object_class_for_source(source: str) -> list[str]: """ Helper method to get the cleaned dummy object class in NRTMv3 reponse for a source, if any. """ From 98d6d58ae2aaf347ab9392035d72ffc40c47f8f7 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Thu, 23 Jan 2025 11:54:02 +1000 Subject: [PATCH 19/22] Implement dummy object feature in export runner --- irrd/mirroring/mirror_runners_export.py | 5 +++ .../tests/test_mirror_runners_export.py | 31 ++++++++++++++----- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/irrd/mirroring/mirror_runners_export.py b/irrd/mirroring/mirror_runners_export.py index 820088878..c8726794d 100644 --- a/irrd/mirroring/mirror_runners_export.py +++ b/irrd/mirroring/mirror_runners_export.py @@ -8,6 +8,7 @@ from irrd.conf import get_setting from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import DatabaseStatusQuery, RPSLDatabaseQuery +from irrd.utils.text import dummify_object_text as dummify_object_text_func from irrd.utils.text import remove_auth_hashes as remove_auth_hashes_func EXPORT_PERMISSIONS = 0o644 @@ -75,6 +76,10 @@ def _export(self, export_destination, remove_auth_hashes=True): object_text = obj["object_text"] if remove_auth_hashes: object_text = remove_auth_hashes_func(object_text) + + object_text = dummify_object_text_func( + object_text, obj["object_class"], self.source, obj["rpsl_pk"] + ) object_bytes = object_text.encode("utf-8") fh.write(object_bytes + b"\n") fh.write(b"# EOF\n") diff --git a/irrd/mirroring/tests/test_mirror_runners_export.py b/irrd/mirroring/tests/test_mirror_runners_export.py index 548dfb2ce..4d0c71441 100644 --- a/irrd/mirroring/tests/test_mirror_runners_export.py +++ b/irrd/mirroring/tests/test_mirror_runners_export.py @@ -16,6 +16,10 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog): "sources": { "TEST": { "export_destination": str(tmpdir), + "nrtm_response_dummy_object_class": "mntner", + "nrtm_response_dummy_attributes": { + "descr": "Dummy description for %s", + }, } } } @@ -34,8 +38,12 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog): repeat({"serial_newest_seen": "424242"}), [ # The CRYPT-PW hash must not appear in the output - {"object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n"}, - {"object_text": "object 2 🌈\n"}, + { + "object_text": "object 1 🦄\ndescr: description\nauth: CRYPT-PW foobar\n", + "object_class": "mntner", + "rpsl_pk": "TEST-MNT", + }, + {"object_text": "object 2 🌈\n", "object_class": "person", "rpsl_pk": "PERSON-TEST"}, ], ] ) @@ -55,7 +63,8 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog): with gzip.open(export_filename) as fh: assert ( fh.read().decode("utf-8") - == "object 1 🦄\nauth: CRYPT-PW DummyValue # Filtered for security\n\nobject 2 🌈\n\n# EOF\n" + == "object 1 🦄\ndescr: Dummy description for TEST-MNT\nauth: CRYPT-PW DummyValue #" + " Filtered for security\n\nobject 2 🌈\n\n# EOF\n" ) assert flatten_mock_calls(mock_dh) == [ @@ -99,8 +108,12 @@ def test_export_unfiltered(self, tmpdir, config_override, monkeypatch, caplog): repeat({"serial_newest_seen": "424242"}), [ # The CRYPT-PW hash should appear in the output - {"object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n"}, - {"object_text": "object 2 🌈\n"}, + { + "object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n", + "object_class": "mntner", + "rpsl_pk": "TEST-MNT", + }, + {"object_text": "object 2 🌈\n", "object_class": "person", "rpsl_pk": "PERSON-TEST"}, ], ] ) @@ -166,8 +179,12 @@ def test_export_no_serial(self, tmpdir, config_override, monkeypatch, caplog): iter([]), [ # The CRYPT-PW hash must not appear in the output - {"object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n"}, - {"object_text": "object 2 🌈\n"}, + { + "object_text": "object 1 🦄\nauth: CRYPT-PW foobar\n", + "object_class": "mntner", + "rpsl_pk": "TEST-MNT", + }, + {"object_text": "object 2 🌈\n", "object_class": "person", "rpsl_pk": "PERSON-TEST"}, ], ] ) From 705a1731eab388ea9660e55dd07300ff8449a051 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Thu, 23 Jan 2025 12:28:39 +1000 Subject: [PATCH 20/22] Introduce a new config for enabling dummy object export --- docs/admins/configuration.rst | 5 +++++ irrd/conf/known_keys.py | 1 + irrd/mirroring/mirror_runners_export.py | 8 +++++--- irrd/mirroring/tests/test_mirror_runners_export.py | 1 + 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index 8d1b4275b..22203c5a6 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -729,6 +729,11 @@ Sources also the granularity of the timer. |br| **Default**: ``3600``. |br| **Change takes effect**: after SIGHUP +* ``sources.{name}.export_dummy_object``: a boolean for whether to export dummy + objects. This setting is effective only if both ``nrtm_response_dummy_object_class`` + and ``nrtm_response_dummy_attributes`` are defined. + |br| **Default**: not defined, no dummy object exported. + |br| **Change takes effect**: after SIGHUP, at the next ``export_timer``. * ``sources.{name}.nrtm_access_list``: a reference to an access list in the configuration, where only IPs in the access list are permitted filtered access to the NRTMv3 stream for this particular source (``-g`` queries). diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py index 93864e633..7abf17de6 100644 --- a/irrd/conf/known_keys.py +++ b/irrd/conf/known_keys.py @@ -95,6 +95,7 @@ "object_class_filter", "export_destination", "export_destination_unfiltered", + "export_dummy_object", "export_timer", "nrtm_access_list", "nrtm_access_list_unfiltered", diff --git a/irrd/mirroring/mirror_runners_export.py b/irrd/mirroring/mirror_runners_export.py index c8726794d..07b9fc04f 100644 --- a/irrd/mirroring/mirror_runners_export.py +++ b/irrd/mirroring/mirror_runners_export.py @@ -77,9 +77,11 @@ def _export(self, export_destination, remove_auth_hashes=True): if remove_auth_hashes: object_text = remove_auth_hashes_func(object_text) - object_text = dummify_object_text_func( - object_text, obj["object_class"], self.source, obj["rpsl_pk"] - ) + export_dummy_object = get_setting(f"sources.{self.source}.export_dummy_object") + if export_dummy_object: + object_text = dummify_object_text_func( + object_text, obj["object_class"], self.source, obj["rpsl_pk"] + ) object_bytes = object_text.encode("utf-8") fh.write(object_bytes + b"\n") fh.write(b"# EOF\n") diff --git a/irrd/mirroring/tests/test_mirror_runners_export.py b/irrd/mirroring/tests/test_mirror_runners_export.py index 4d0c71441..271a2edda 100644 --- a/irrd/mirroring/tests/test_mirror_runners_export.py +++ b/irrd/mirroring/tests/test_mirror_runners_export.py @@ -16,6 +16,7 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog): "sources": { "TEST": { "export_destination": str(tmpdir), + "export_dummy_object": True, "nrtm_response_dummy_object_class": "mntner", "nrtm_response_dummy_attributes": { "descr": "Dummy description for %s", From 6000dc9b8927b41c4a58b7f5750520f055748366 Mon Sep 17 00:00:00 2001 From: Justin Yang Date: Thu, 23 Jan 2025 15:12:13 +1000 Subject: [PATCH 21/22] Implement dummy object feature in nrtmv4 server --- irrd/mirroring/nrtm4/nrtm4_server.py | 20 ++++++++++++++++--- .../nrtm4/tests/test_nrtm4_server.py | 20 ++++++++++++++++--- irrd/utils/rpsl_samples.py | 1 + 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/irrd/mirroring/nrtm4/nrtm4_server.py b/irrd/mirroring/nrtm4/nrtm4_server.py index e8acf580b..c3a924ba5 100644 --- a/irrd/mirroring/nrtm4/nrtm4_server.py +++ b/irrd/mirroring/nrtm4/nrtm4_server.py @@ -24,7 +24,7 @@ RPSLDatabaseQuery, ) from irrd.utils.crypto import eckey_from_config, eckey_public_key_as_str, jws_serialize -from irrd.utils.text import remove_auth_hashes +from irrd.utils.text import dummify_object_text, remove_auth_hashes from ...utils.process_support import get_lockfile from ..retrieval import file_hash_sha256 @@ -318,7 +318,17 @@ def _write_snapshot(self, version: int) -> str: header.model_dump(mode="json", include=header.model_fields_set), outstream, ) - objs_filtered = ({"object": remove_auth_hashes(obj["object_text"])} for obj in objs) + objs_filtered = ( + { + "object": dummify_object_text( + remove_auth_hashes(obj["object_text"]), + obj["object_class"], + self.source, + obj["rpsl_pk"], + ) + } + for obj in objs + ) jsonseq_encode(objs_filtered, outstream) return filename @@ -351,9 +361,13 @@ def _write_delta(self, version: int, serial_global_start: int) -> Optional[str]: ) for journal_entry in journal_entries: if journal_entry["operation"] == DatabaseOperation.add_or_update: + text = remove_auth_hashes(journal_entry["object_text"]) + text = dummify_object_text( + text, journal_entry["object_class"], self.source, journal_entry["rpsl_pk"] + ) entry_encoded = { "action": "add_modify", - "object": remove_auth_hashes(journal_entry["object_text"]), + "object": text, } elif journal_entry["operation"] == DatabaseOperation.delete: entry_encoded = { diff --git a/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py b/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py index 6ab97719d..52cd98b57 100644 --- a/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py +++ b/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py @@ -22,7 +22,7 @@ from irrd.utils.crypto import jws_deserialize from irrd.utils.rpsl_samples import SAMPLE_MNTNER from irrd.utils.test_utils import MockDatabaseHandler -from irrd.utils.text import remove_auth_hashes +from irrd.utils.text import dummify_object_text, remove_auth_hashes class TestNRTM4Server: @@ -64,6 +64,12 @@ def test_nrtm4_server(self, tmpdir, config_override): "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR, "nrtm4_server_local_path": str(nrtm_path), # "nrtm4_server_snapshot_frequency": 0, + "nrtm_response_dummy_object_class": "mntner", + "nrtm_response_dummy_attributes": { + "descr": "Dummy description for %s", + "upd-to": "unread@ripe.net", + }, + "nrtm_response_dummy_remarks": "Invalid object", } }, } @@ -107,7 +113,9 @@ def test_nrtm4_server(self, tmpdir, config_override): assert snapshot[0]["type"] == "snapshot" assert snapshot[0]["session_id"] == unf["session_id"] assert snapshot[0]["version"] == unf["snapshot"]["version"] - assert snapshot[1]["object"] == remove_auth_hashes(SAMPLE_MNTNER) + assert snapshot[1]["object"] == dummify_object_text( + remove_auth_hashes(SAMPLE_MNTNER), "mntner", "TEST", "TEST-MNT" + ) assert PASSWORD_HASH_DUMMY_VALUE in snapshot[1]["object"] assert len(mock_dh.other_calls) == 2 @@ -131,6 +139,8 @@ def test_nrtm4_server(self, tmpdir, config_override): { "operation": DatabaseOperation.add_or_update, "object_text": SAMPLE_MNTNER, + "object_class": "mntner", + "rpsl_pk": "TEST-MNT", } ], ) @@ -156,7 +166,9 @@ def test_nrtm4_server(self, tmpdir, config_override): assert delta[0]["type"] == "delta" assert delta[0]["session_id"] == unf["session_id"] assert delta[0]["version"] == new_unf["version"] - assert delta[1]["object"] == remove_auth_hashes(SAMPLE_MNTNER) + assert delta[1]["object"] == dummify_object_text( + remove_auth_hashes(SAMPLE_MNTNER), "mntner", "TEST", "TEST-MNT" + ) assert delta[1]["action"] == "add_modify" assert PASSWORD_HASH_DUMMY_VALUE in delta[1]["object"] @@ -268,6 +280,8 @@ def _run_writer(self, mock_dh, status, journal=None): [ { "object_text": SAMPLE_MNTNER, + "object_class": "mntner", + "rpsl_pk": "TEST-MNT", } ] ) diff --git a/irrd/utils/rpsl_samples.py b/irrd/utils/rpsl_samples.py index 2e2c1b9cc..8fbdf7561 100644 --- a/irrd/utils/rpsl_samples.py +++ b/irrd/utils/rpsl_samples.py @@ -509,6 +509,7 @@ SAMPLE_MNTNER_CRYPT = "crypt-password" SAMPLE_MNTNER_BCRYPT = "bcrypt-password" SAMPLE_MNTNER = """mntner: TEST-MNT +descr: description admin-c: PERSON-TEST notify: notify@example.net upd-to: upd-to@example.net From 15b3f3c4ea3167d8b72c4bc548669d52a9e2b595 Mon Sep 17 00:00:00 2001 From: Sasha Romijn Date: Mon, 3 Feb 2025 20:33:07 +0100 Subject: [PATCH 22/22] Lots of adjustments --- docs/admins/configuration.rst | 53 +++++-------- docs/releases/4.5.0.rst | 11 +++ docs/spelling_wordlist.txt | 4 +- irrd/conf/__init__.py | 2 - irrd/conf/known_keys.py | 8 +- irrd/conf/test_conf.py | 5 +- irrd/mirroring/mirror_runners_export.py | 7 +- irrd/mirroring/nrtm4/nrtm4_server.py | 8 +- .../nrtm4/tests/test_nrtm4_server.py | 6 +- irrd/mirroring/nrtm_generator.py | 4 +- .../tests/test_mirror_runners_export.py | 5 +- irrd/mirroring/tests/test_nrtm_generator.py | 10 +-- irrd/scripts/tests/test_irr_rpsl_submit.py | 1 + irrd/server/whois/query_parser.py | 5 +- irrd/server/whois/tests/test_query_parser.py | 4 +- irrd/utils/tests/test_text.py | 49 +++++++----- irrd/utils/text.py | 77 ++++++++++--------- 17 files changed, 133 insertions(+), 126 deletions(-) diff --git a/docs/admins/configuration.rst b/docs/admins/configuration.rst index 22203c5a6..50eb08a8a 100644 --- a/docs/admins/configuration.rst +++ b/docs/admins/configuration.rst @@ -717,7 +717,7 @@ Sources * ``sources.{name}.export_destination_unfiltered``: a path to save full exports, including a serial file, of this source. This is identical to ``export_destination``, except that the files saved here contain full unfiltered - password hashes from mntner objects. + password hashes from mntner objects and objects are never dummified. Sharing password hashes externally is a security risk, the unfiltered data is intended only to support :doc:`availability and data migration `. @@ -729,11 +729,6 @@ Sources also the granularity of the timer. |br| **Default**: ``3600``. |br| **Change takes effect**: after SIGHUP -* ``sources.{name}.export_dummy_object``: a boolean for whether to export dummy - objects. This setting is effective only if both ``nrtm_response_dummy_object_class`` - and ``nrtm_response_dummy_attributes`` are defined. - |br| **Default**: not defined, no dummy object exported. - |br| **Change takes effect**: after SIGHUP, at the next ``export_timer``. * ``sources.{name}.nrtm_access_list``: a reference to an access list in the configuration, where only IPs in the access list are permitted filtered access to the NRTMv3 stream for this particular source (``-g`` queries). @@ -746,7 +741,7 @@ Sources * ``sources.{name}.nrtm_access_list_unfiltered``: a reference to an access list in the configuration, where IPs in the access list are permitted unfiltered access to the NRTMv3 stream for this particular source (``-g`` queries). - Unfiltered means full password hashes are included. + Unfiltered means full password hashes are included and objects are never dummified. Sharing password hashes externally is a security risk, the unfiltered data is intended only to support :doc:`availability and data migration `. @@ -780,32 +775,24 @@ Sources use the `|` style to preserve newlines. |br| **Default**: not defined, no additional header added. |br| **Change takes effect**: after SIGHUP, upon next request. -* ``sources.{name}.nrtm_original_data_access_list``: a reference to an access - list in the configuration, where only IPs in the list are permitted access - to the original data in the NRTMv3 stream for this particular source (``-g`` queries), - regardless of any dummy data defined. - IPs not in the list will get the dummy data if defined. - |br| **Default**: not defined, all NRTMv3 clients get the dummy data if defined. -* ``sources.{name}.nrtm_response_dummy_object_class``: a list of object classes - that will contain the dummy data within the NRTMv3 responses. - IRRD will dummy an object class only if the ``nrtm_response_dummy_attributes`` - is defined and the object class attribute keys are in the dummy attributes. - |br| **Default**: not defined, no objects dummied. - |br| **Change takes effect**: after SIGHUP, upon next request. -* ``sources.{name}.nrtm_response_dummy_attributes``: object attributes that contain - dummy data. This is a dictionary with the attribute keys and corresponding dummy - attribute string values. - The attributes will be replaced only if ``nrtm_response_dummy_object_class`` - is defined and the attributes are in the defined object class. - If the attribute value has the ``%s``, IRRD will replace it by the object primary key, - e.g. ``person: Dummy name for %s``. - - |br| **Default**: not defined. no objects dummied. - |br| **Change takes effect**: after SIGHUP, upon next request. -* ``sources.{name}.nrtm_response_dummy_remarks``: an additional remarks to be added - to the end of the dummied object in the NRTMv3 response. - IRRD will only add the remarks to the object classes defined in the - ``nrtm_response_dummy_object_class``. +* ``sources.{name}.nrtm_dummified_object_classes``: a list of object classes + that will be dummified in exports and NRTM, i.e. any values + of attributes listed in ``nrtm_dummified_attributes`` are removed. + This affects exports, NRTMv3 responses, and NRTMv4. + |br| **Default**: not defined, no objects dummified. + |br| **Change takes effect**: after SIGHUP, upon next request/export. +* ``sources.{name}.nrtm_dummified_attributes``: a dictionary where keys are names + of attributes to dummify, and values are the dummy data. + If the attribute value contains ``%s``, IRRD will replace it by the object primary key, + e.g. key ``person``, value ``Dummy name for person %s``. + This is only applied when the RPSL object class is listed in + ``nrtm_dummified_object_classes``. + This affects exports, NRTMv3 responses, and NRTMv4. + |br| **Default**: not defined. no objects dummified. + |br| **Change takes effect**: after SIGHUP, upon next request/export. +* ``sources.{name}.nrtm_dummified_remarks``: additional remarks to add + when an object is dummified, e.g. to indicate this is not the + original object. This can have multiple lines. When adding this to the configuration, use the `|` style to preserve newlines. |br| **Default**: not defined, no additional remarks added. diff --git a/docs/releases/4.5.0.rst b/docs/releases/4.5.0.rst index c9425b7fe..7fa661497 100644 --- a/docs/releases/4.5.0.rst +++ b/docs/releases/4.5.0.rst @@ -37,6 +37,17 @@ was added to the HTTP server in IRRD, on ``/metrics/``. This page is only accessible to IPs :doc:`configured ` in the access list set in the ``server.http.status_access_list`` setting. +RPSL object text dummification for exports and NRTM +--------------------------------------------------- +IRRD can now dummify RPSL object text in exports and NRTM, to remove +sensitive or personal information. This is controlled by the new +``sources.{name}.nrtm_dummified_object_classes`` and +``sources.{name}.nrtm_dummified_attributes`` settings. + +Dummification applies to exports, NRTMv3 responses, and NRTMv4. It does +not apply to interactive query responses like whois or GraphQL. +By default, dummification is disabled. + New "RPSL data updated" status timestamp ---------------------------------------- Various status overviews of IRRD would show a "last update" per source. diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index e05ac75d1..ae7583951 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -107,4 +107,6 @@ validator validators virtualenv whois -dummied +dummified +dummify +dummification diff --git a/irrd/conf/__init__.py b/irrd/conf/__init__.py index e7857b0d4..076e74a6e 100644 --- a/irrd/conf/__init__.py +++ b/irrd/conf/__init__.py @@ -531,8 +531,6 @@ def _validate_subconfig(key, value): expected_access_lists.add(details.get("nrtm_access_list")) if details.get("nrtm_access_list_unfiltered"): expected_access_lists.add(details.get("nrtm_access_list_unfiltered")) - if details.get("nrtm_original_data_access_list"): - expected_access_lists.add(details.get("nrtm_original_data_access_list")) source_keys_no_duplicates = ["nrtm4_server_local_path"] for key in source_keys_no_duplicates: diff --git a/irrd/conf/known_keys.py b/irrd/conf/known_keys.py index 7abf17de6..64734be64 100644 --- a/irrd/conf/known_keys.py +++ b/irrd/conf/known_keys.py @@ -95,17 +95,15 @@ "object_class_filter", "export_destination", "export_destination_unfiltered", - "export_dummy_object", "export_timer", "nrtm_access_list", "nrtm_access_list_unfiltered", - "nrtm_original_data_access_list", "nrtm_query_serial_days_limit", "nrtm_query_serial_range_limit", "nrtm_response_header", - "nrtm_response_dummy_attributes", - "nrtm_response_dummy_object_class", - "nrtm_response_dummy_remarks", + "nrtm_dummified_attributes", + "nrtm_dummified_object_classes", + "nrtm_dummified_remarks", "nrtm4_client_notification_file_url", "nrtm4_client_initial_public_key", "nrtm4_server_private_key", diff --git a/irrd/conf/test_conf.py b/irrd/conf/test_conf.py index 679050480..e9a25834f 100644 --- a/irrd/conf/test_conf.py +++ b/irrd/conf/test_conf.py @@ -108,8 +108,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp "suspension_enabled": True, "nrtm_query_serial_range_limit": 10, "object_class_filter": "route", - "nrtm_original_data_access_list": "valid-list", - "nrtm_response_dummy_object_class": "person", + "nrtm_dummified_object_classes": "person", }, "TESTDB2": { "nrtm_host": "192.0.2.1", @@ -118,7 +117,7 @@ def test_load_valid_reload_valid_config(self, monkeypatch, save_yaml_config, tmp "keep_journal": True, "route_object_preference": 200, "object_class_filter": ["ROUTE"], - "nrtm_response_dummy_object_class": ["PERSON"], + "nrtm_dummified_object_classes": ["PERSON"], }, "TESTDB3": { "export_destination_unfiltered": "/tmp", diff --git a/irrd/mirroring/mirror_runners_export.py b/irrd/mirroring/mirror_runners_export.py index 07b9fc04f..929a18a3a 100644 --- a/irrd/mirroring/mirror_runners_export.py +++ b/irrd/mirroring/mirror_runners_export.py @@ -47,7 +47,7 @@ def run(self) -> None: f"Starting an unfiltered source export for {self.source} " f"to {export_destination_unfiltered}" ) - self._export(export_destination_unfiltered, remove_auth_hashes=False) + self._export(export_destination_unfiltered, remove_auth_hashes=False, dummify_object=False) self.database_handler.commit() except Exception as exc: @@ -58,7 +58,7 @@ def run(self) -> None: finally: self.database_handler.close() - def _export(self, export_destination, remove_auth_hashes=True): + def _export(self, export_destination, remove_auth_hashes=True, dummify_object=True): filename_export = Path(export_destination) / f"{self.source.lower()}.db.gz" export_tmpfile = NamedTemporaryFile(delete=False) filename_serial = Path(export_destination) / f"{self.source.upper()}.CURRENTSERIAL" @@ -77,8 +77,7 @@ def _export(self, export_destination, remove_auth_hashes=True): if remove_auth_hashes: object_text = remove_auth_hashes_func(object_text) - export_dummy_object = get_setting(f"sources.{self.source}.export_dummy_object") - if export_dummy_object: + if dummify_object: object_text = dummify_object_text_func( object_text, obj["object_class"], self.source, obj["rpsl_pk"] ) diff --git a/irrd/mirroring/nrtm4/nrtm4_server.py b/irrd/mirroring/nrtm4/nrtm4_server.py index c3a924ba5..c7d8a1953 100644 --- a/irrd/mirroring/nrtm4/nrtm4_server.py +++ b/irrd/mirroring/nrtm4/nrtm4_server.py @@ -361,13 +361,13 @@ def _write_delta(self, version: int, serial_global_start: int) -> Optional[str]: ) for journal_entry in journal_entries: if journal_entry["operation"] == DatabaseOperation.add_or_update: - text = remove_auth_hashes(journal_entry["object_text"]) - text = dummify_object_text( - text, journal_entry["object_class"], self.source, journal_entry["rpsl_pk"] + object_text = remove_auth_hashes(journal_entry["object_text"]) + object_text = dummify_object_text( + object_text, journal_entry["object_class"], self.source, journal_entry["rpsl_pk"] ) entry_encoded = { "action": "add_modify", - "object": text, + "object": object_text, } elif journal_entry["operation"] == DatabaseOperation.delete: entry_encoded = { diff --git a/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py b/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py index 52cd98b57..5bf4897ad 100644 --- a/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py +++ b/irrd/mirroring/nrtm4/tests/test_nrtm4_server.py @@ -64,12 +64,12 @@ def test_nrtm4_server(self, tmpdir, config_override): "nrtm4_server_private_key": MOCK_UNF_PRIVATE_KEY_STR, "nrtm4_server_local_path": str(nrtm_path), # "nrtm4_server_snapshot_frequency": 0, - "nrtm_response_dummy_object_class": "mntner", - "nrtm_response_dummy_attributes": { + "nrtm_dummified_object_classes": "mntner", + "nrtm_dummified_attributes": { "descr": "Dummy description for %s", "upd-to": "unread@ripe.net", }, - "nrtm_response_dummy_remarks": "Invalid object", + "nrtm_dummified_remarks": "Invalid object", } }, } diff --git a/irrd/mirroring/nrtm_generator.py b/irrd/mirroring/nrtm_generator.py index fe40d7a33..fd6fd0826 100644 --- a/irrd/mirroring/nrtm_generator.py +++ b/irrd/mirroring/nrtm_generator.py @@ -126,9 +126,7 @@ def generate( text = remove_auth_hashes_func(text) if not client_is_dummifying_exempt: - object_class = operation["object_class"] - pk = operation["rpsl_pk"] - text = dummify_object_text_func(text, object_class, source, pk) + text = dummify_object_text_func(text, operation["object_class"], source, operation["rpsl_pk"]) operation_str += "\n\n" + text output.append(operation_str) diff --git a/irrd/mirroring/tests/test_mirror_runners_export.py b/irrd/mirroring/tests/test_mirror_runners_export.py index 271a2edda..563b0e402 100644 --- a/irrd/mirroring/tests/test_mirror_runners_export.py +++ b/irrd/mirroring/tests/test_mirror_runners_export.py @@ -16,9 +16,8 @@ def test_export(self, tmpdir, config_override, monkeypatch, caplog): "sources": { "TEST": { "export_destination": str(tmpdir), - "export_dummy_object": True, - "nrtm_response_dummy_object_class": "mntner", - "nrtm_response_dummy_attributes": { + "nrtm_dummified_object_classes": "mntner", + "nrtm_dummified_attributes": { "descr": "Dummy description for %s", }, } diff --git a/irrd/mirroring/tests/test_nrtm_generator.py b/irrd/mirroring/tests/test_nrtm_generator.py index ceaa9f167..247fb7ccc 100644 --- a/irrd/mirroring/tests/test_nrtm_generator.py +++ b/irrd/mirroring/tests/test_nrtm_generator.py @@ -359,9 +359,9 @@ def test_nrtm_response_dummy_object(self, prepare_generator, config_override): "sources": { "TEST": { "keep_journal": True, - "nrtm_response_dummy_object_class": "mntner", - "nrtm_response_dummy_attributes": {"descr": "Dummy description"}, - "nrtm_response_dummy_remarks": "THIS OBJECT IS NOT VALID", + "nrtm_dummified_object_classes": "mntner", + "nrtm_dummified_attributes": {"descr": "Dummy description"}, + "nrtm_dummified_remarks": "THIS OBJECT IS NOT VALID", } } } @@ -393,8 +393,8 @@ def test_nrtm_response_dummy_object_without_remarks(self, prepare_generator, con "sources": { "TEST": { "keep_journal": True, - "nrtm_response_dummy_object_class": "mntner", - "nrtm_response_dummy_attributes": {"descr": "Dummy description"}, + "nrtm_dummified_object_classes": "mntner", + "nrtm_dummified_attributes": {"descr": "Dummy description"}, } } } diff --git a/irrd/scripts/tests/test_irr_rpsl_submit.py b/irrd/scripts/tests/test_irr_rpsl_submit.py index a15f0b0fc..59929cf2f 100755 --- a/irrd/scripts/tests/test_irr_rpsl_submit.py +++ b/irrd/scripts/tests/test_irr_rpsl_submit.py @@ -784,6 +784,7 @@ def test_030_mixed_object_delete(self): ) self.assertRegex(result.stderr, REGEX_MIXED_DELETE) + # test DNS failure def test_040_unresolvable_host(self): table = [ ["-u", UNRESOVABLE_URL], diff --git a/irrd/server/whois/query_parser.py b/irrd/server/whois/query_parser.py index 5bf7b081a..6e69f357a 100644 --- a/irrd/server/whois/query_parser.py +++ b/irrd/server/whois/query_parser.py @@ -577,9 +577,6 @@ def handle_nrtm_request(self, param): if not in_access_list and not in_unfiltered_access_list: raise InvalidQueryException("Access denied") - in_nrtm_original_data_access_list = is_client_permitted( - self.client_ip, f"sources.{source}.nrtm_original_data_access_list", log=False - ) try: return NRTMGenerator().generate( source, @@ -588,7 +585,7 @@ def handle_nrtm_request(self, param): serial_end, self.database_handler, remove_auth_hashes=not in_unfiltered_access_list, - client_is_dummifying_exempt=in_nrtm_original_data_access_list, + client_is_dummifying_exempt=in_unfiltered_access_list, ) except NRTMGeneratorException as nge: raise InvalidQueryException(str(nge)) diff --git a/irrd/server/whois/tests/test_query_parser.py b/irrd/server/whois/tests/test_query_parser.py index b815c3955..6d46864bd 100644 --- a/irrd/server/whois/tests/test_query_parser.py +++ b/irrd/server/whois/tests/test_query_parser.py @@ -366,7 +366,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/False/False" + assert response.result == "TEST1/3/1/None/False/True" assert not response.remove_auth_hashes config_override( @@ -386,7 +386,7 @@ def test_nrtm_request(self, prepare_parser, monkeypatch, config_override): response = parser.handle_query("-g TEST1:3:1-LAST") assert response.response_type == WhoisQueryResponseType.SUCCESS assert response.mode == WhoisQueryResponseMode.RIPE - assert response.result == "TEST1/3/1/None/False/False" + assert response.result == "TEST1/3/1/None/False/True" assert not response.remove_auth_hashes response = parser.handle_query("-g TEST1:9:1-LAST") diff --git a/irrd/utils/tests/test_text.py b/irrd/utils/tests/test_text.py index b460ce22c..938c02d1a 100644 --- a/irrd/utils/tests/test_text.py +++ b/irrd/utils/tests/test_text.py @@ -75,21 +75,25 @@ def test_dummify_object_text(config_override): "sources": { "TEST": { "keep_journal": True, - "nrtm_response_dummy_object_class": "role", + "nrtm_dummified_object_classes": "person", } } } ) assert ( dummify_object_text( - "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " - " +31 20 000 0000", + "person: Test person\n" + "address: address\n" + "nic-hdl: PERSON-TEST\n" + "phone: +31 20 000 0000", "person", "TEST", "PERSON-TEST", ) - == "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " - " +31 20 000 0000" + == "person: Test person\n" + "address: address\n" + "nic-hdl: PERSON-TEST\n" + "phone: +31 20 000 0000" ) config_override( @@ -97,8 +101,8 @@ def test_dummify_object_text(config_override): "sources": { "TEST": { "keep_journal": True, - "nrtm_response_dummy_object_class": "person", - "nrtm_response_dummy_attributes": { + "nrtm_dummified_object_classes": "person", + "nrtm_dummified_attributes": { "person": "Dummy person for %s", "address": "Dummy address", "phone": 1234, @@ -109,14 +113,18 @@ def test_dummify_object_text(config_override): ) assert ( dummify_object_text( - "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " - " +31 20 000 0000", + "person: Test person\n" + "address: address\n" + "nic-hdl: PERSON-TEST\n" + "phone: +31 20 000 0000", "person", "TEST", "PERSON-TEST", ) - == "person: Dummy person for PERSON-TEST\naddress: Dummy address\nnic-hdl: " - " PERSON-TEST\nphone: 1234\n" + == "person: Dummy person for PERSON-TEST\n" + "address: Dummy address\n" + "nic-hdl: PERSON-TEST\n" + "phone: 1234\n" ) config_override( @@ -124,25 +132,30 @@ def test_dummify_object_text(config_override): "sources": { "TEST": { "keep_journal": True, - "nrtm_response_dummy_object_class": "person", - "nrtm_response_dummy_attributes": { + "nrtm_dummified_object_classes": "person", + "nrtm_dummified_attributes": { "person": "Dummy person for %s", "address": "Dummy address", "phone": 1234, }, - "nrtm_response_dummy_remarks": "Invalid object", + "nrtm_dummified_remarks": "Invalid object", } } } ) assert ( dummify_object_text( - "person: Test person\naddress: address\nnic-hdl: PERSON-TEST\nphone: " - " +31 20 000 0000", + "person: Test person\n" + "address: address\n" + "nic-hdl: PERSON-TEST\n" + "phone: +31 20 000 0000", "person", "TEST", "PERSON-TEST", ) - == "person: Dummy person for PERSON-TEST\naddress: Dummy address\nnic-hdl: " - " PERSON-TEST\nphone: 1234\nremarks: Invalid object\n" + == "person: Dummy person for PERSON-TEST\n" + "address: Dummy address\n" + "nic-hdl: PERSON-TEST\n" + "phone: 1234\n" + "remarks: Invalid object\n" ) diff --git a/irrd/utils/text.py b/irrd/utils/text.py index 93d82ec83..1f7fb052b 100644 --- a/irrd/utils/text.py +++ b/irrd/utils/text.py @@ -101,11 +101,11 @@ def clean_ip_value_error(value_error): return re.sub(re_clean_ip_error, "", str(value_error)) -def get_nrtm_response_dummy_object_class_for_source(source: str) -> list[str]: +def get_nrtm_dummified_object_classes_for_source(source: str) -> list[str]: """ - Helper method to get the cleaned dummy object class in NRTMv3 reponse for a source, if any. + Helper method to get the cleaned dummy object classes for a source, if any. """ - dummy_object_class = get_setting(f"sources.{source}.nrtm_response_dummy_object_class") + dummy_object_class = get_setting(f"sources.{source}.nrtm_dummified_object_classes") if dummy_object_class: if isinstance(dummy_object_class, str): dummy_object_class = [dummy_object_class] @@ -116,47 +116,52 @@ def get_nrtm_response_dummy_object_class_for_source(source: str) -> list[str]: def dummify_object_text(rpsl_text: str, object_class: str, source: str, pk: str): """ - Dummifies the provided RPSL text by replacing certain attributes with dummy values, + Dummifiy the provided RPSL text by replacing certain attributes with dummy values, based on the configuration defined for the given source. This function retrieves the configuration for dummy object class, dummy attributes and remarks from the settings corresponding to the provided source. If dummy object class and attributes are configured for the provided object class, attributes will be replaced in the RPSL text with dummy values. Additionally, if dummy remarks are configured, - they will be appended to the end of the dummied object. + they will be appended to the end of the dummified object. """ if not rpsl_text: return rpsl_text - nrtm_response_dummy_object_class = get_nrtm_response_dummy_object_class_for_source(source) - if object_class in nrtm_response_dummy_object_class: - dummy_attributes = get_setting(f"sources.{source}.nrtm_response_dummy_attributes") - if dummy_attributes: - if get_setting(f"sources.{source}.nrtm_response_dummy_remarks"): - dummy_remarks = textwrap.indent( - get_setting(f"sources.{source}.nrtm_response_dummy_remarks"), - "remarks:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH), - ) - else: - dummy_remarks = None - - lines = rpsl_text.splitlines() - - for index, line in enumerate(lines): - for key, value in dummy_attributes.items(): - if "%s" in str(value): - value = str(value).replace("%s", pk) - - if line.startswith(f"{key}:"): - format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) - lines[index] = format_key + str(value) - - dummyfied_rpsl_object = "\n".join(lines) + "\n" - - if rpsl_text != dummyfied_rpsl_object: - if dummy_remarks: - dummyfied_rpsl_object += dummy_remarks.strip() + "\n" - return dummyfied_rpsl_object - - return rpsl_text + nrtm_dummified_object_classes = get_nrtm_dummified_object_classes_for_source(source) + if object_class not in nrtm_dummified_object_classes: + return rpsl_text + + dummified_attributes = get_setting(f"sources.{source}.nrtm_dummified_attributes") + if not dummified_attributes: + return rpsl_text + + if get_setting(f"sources.{source}.nrtm_dummified_remarks"): + dummy_remarks = ( + textwrap.indent( + get_setting(f"sources.{source}.nrtm_dummified_remarks"), + "remarks:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH), + ).strip() + + "\n" + ) + else: + dummy_remarks = None + + lines = rpsl_text.splitlines() + + for index, line in enumerate(lines): + for key, value in dummified_attributes.items(): + if "%s" in str(value): + value = str(value).replace("%s", pk) + + if line.startswith(f"{key}:"): + format_key = f"{key}:".ljust(RPSL_ATTRIBUTE_TEXT_WIDTH) + lines[index] = format_key + str(value) + + dummified_rpsl_object = "\n".join(lines) + "\n" + + if rpsl_text != dummified_rpsl_object and dummy_remarks: + dummified_rpsl_object += dummy_remarks + + return dummified_rpsl_object