diff --git a/docs/admins/scopefilter.rst b/docs/admins/scopefilter.rst index be99b9773..71010906a 100644 --- a/docs/admins/scopefilter.rst +++ b/docs/admins/scopefilter.rst @@ -33,10 +33,6 @@ Objects that may be filtered or the prefix overlaps with any out of scope prefix. * An `aut-num` object is out of scope if its primary key is an out of scope ASN. -* A `route-set` object is out of scope if any of the direct members - (without recursion) overlaps with any out of scope prefix. -* An `as-set` object is rejected if any of the direct members are out - of scope ASNs. * Other object classes are never out of scope. "Overlaps" for prefixes includes an exact match, less specific or more diff --git a/irrd/scopefilter/tests/test_scopefilter.py b/irrd/scopefilter/tests/test_scopefilter.py index eab956b26..30cbc359d 100644 --- a/irrd/scopefilter/tests/test_scopefilter.py +++ b/irrd/scopefilter/tests/test_scopefilter.py @@ -6,7 +6,7 @@ from irrd.rpsl.rpsl_objects import rpsl_object_from_text from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery -from irrd.utils.rpsl_samples import SAMPLE_ROUTE, SAMPLE_ROUTE_SET, SAMPLE_AS_SET, SAMPLE_INETNUM +from irrd.utils.rpsl_samples import SAMPLE_ROUTE, SAMPLE_INETNUM from irrd.utils.test_utils import flatten_mock_calls from ..status import ScopeFilterStatus from ..validators import ScopeFilterValidator @@ -79,30 +79,6 @@ def test_validate_rpsl_object(self, config_override): result = validator.validate_rpsl_object(obj) assert result == (ScopeFilterStatus.out_scope_prefix, 'prefix 192.0.2.0/24 is out of scope') - obj = rpsl_object_from_text(SAMPLE_ROUTE_SET) - assert validator.validate_rpsl_object(obj) == (ScopeFilterStatus.in_scope, '') - - config_override({ - 'scopefilter': { - 'prefixes': ['2001::/16'], - }, - }) - validator.load_filters() - result = validator.validate_rpsl_object(obj) - assert result == (ScopeFilterStatus.out_scope_prefix, 'member prefix 2001:db8::/48 is out of scope') - - obj = rpsl_object_from_text(SAMPLE_AS_SET) - assert validator.validate_rpsl_object(obj) == (ScopeFilterStatus.in_scope, '') - - config_override({ - 'scopefilter': { - 'asns': ['65539'], - }, - }) - validator.load_filters() - result = validator.validate_rpsl_object(obj) - assert result == (ScopeFilterStatus.out_scope_as, 'member ASN AS65539 is out of scope') - config_override({ 'scopefilter': { 'prefix': ['0/0'], @@ -130,63 +106,53 @@ def test_validate_all_rpsl_objects(self, config_override, monkeypatch): }, }) - mock_query_result = iter([ - [ - { - # Should become in_scope - 'rpsl_pk': '192.0.2.128/25,AS65547', - 'ip_first': '192.0.2.128', - 'prefix_length': 25, - 'asn_first': 65547, - 'source': 'TEST', - 'object_class': 'route', - 'object_text': 'text', - 'scopefilter_status': ScopeFilterStatus.out_scope_prefix, - }, - { - # Should become out_scope_prefix - 'rpsl_pk': '192.0.2.0/25,AS65547', - 'ip_first': '192.0.2.0', - 'prefix_length': 25, - 'asn_first': 65547, - 'source': 'TEST', - 'object_class': 'route', - 'object_text': 'text', - 'scopefilter_status': ScopeFilterStatus.in_scope, - }, - ], - [ - { - # Should become out_scope_as - 'rpsl_pk': 'AS-TEST', - 'ip_first': None, - 'prefix_length': None, - 'asn_first': None, - 'source': 'TEST', - 'object_class': 'as-set', - 'object_text': 'text', - 'scopefilter_status': ScopeFilterStatus.out_scope_prefix, - 'parsed_data': { - 'members': ['AS1', 'AS23456'] - }, - }, - { - # Should not change - 'rpsl_pk': 'RS-TEST', - 'ip_first': None, - 'prefix_length': None, - 'asn_first': None, - 'source': 'TEST', - 'object_class': 'route-set', - 'object_text': 'text', - 'scopefilter_status': ScopeFilterStatus.out_scope_prefix, - 'parsed_data': { - 'members': ['192.0.2.0/24'] - }, - }, - ] - ]) - mock_dh.execute_query = lambda query: next(mock_query_result) + mock_query_result = [ + { + # Should become in_scope + 'rpsl_pk': '192.0.2.128/25,AS65547', + 'ip_first': '192.0.2.128', + 'prefix_length': 25, + 'asn_first': 65547, + 'source': 'TEST', + 'object_class': 'route', + 'object_text': 'text', + 'scopefilter_status': ScopeFilterStatus.out_scope_prefix, + }, + { + # Should become out_scope_prefix + 'rpsl_pk': '192.0.2.0/25,AS65547', + 'ip_first': '192.0.2.0', + 'prefix_length': 25, + 'asn_first': 65547, + 'source': 'TEST', + 'object_class': 'route', + 'object_text': 'text', + 'scopefilter_status': ScopeFilterStatus.in_scope, + }, + { + # Should become out_scope_as + 'rpsl_pk': '192.0.2.128/25,AS65547', + 'ip_first': '192.0.2.128', + 'prefix_length': 25, + 'asn_first': 23456, + 'source': 'TEST', + 'object_class': 'route', + 'object_text': 'text', + 'scopefilter_status': ScopeFilterStatus.out_scope_prefix, + }, + { + # Should not change + 'rpsl_pk': '192.0.2.128/25,AS65548', + 'ip_first': '192.0.2.128', + 'prefix_length': 25, + 'asn_first': 65548, + 'source': 'TEST', + 'object_class': 'route', + 'object_text': 'text', + 'scopefilter_status': ScopeFilterStatus.in_scope, + }, + ] + mock_dh.execute_query = lambda query: mock_query_result validator = ScopeFilterValidator() result = validator.validate_all_rpsl_objects(mock_dh) @@ -199,13 +165,12 @@ def test_validate_all_rpsl_objects(self, config_override, monkeypatch): assert now_in_scope[0]['rpsl_pk'] == '192.0.2.128/25,AS65547' assert now_in_scope[0]['old_status'] == ScopeFilterStatus.out_scope_prefix - assert now_out_scope_as[0]['rpsl_pk'] == 'AS-TEST' + assert now_out_scope_as[0]['rpsl_pk'] == '192.0.2.128/25,AS65547' assert now_out_scope_as[0]['old_status'] == ScopeFilterStatus.out_scope_prefix assert now_out_scope_prefix[0]['rpsl_pk'] == '192.0.2.0/25,AS65547' assert now_out_scope_prefix[0]['old_status'] == ScopeFilterStatus.in_scope assert flatten_mock_calls(mock_dq) == [ - ['object_classes', (['route', 'route6', 'aut-num'],), {}], - ['object_classes', (['as-set', 'route-set'],), {}] + ['object_classes', (['route', 'route6'],), {}], ] diff --git a/irrd/scopefilter/validators.py b/irrd/scopefilter/validators.py index e3012f206..786957ff5 100644 --- a/irrd/scopefilter/validators.py +++ b/irrd/scopefilter/validators.py @@ -7,7 +7,6 @@ from irrd.rpsl.parser import RPSLObject from irrd.storage.database_handler import DatabaseHandler from irrd.storage.queries import RPSLDatabaseQuery -from irrd.utils.validators import parse_as_number from .status import ScopeFilterStatus @@ -63,15 +62,14 @@ def validate(self, source: str, prefix: Optional[IP]=None, asn: Optional[int]=No return ScopeFilterStatus.in_scope def _validate_rpsl_data(self, source: str, object_class: str, prefix: Optional[IP], - asn_first: Optional[int], members: List[str], mp_members: List[str] - ) -> Tuple[ScopeFilterStatus, str]: + asn_first: Optional[int]) -> Tuple[ScopeFilterStatus, str]: """ Validate whether a particular set of RPSL data is in scope. Depending on object_class, members and mp_members are also validated. Returns a ScopeFilterStatus. """ out_of_scope = [ScopeFilterStatus.out_scope_prefix, ScopeFilterStatus.out_scope_as] - if object_class not in ['route', 'route6', 'aut-num', 'as-set', 'route-set']: + if object_class not in ['route', 'route6']: return ScopeFilterStatus.in_scope, '' if prefix: @@ -84,26 +82,6 @@ def _validate_rpsl_data(self, source: str, object_class: str, prefix: Optional[I if asn_state in out_of_scope: return asn_state, f'ASN {asn_first} is out of scope' - if object_class == 'route-set': - for member in members + mp_members: - try: - prefix = IP(member) - except ValueError: - continue - prefix_state = self.validate(source, prefix=prefix) - if prefix_state in out_of_scope: - return prefix_state, f'member prefix {member} is out of scope' - - if object_class == 'as-set': - for member in members: - try: - _, asn = parse_as_number(member) - except ValueError: - continue - asn_state = self.validate(source, asn=asn) - if asn_state in out_of_scope: - return asn_state, f'member ASN {member} is out of scope' - return ScopeFilterStatus.in_scope, '' def validate_rpsl_object(self, rpsl_object: RPSLObject) -> Tuple[ScopeFilterStatus, str]: @@ -116,8 +94,6 @@ def validate_rpsl_object(self, rpsl_object: RPSLObject) -> Tuple[ScopeFilterStat rpsl_object.rpsl_object_class, rpsl_object.prefix, rpsl_object.asn_first, - rpsl_object.parsed_data.get('members', []), - rpsl_object.parsed_data.get('mp-members', []), ) def validate_all_rpsl_objects(self, database_handler: DatabaseHandler) -> \ @@ -141,35 +117,25 @@ def validate_all_rpsl_objects(self, database_handler: DatabaseHandler) -> \ objs_changed: Dict[ScopeFilterStatus, List[Dict[str, str]]] = defaultdict(list) - def process_results(results): - for result in results: - current_status = result['scopefilter_status'] - result['old_status'] = current_status - prefix = None - if result['ip_first']: - prefix = IP(result['ip_first'] + '/' + str(result['prefix_length'])) - new_status, _ = self._validate_rpsl_data( - result['source'], - result['object_class'], - prefix, - result['asn_first'], - result.get('parsed_data', {}).get('members', []), - result.get('parsed_data', {}).get('mp-members', []), - ) - if new_status != current_status: - result['scopefilter_status'] = new_status - objs_changed[new_status].append(result) - q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False) - q = q.object_classes(['route', 'route6', 'aut-num']) - process_results(database_handler.execute_query(q)) - - # parsed_data is only retrieved when needed, as it has a performance impact - columns.append('parsed_data') - q = RPSLDatabaseQuery(column_names=columns, enable_ordering=False) - q = q.object_classes(['as-set', 'route-set']) - process_results(database_handler.execute_query(q)) - + q = q.object_classes(['route', 'route6']) + results = database_handler.execute_query(q) + + for result in results: + current_status = result['scopefilter_status'] + result['old_status'] = current_status + prefix = None + if result['ip_first']: + prefix = IP(result['ip_first'] + '/' + str(result['prefix_length'])) + new_status, _ = self._validate_rpsl_data( + result['source'], + result['object_class'], + prefix, + result['asn_first'], + ) + if new_status != current_status: + result['scopefilter_status'] = new_status + objs_changed[new_status].append(result) return (objs_changed[ScopeFilterStatus.in_scope], objs_changed[ScopeFilterStatus.out_scope_as], objs_changed[ScopeFilterStatus.out_scope_prefix])