-
Notifications
You must be signed in to change notification settings - Fork 52
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Fix #290 - Add last-modified to authoritative objects
- Loading branch information
Showing
12 changed files
with
208 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -40,6 +40,8 @@ Redistributions | |
| reStructuredText | ||
| RPSL | ||
| rpsl | ||
| rpki-ov-state | ||
| ov | ||
| rr | ||
| setcap | ||
| snapshotting | ||
|
|
||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| #!/usr/bin/env python | ||
| # flake8: noqa: E402 | ||
| import argparse | ||
| import logging | ||
| import sys | ||
| from pathlib import Path | ||
|
|
||
| from irrd.rpsl.rpsl_objects import rpsl_object_from_text | ||
| from irrd.storage.models import RPSLDatabaseObject | ||
| from irrd.storage.queries import RPSLDatabaseQuery | ||
|
|
||
| """ | ||
| Set last-modified attribute on all authoritative objects. | ||
| """ | ||
|
|
||
| logger = logging.getLogger(__name__) | ||
| sys.path.append(str(Path(__file__).resolve().parents[2])) | ||
|
|
||
| from irrd.conf import config_init, CONFIG_PATH_DEFAULT, get_setting | ||
| from irrd.storage.database_handler import DatabaseHandler | ||
|
|
||
|
|
||
| def set_last_modified(): | ||
| dh = DatabaseHandler() | ||
| auth_sources = [k for k, v in get_setting('sources').items() if v.get('authoritative')] | ||
| q = RPSLDatabaseQuery(column_names=['pk', 'object_text', 'updated'], enable_ordering=False) | ||
| q = q.sources(auth_sources) | ||
|
|
||
| results = list(dh.execute_query(q)) | ||
| print(f'Updating {len(results)} objects in sources {auth_sources}') | ||
| for result in results: | ||
| rpsl_obj = rpsl_object_from_text(result['object_text'], strict_validation=False) | ||
| if rpsl_obj.messages.errors(): # pragma: no cover | ||
| print(f'Failed to process {rpsl_obj}: {rpsl_obj.messages.errors()}') | ||
| continue | ||
| new_text = rpsl_obj.render_rpsl_text(result['updated']) | ||
| stmt = RPSLDatabaseObject.__table__.update().where( | ||
| RPSLDatabaseObject.__table__.c.pk == result['pk']).values( | ||
| object_text=new_text, | ||
| ) | ||
| dh.execute_statement(stmt) | ||
| dh.commit() | ||
| dh.close() | ||
|
|
||
|
|
||
| def main(): # pragma: no cover | ||
| description = """Set last-modified attribute on all authoritative objects.""" | ||
| parser = argparse.ArgumentParser(description=description) | ||
| parser.add_argument('--config', dest='config_file_path', type=str, | ||
| help=f'use a different IRRd config file (default: {CONFIG_PATH_DEFAULT})') | ||
| args = parser.parse_args() | ||
|
|
||
| config_init(args.config_file_path) | ||
|
|
||
| sys.exit(set_last_modified()) | ||
|
|
||
|
|
||
| if __name__ == '__main__': # pragma: no cover | ||
| main() |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,48 @@ | ||
| import datetime | ||
| import uuid | ||
| from unittest.mock import Mock | ||
|
|
||
| from pytz import timezone | ||
|
|
||
| from irrd.utils.rpsl_samples import SAMPLE_RTR_SET | ||
| from irrd.utils.test_utils import flatten_mock_calls | ||
| from ..set_last_modified_auth import set_last_modified | ||
|
|
||
|
|
||
| def test_set_force_reload(capsys, monkeypatch, config_override): | ||
| config_override({ | ||
| 'sources': { | ||
| 'TEST': {'authoritative': True}, | ||
| 'TEST2': {}, | ||
| } | ||
| }) | ||
| mock_dh = Mock() | ||
| monkeypatch.setattr('irrd.scripts.set_last_modified_auth.DatabaseHandler', lambda: mock_dh) | ||
| mock_dq = Mock() | ||
| monkeypatch.setattr('irrd.scripts.set_last_modified_auth.RPSLDatabaseQuery', lambda column_names, enable_ordering: mock_dq) | ||
|
|
||
| object_pk = uuid.uuid4() | ||
| mock_query_result = [ | ||
| { | ||
| 'pk': object_pk, | ||
| 'object_text': SAMPLE_RTR_SET + 'last-modified: old\n', | ||
| 'updated': datetime.datetime(2020, 1, 1, tzinfo=timezone('UTC')), | ||
| }, | ||
| ] | ||
| mock_dh.execute_query = lambda query: mock_query_result | ||
|
|
||
| set_last_modified() | ||
|
|
||
| assert flatten_mock_calls(mock_dq) == [ | ||
| ['sources', (['TEST'],), {}] | ||
| ] | ||
| assert mock_dh.mock_calls[0][0] == 'execute_statement' | ||
| statement = mock_dh.mock_calls[0][1][0] | ||
| new_text = statement.parameters['object_text'] | ||
| assert new_text == SAMPLE_RTR_SET + 'last-modified: 2020-01-01T00:00:00Z\n' | ||
|
|
||
| assert flatten_mock_calls(mock_dh)[1:] == [ | ||
| ['commit', (), {}], | ||
| ['close', (), {}] | ||
| ] | ||
| assert capsys.readouterr().out == "Updating 1 objects in sources ['TEST']\n" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.