From 963850fe767902b6d8271135d90f3e5b8eae5ce6 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 24 Jan 2024 11:24:46 +0100 Subject: [PATCH 1/3] Fix: All node metrics were returned by default This causes a lot of data to be returned in the case of old nodes and is usually unnecessary. Solution: Default to limiting the query to the last 2 weeks of data, or to 2 weeks before the specified date of last data. --- src/aleph/db/accessors/metrics.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/src/aleph/db/accessors/metrics.py b/src/aleph/db/accessors/metrics.py index 045ebf59f..e8bf05900 100644 --- a/src/aleph/db/accessors/metrics.py +++ b/src/aleph/db/accessors/metrics.py @@ -1,3 +1,4 @@ +import time from typing import Optional from sqlalchemy import select, text @@ -66,6 +67,12 @@ def query_metric_ccn( end_date: Optional[float] = None, sort_order: Optional[str] = None, ): + # Default to the last 2 weeks from now, or 2 weeks before the `end_date`. + if not start_date and not end_date: + start_date = time.time() - 60 * 60 * 24 * 14 + elif end_date and not start_date: + start_date = end_date - 60 * 60 * 24 * 14 + select_stmt = select( [ text("item_hash"), @@ -100,6 +107,12 @@ def query_metric_crn( end_date: Optional[float] = None, sort_order: Optional[str] = None, ): + # Default to the last 2 weeks from now, or 2 weeks before the `end_date`. + if not start_date and not end_date: + start_date = time.time() - 60 * 60 * 24 * 14 + elif end_date and not start_date: + start_date = end_date - 60 * 60 * 24 * 14 + select_stmt = select( [ text("item_hash"), From 3ca0f524116c683f63d46224371088d2e847a6fc Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 24 Jan 2024 15:43:56 +0100 Subject: [PATCH 2/3] Try fix tests with mock time.time --- tests/api/test_new_metric.py | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/tests/api/test_new_metric.py b/tests/api/test_new_metric.py index 37f88a3f0..a790040cf 100644 --- a/tests/api/test_new_metric.py +++ b/tests/api/test_new_metric.py @@ -1,4 +1,5 @@ from typing import Any, Dict, Sequence +from unittest import mock import pytest import pytest_asyncio @@ -24,7 +25,9 @@ async def test_node_core_metrics(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "core", "b8b9104da69c54e58531212234fa31f49ef4c668a39a0bf6793322407857b821" ) - response = await ccn_api_client.get(uri) + with mock.patch("aleph.web.accessors.metrics.time", return_value=1701261227): + response = await ccn_api_client.get(uri) + test_data = await response.json() assert response.status == 200 @@ -43,7 +46,8 @@ async def test_node_core_metrics_sort(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "core", "2e7cd67ff8f556b0b3fb8a2ef8ab0e8e1466cfa279dd7b9bfbc8aba92e0c5672" ) - response = await ccn_api_client.get(uri, params={"sort": "DESC"}) + with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + response = await ccn_api_client.get(uri, params={"sort": "DESC"}) test_data = await response.json() assert response.status == 200 @@ -62,7 +66,8 @@ async def test_node_core_metrics_end_date(fixture_metrics_messages, ccn_api_clie uri = _generate_uri( "core", "b8b9104da69c54e58531212234fa31f49ef4c668a39a0bf6793322407857b821" ) - response = await ccn_api_client.get(uri, params={"end_date": 1701261023}) + with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + response = await ccn_api_client.get(uri, params={"end_date": 1701261023}) test_data = await response.json() assert response.status == 200 @@ -77,7 +82,8 @@ async def test_node_core_metrics_start_date(fixture_metrics_messages, ccn_api_cl uri = _generate_uri( "core", "b8b9104da69c54e58531212234fa31f49ef4c668a39a0bf6793322407857b821" ) - response = await ccn_api_client.get(uri, params={"start_date": 1701261023}) + with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + response = await ccn_api_client.get(uri, params={"start_date": 1701261023}) test_data = await response.json() assert response.status == 200 @@ -90,7 +96,8 @@ async def test_node_core_metrics_start_date(fixture_metrics_messages, ccn_api_cl @pytest.mark.asyncio async def test_node_core_not_exist(fixture_metrics_messages, ccn_api_client): uri = _generate_uri("core", "This_is_a_node_id") - response = await ccn_api_client.get(uri) + with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + response = await ccn_api_client.get(uri) assert response.status == 404 @@ -100,7 +107,8 @@ async def test_node_compute_metric(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "compute", "d491f38ec66fe23a9c9ad398a04fd4dcb44a115b948ef612db844caea85cd59a" ) - response = await ccn_api_client.get(uri) + with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + response = await ccn_api_client.get(uri) test_data = await response.json() assert response.status == 200 @@ -114,6 +122,7 @@ async def test_node_compute_metric_not_exist(fixture_metrics_messages, ccn_api_c uri = _generate_uri( "compute", "This_is_a_node_id" ) - response = await ccn_api_client.get(uri) + with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + response = await ccn_api_client.get(uri) assert response.status == 404 \ No newline at end of file From 30ea9df5092797d9c5e000b8a37f4be275272106 Mon Sep 17 00:00:00 2001 From: Hugo Herter Date: Wed, 24 Jan 2024 15:56:09 +0100 Subject: [PATCH 3/3] fixup! Try fix tests with mock time.time --- tests/api/test_new_metric.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/api/test_new_metric.py b/tests/api/test_new_metric.py index a790040cf..2d8cdc9b5 100644 --- a/tests/api/test_new_metric.py +++ b/tests/api/test_new_metric.py @@ -25,7 +25,7 @@ async def test_node_core_metrics(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "core", "b8b9104da69c54e58531212234fa31f49ef4c668a39a0bf6793322407857b821" ) - with mock.patch("aleph.web.accessors.metrics.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri) test_data = await response.json() @@ -46,7 +46,7 @@ async def test_node_core_metrics_sort(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "core", "2e7cd67ff8f556b0b3fb8a2ef8ab0e8e1466cfa279dd7b9bfbc8aba92e0c5672" ) - with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri, params={"sort": "DESC"}) test_data = await response.json() @@ -66,7 +66,7 @@ async def test_node_core_metrics_end_date(fixture_metrics_messages, ccn_api_clie uri = _generate_uri( "core", "b8b9104da69c54e58531212234fa31f49ef4c668a39a0bf6793322407857b821" ) - with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri, params={"end_date": 1701261023}) test_data = await response.json() @@ -82,7 +82,7 @@ async def test_node_core_metrics_start_date(fixture_metrics_messages, ccn_api_cl uri = _generate_uri( "core", "b8b9104da69c54e58531212234fa31f49ef4c668a39a0bf6793322407857b821" ) - with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri, params={"start_date": 1701261023}) test_data = await response.json() @@ -96,7 +96,7 @@ async def test_node_core_metrics_start_date(fixture_metrics_messages, ccn_api_cl @pytest.mark.asyncio async def test_node_core_not_exist(fixture_metrics_messages, ccn_api_client): uri = _generate_uri("core", "This_is_a_node_id") - with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri) assert response.status == 404 @@ -107,7 +107,7 @@ async def test_node_compute_metric(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "compute", "d491f38ec66fe23a9c9ad398a04fd4dcb44a115b948ef612db844caea85cd59a" ) - with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri) test_data = await response.json() @@ -117,12 +117,13 @@ async def test_node_compute_metric(fixture_metrics_messages, ccn_api_client): == "56c82c6d3b28b76456594b4b57154b6826a6d5fb97d355d0428e5ca7d08193b9" ) + @pytest.mark.asyncio async def test_node_compute_metric_not_exist(fixture_metrics_messages, ccn_api_client): uri = _generate_uri( "compute", "This_is_a_node_id" ) - with mock.patch("aleph.web.accessors.metrics.time.time", return_value=1701261227): + with mock.patch("aleph.db.accessors.metrics.time.time", return_value=1701261227): response = await ccn_api_client.get(uri) assert response.status == 404 \ No newline at end of file