Merge pull request #485 from OpenTrafficCam/task/4190-implement-regre…

…ssion-test-for-otanalytics-cli-using-test-data

task/4190-implement-regression-test-for-otanalytics-cli-using-test-data

.github/workflows/regression-test.yml

@@ -0,0 +1,31 @@
+name: Regression Test With Pytest
+on:
+  schedule:
+    - cron: '0 0 * * *'
+  workflow_dispatch:
+
+jobs:
+  test:
+    name: Run pytest regression test
+    runs-on: [self-hosted]
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: 3.11
+      - name: Download test data
+        run: gh release download --pattern '*.zip' -R platomo/OpenTrafficCam-testdata -D tests/data
+        env:
+          GH_TOKEN: ${{ secrets.PLATOMO_OTC_TESTDATA_ACCESS }}
+      - name: Unzip test data
+        run: |
+          cd tests/data
+          unzip \*.zip
+          rm *.zip
+          cd ..
+      - name: Install requirements
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r requirements-dev.txt
+      - name: Run regression tests
+        run: pytest ./tests/regression_otanalytics.py

OTAnalytics/__main__.py

@@ -1,5 +1,9 @@
+import warnings
+
 from OTAnalytics.plugin_ui.main_application import ApplicationStarter
 
+warnings.simplefilter(action="ignore", category=FutureWarning)
+
 
 def main() -> None:
     ApplicationStarter().start()

tests/OTAnalytics/plugin_parser/test_otvision_parser.py

@@ -23,7 +23,6 @@
     SectionId,
 )
 from OTAnalytics.domain.track import (
-    Detection,
     Track,
     TrackClassificationCalculator,
     TrackId,
@@ -141,9 +140,9 @@ def test_fix_occurrence(
             detection, False, False
         )
         expected_detection = serialized_detection.copy()
-        serialized_detection[
-            ottrk_dataformat.OCCURRENCE
-        ] = detection.occurrence.strftime(ottrk_dataformat.DATE_FORMAT)
+        serialized_detection[ottrk_dataformat.OCCURRENCE] = (
+            detection.occurrence.strftime(ottrk_dataformat.DATE_FORMAT)
+        )
 
         fixer = Otdet_Version_1_0_To_1_2()
 
@@ -293,9 +292,9 @@ def test_parse_detections_output_has_same_order_as_input(
         track_builder_setup_with_sample_data: TrackBuilder,
         parser: PythonDetectionParser,
     ) -> None:
-        detections: list[
-            dict
-        ] = track_builder_setup_with_sample_data.build_serialized_detections()
+        detections: list[dict] = (
+            track_builder_setup_with_sample_data.build_serialized_detections()
+        )
         metadata_video = track_builder_setup_with_sample_data.get_metadata()[
             ottrk_dataformat.VIDEO
         ]
@@ -326,9 +325,9 @@ def test_parse_tracks(
         parser: PythonDetectionParser,
     ) -> None:
         mocked_classificator.calculate.return_value = "car"
-        detections: list[
-            dict
-        ] = track_builder_setup_with_sample_data.build_serialized_detections()
+        detections: list[dict] = (
+            track_builder_setup_with_sample_data.build_serialized_detections()
+        )
         metadata_video = track_builder_setup_with_sample_data.get_metadata()[
             ottrk_dataformat.VIDEO
         ]
@@ -350,9 +349,9 @@ def test_parse_tracks_merge_with_existing(
         mocked_classificator: Mock,
         parser: PythonDetectionParser,
     ) -> None:
-        detections: list[
-            dict
-        ] = track_builder_setup_with_sample_data.build_serialized_detections()
+        detections: list[dict] = (
+            track_builder_setup_with_sample_data.build_serialized_detections()
+        )
         deserialized_detections = (
             track_builder_setup_with_sample_data.build_detections()
         )
@@ -399,9 +398,9 @@ def test_parse_tracks_consider_minimum_length(
             mocked_track_repository,
             track_length_limit,
         )
-        detections: list[
-            dict
-        ] = track_builder_setup_with_sample_data.build_serialized_detections()
+        detections: list[dict] = (
+            track_builder_setup_with_sample_data.build_serialized_detections()
+        )
 
         metadata_video = track_builder_setup_with_sample_data.get_metadata()[
             ottrk_dataformat.VIDEO
@@ -410,19 +409,6 @@ def test_parse_tracks_consider_minimum_length(
 
         assert len(result_sorted_input) == 0
 
-    def assert_detection_equal(self, d1: Detection, d2: Detection) -> None:
-        assert d1.classification == d2.classification
-        assert d1.confidence == d2.confidence
-        assert d1.x == d2.x
-        assert d1.y == d2.y
-        assert d1.w == d2.w
-        assert d1.h == d2.h
-        assert d1.frame == d2.frame
-        assert d1.occurrence == d2.occurrence
-        assert d1.video_name == d2.video_name
-        assert d1.interpolated_detection == d2.interpolated_detection
-        assert d1.track_id == d2.track_id
-
 
 class TestOtFlowParser:
     def test_parse_sections_and_flows(self, test_data_tmp_dir: Path) -> None:

tests/benchmark_otanalytics.py

@@ -15,7 +15,6 @@
     CUTTING_SECTION_MARKER,
 )
 from OTAnalytics.application.datastore import DetectionMetadata, TrackParser
-from OTAnalytics.application.parser.cli_parser import CliArguments
 from OTAnalytics.application.run_configuration import RunConfiguration
 from OTAnalytics.application.use_cases.create_events import CreateEvents
 from OTAnalytics.application.use_cases.cut_tracks_with_sections import (
@@ -74,8 +73,8 @@
     CLASS_SCOOTER,
 )
 from OTAnalytics.plugin_ui.main_application import ApplicationStarter
+from tests.utils.builders.run_configuration import NUM_PROCESSES, create_run_config
 
-NUM_PROCESSES = 1
 PYTHON = "PYTHON"
 PANDAS = "PANDAS"
 CURRENT_DATASET_TYPE = PANDAS
@@ -112,7 +111,8 @@ class UseCaseProvider:
 
     @property
     def run_config(self) -> RunConfiguration:
-        cli_args = CliArguments(
+        return create_run_config(
+            flow_parser=self._flow_parser,
             start_cli=True,
             debug=False,
             logfile_overwrite=True,
@@ -121,11 +121,9 @@ def run_config(self) -> RunConfiguration:
             otflow_file=str(self._otflow_file),
             save_dir=self._save_dir,
             event_formats=["otevents"],
-            num_processes=NUM_PROCESSES,
             include_classes=list(self._include_classes),
             exclude_classes=list(self._exclude_classes),
         )
-        return RunConfiguration(self._flow_parser, cli_args)
 
     @property
     def sections(self) -> list[Section]:

tests/regression_otanalytics.py

@@ -0,0 +1,148 @@
+from pathlib import Path
+
+import pytest
+
+from OTAnalytics.application.parser.flow_parser import FlowParser
+from OTAnalytics.plugin_parser.otvision_parser import OtFlowParser
+from OTAnalytics.plugin_ui.main_application import ApplicationStarter
+from tests.utils.assertions import assert_two_files_equal_sorted
+from tests.utils.builders.run_configuration import create_run_config
+
+
+def to_cli_path(test_data_dir: Path, input_file: str) -> str:
+    return str(Path(test_data_dir / input_file).absolute())
+
+
+@pytest.fixture(scope="module")
+def track_file_15min(test_data_dir: Path) -> list[str]:
+    return [to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_08-00-00.ottrk")]
+
+
+@pytest.fixture(scope="module")
+def track_files_2hours(test_data_dir: Path) -> list[str]:
+    return [
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_08-00-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_08-15-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_08-30-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_08-45-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_09-00-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_09-15-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_09-30-00.ottrk"),
+        to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24_09-45-00.ottrk"),
+    ]
+
+
+@pytest.fixture(scope="module")
+def otflow_file(test_data_dir: Path) -> str:
+    return to_cli_path(test_data_dir, "OTCamera19_FR20_2023-05-24.otflow")
+
+
+@pytest.fixture
+def otflow_parser() -> FlowParser:
+    return OtFlowParser()
+
+
+class TestRegressionCompleteApplication:
+    def test_15_min(
+        self,
+        otflow_file: str,
+        test_data_dir: Path,
+        test_data_tmp_dir: Path,
+        track_file_15min: list[str],
+        otflow_parser: FlowParser,
+    ) -> None:
+        test_data = track_file_15min
+        test_interval = "15min"
+        self._execute_test(
+            otflow_file,
+            test_data,
+            test_data_dir,
+            test_data_tmp_dir,
+            test_interval,
+            otflow_parser,
+            count_interval=15,
+        )
+
+    def test_2_h_single(
+        self,
+        otflow_file: str,
+        test_data_dir: Path,
+        test_data_tmp_dir: Path,
+        track_files_2hours: list[str],
+        otflow_parser: FlowParser,
+    ) -> None:
+        for track_file in track_files_2hours:
+            test_data = [track_file]
+            test_interval = "15min"
+            self._execute_test(
+                otflow_file,
+                test_data,
+                test_data_dir,
+                test_data_tmp_dir,
+                test_interval,
+                otflow_parser,
+                count_interval=15,
+            )
+
+    def test_2_h(
+        self,
+        otflow_file: str,
+        test_data_dir: Path,
+        test_data_tmp_dir: Path,
+        track_files_2hours: list[str],
+        otflow_parser: FlowParser,
+    ) -> None:
+        test_data = track_files_2hours
+        test_interval = "2h"
+        self._execute_test(
+            otflow_file,
+            test_data,
+            test_data_dir,
+            test_data_tmp_dir,
+            test_interval,
+            otflow_parser,
+            count_interval=120,
+        )
+
+    def _execute_test(
+        self,
+        otflow_file: str,
+        test_data: list[str],
+        test_data_dir: Path,
+        test_data_tmp_dir: Path,
+        test_interval: str,
+        otflow_parser: FlowParser,
+        count_interval: int,
+    ) -> None:
+        save_name = f"{Path(test_data[0]).stem}_{test_interval}"
+
+        run_config = create_run_config(
+            track_files=[str(_file) for _file in test_data],
+            otflow_file=str(otflow_file),
+            save_dir=str(test_data_tmp_dir),
+            save_name=save_name,
+            event_formats=["csv"],
+            count_intervals=[count_interval],
+            flow_parser=otflow_parser,
+        )
+        ApplicationStarter().start_cli(run_config)
+
+        actual_events_file = Path(test_data_tmp_dir / save_name).with_suffix(
+            ".events.csv"
+        )
+        expected_events_file = Path(test_data_dir / save_name).with_suffix(
+            ".events.csv"
+        )
+        assert_two_files_equal_sorted(actual_events_file, expected_events_file)
+
+        actual_counts_file = (
+            Path(test_data_tmp_dir / save_name)
+            .with_suffix(f".counts_{count_interval}min.csv")
+            .absolute()
+        )
+        expected_counts_file = (
+            Path(test_data_dir / save_name)
+            .with_suffix(f".counts_{count_interval}min.csv")
+            .absolute()
+        )
+        assert_two_files_equal_sorted(actual_counts_file, expected_counts_file)

tests/utils/assertions.py

@@ -1,3 +1,4 @@
+from pathlib import Path
 from typing import Iterable
 from unittest.mock import Mock
 
@@ -56,3 +57,13 @@ def assert_track_dataset_has_tracks(dataset: TrackDataset, tracks: list[Track])
         actual = dataset.get_for(expected.id)
         assert actual
         assert_equal_track_properties(actual, expected)
+
+
+def assert_two_files_equal_sorted(
+    actual_counts_file: Path, expected_counts_file: Path
+) -> None:
+    with open(actual_counts_file, mode="r") as actual:
+        actual_lines = sorted(actual.readlines())
+        with open(expected_counts_file, mode="r") as expected:
+            expected_lines = sorted(expected.readlines())
+            assert actual_lines == expected_lines

tests/utils/builders/run_configuration.py

@@ -0,0 +1,45 @@
+from OTAnalytics.application.parser.cli_parser import CliArguments
+from OTAnalytics.application.parser.flow_parser import FlowParser
+from OTAnalytics.application.run_configuration import RunConfiguration
+
+NUM_PROCESSES = 1
+
+
+def create_run_config(
+    track_files: list[str],
+    otflow_file: str,
+    save_dir: str,
+    event_formats: list[str],
+    flow_parser: FlowParser,
+    start_cli: bool = True,
+    debug: bool = False,
+    logfile_overwrite: bool = True,
+    track_export: bool = False,
+    num_processes: int = NUM_PROCESSES,
+    count_intervals: list[int] | None = None,
+    config_file: str | None = None,
+    save_name: str | None = None,
+    save_suffix: str | None = None,
+    log_file: str | None = None,
+    include_classes: list[str] | None = None,
+    exclude_classes: list[str] | None = None,
+) -> RunConfiguration:
+    cli_args = CliArguments(
+        start_cli=start_cli,
+        debug=debug,
+        logfile_overwrite=logfile_overwrite,
+        track_export=track_export,
+        track_files=track_files,
+        otflow_file=otflow_file,
+        save_dir=save_dir,
+        save_name=save_name,
+        count_intervals=count_intervals,
+        event_formats=event_formats,
+        num_processes=num_processes,
+        config_file=config_file,
+        save_suffix=save_suffix,
+        log_file=log_file,
+        include_classes=include_classes,
+        exclude_classes=exclude_classes,
+    )
+    return RunConfiguration(flow_parser, cli_args)