From d067ac55bceb0162a03bde3eb060b96314ecec6a Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Tue, 17 Oct 2023 15:40:22 +0300
Subject: [PATCH 01/14] New test checking fps for every pair of profiles, as
 well as for all profiles on

---
 .../live/frames/test-fps-permutations.py      | 304 ++++++++++++++++++
 1 file changed, 304 insertions(+)
 create mode 100644 unit-tests/live/frames/test-fps-permutations.py

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
new file mode 100644
index 0000000000..8087ae26b1
--- /dev/null
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -0,0 +1,304 @@
+# License: Apache 2.0. See LICENSE file in root directory.
+# Copyright(c) 2023 Intel Corporation. All Rights Reserved.
+
+# test:donotrun:!nightly
+# test:timeout 750
+# timeout - on the worst case, we're testing on D585S, which have 8 sensors, so:
+# timeout = (8 choose 2) * 24 + 24 = 696
+# 8 choose 2 tests to do (one for each pair), plus one for all sensors on, each test takes 24 secs
+
+from rspy import test, log
+import time
+import itertools
+import math
+
+
+# To add a new test mode (ex. all singles or triplets) only add it below and on should_test
+# add it in get_time_est_string for an initial time calculation
+# run perform_fps_test(sensor_profiles_array, [mode])
+ALL_PERMUTATIONS = 1
+ALL_PAIRS = 2
+ALL_SENSORS = 3
+
+# global variable used to count on all the sensors simultaneously
+count_frames = False
+
+# tests parameters
+TEST_ALL_COMBINATIONS = False
+seconds_till_steady_state = 4
+seconds_to_count_frames = 20
+
+
+##########################################
+# ---------- Helper Functions ---------- #
+##########################################
+def get_resolution(profile):
+    return profile.as_video_stream_profile().width(), profile.as_video_stream_profile().height()
+
+
+def check_fps_pair(fps, expected_fps):
+    delta_Hz = expected_fps * 0.05  # Validation KPI is 5%
+    return (fps <= (expected_fps + delta_Hz) and fps >= (expected_fps - delta_Hz))
+
+
+def get_expected_fps(sensor_profiles_dict):
+    """
+    For every sensor, find the expected fps according to its profiles
+    Return a dictionary between the sensor and its expected fps
+    """
+    expected_fps_dict = {}
+    # print(sensor_profiles_dict)
+    for key in sensor_profiles_dict:
+        avg = 0
+        for profile in sensor_profiles_dict[key]:
+            avg += profile.fps()
+
+        avg /= len(sensor_profiles_dict[key])
+        expected_fps_dict[key] = avg
+    return expected_fps_dict
+
+
+def should_test(mode, permutation):
+    return ((mode == ALL_PERMUTATIONS) or
+            (mode == ALL_SENSORS and all(v == 1 for v in permutation)) or
+            (mode == ALL_PAIRS and permutation.count(1) == 2))
+
+
+def get_dict_for_permutation(sensor_profiles_arr, permutation):
+    """
+    Given an array of tuples (sensor, profile) and a permutation of the same length,
+    return a sensor-profiles dictionary containing the relevant profiles
+     - profile will be added only if there's 1 in the corresponding element in the permutation
+    """
+    partial_dict = {}
+    for i, j in enumerate(permutation):
+        if j == 1:
+            sensor = sensor_profiles_arr[i][0]
+            partial_dict[sensor] = partial_dict.get(sensor, []) + [sensor_profiles_arr[i][1]]
+    return partial_dict
+
+
+def get_time_est_string(num_profiles, modes):
+    s = "Estimated time for test:"
+    details_str = ""
+    total_time = 0
+    global seconds_to_count_frames
+    global seconds_till_steady_state
+    time_per_test = seconds_to_count_frames + seconds_till_steady_state
+    for mode in modes:
+        test_time = 0
+        if mode == ALL_PERMUTATIONS:
+            test_time = math.factorial(num_profiles) * time_per_test
+            details_str += f"{math.factorial(num_profiles)} tests for all permutations"
+        elif mode == ALL_PAIRS:
+            test_time = math.comb(num_profiles, 2) * time_per_test
+            details_str += f"{math.comb(num_profiles,2)} tests for all pairs"
+        elif mode == ALL_SENSORS:
+            test_time = time_per_test
+            details_str += f"1 test for all sensors on"
+
+        details_str += " + "
+        total_time += test_time
+
+    details_str = details_str[:-3]
+    details_str = f"({details_str})"
+    details_str += f" * {time_per_test} secs per test"
+    s = f"{s} {total_time} secs ({details_str})"
+    return s
+
+
+def get_tested_profiles_string(sensor_profiles_dict):
+    s = ""
+    for sensor in sensor_profiles_dict:
+        for profile in sensor_profiles_dict[sensor]:
+            s += sensor.name + " / " + profile.stream_name() + " + "
+    s = s[:-3]
+    return s
+
+
+#############################################
+# ---------- Core Test Functions ---------- #
+#############################################
+def check_fps(expected_fps, fps_measured):
+    all_fps_ok = True
+    for key in expected_fps:
+        res = check_fps_pair(expected_fps[key], fps_measured[key])
+        if not res:
+            all_fps_ok = False
+            log.e(f"Expected {expected_fps[key]} fps, received {fps_measured[key]} fps in sensor {key.name}")
+    return all_fps_ok
+
+
+def generate_functions(sensor_profiles_dict):
+    """
+    Creates callable functions for each sensor to be triggered when a new frame arrives
+    Used to count frames received for measuring fps
+    """
+    sensor_function_dict = {}
+    for sensor_key in sensor_profiles_dict:
+        def on_frame_received(frame, key=sensor_key):
+            global count_frames
+            if count_frames:
+                sensor_profiles_dict[key] += 1
+
+        sensor_function_dict[sensor_key] = on_frame_received
+    return sensor_function_dict
+
+
+def measure_fps(sensor_profiles_dict):
+    """
+    Given a dictionary of sensors and profiles to test, activate all sensors on the given profiles
+    and measure fps
+    Return a dictionary of sensors and the fps measured for them
+    """
+    global seconds_till_steady_state
+    global seconds_to_count_frames
+
+    global count_frames
+    count_frames = False
+
+    # initialize fps dict
+    sensor_fps_dict = {}
+    for key in sensor_profiles_dict:
+        sensor_fps_dict[key] = 0
+
+    # generate sensor-callable dictionary
+    funcs_dict = generate_functions(sensor_fps_dict)
+
+    for key in sensor_profiles_dict:
+        sensor = key
+        profiles = sensor_profiles_dict[key]
+        sensor.open(profiles)
+        sensor.start(funcs_dict[key])
+
+    # the core of the test - frames are counted during sleep when count_frames is on
+    time.sleep(seconds_till_steady_state)
+    count_frames = True  # Start counting frames
+    time.sleep(seconds_to_count_frames)
+    count_frames = False  # Stop counting
+
+    for key in sensor_profiles_dict:
+        sensor_fps_dict[key] /= len(sensor_profiles_dict[key])  # number of profiles on the sensor
+        sensor_fps_dict[key] /= seconds_to_count_frames
+        # now for each sensor we have the average fps received
+
+        sensor = key
+        sensor.stop()
+        sensor.close()
+
+    return sensor_fps_dict
+
+
+def get_test_details_str(sensor_profile_dict, expected_fps_dict):
+    s = ""
+    for sensor_key in sensor_profile_dict:
+        if len(sensor_profile_dict[sensor_key]) > 1:
+            s += f"Expected average fps for {sensor_key.name} is {expected_fps_dict[sensor_key]}:\n"
+            for profile in sensor_profile_dict[sensor_key]:
+                s += f"Profile {profile.stream_name()} expects {profile.fps()} fps on {get_resolution(profile)}\n"
+        else:
+            s += (f"Expected fps for sensor {sensor_key.name} on profile "
+                  f"{sensor_profile_dict[sensor_key][0].stream_name()} is {expected_fps_dict[sensor_key]}"
+                  f" on {get_resolution(sensor_profile_dict[sensor_key][0])}\n")
+    s = s.replace("on (0, 0)", "")  # remove no resolution for Motion Module profiles
+    s += "***************"
+    return s
+
+
+def perform_fps_test(sensor_profiles_arr, modes):
+
+    log.d(get_time_est_string(len(sensor_profiles_arr), modes))
+
+    for mode in modes:
+        perms = list(itertools.product([0, 1], repeat=len(sensor_profiles_arr)))
+        for perm in perms:
+            # print(perm)
+            if all(v == 0 for v in perm):
+                continue
+            if should_test(mode, perm):
+                partial_dict = get_dict_for_permutation(sensor_profiles_arr, perm)
+                test.start("Testing", get_tested_profiles_string(partial_dict))
+                expected_fps = get_expected_fps(partial_dict)
+                log.d(get_test_details_str(partial_dict, expected_fps))
+                fps_dict = measure_fps(partial_dict)
+                test.check(check_fps(expected_fps, fps_dict))
+                test.finish()
+
+
+####################################################
+# ---------- Test Preparation Functions ---------- #
+####################################################
+def get_profiles_by_resolution(sensor, resolution, fps=None):
+    # resolution is a required parameter because on a sensor all active profiles must have the same resolution
+    profiles = []
+    p_streams_added = []
+    for p in sensor.get_stream_profiles():
+        if get_resolution(p) == resolution:
+            if fps is None or p.fps() == fps:
+                if p.stream_type() not in p_streams_added:  # can't open same stream twice
+                    profiles.append(p)
+                    p_streams_added.append(p.stream_type())
+    return profiles
+
+
+def get_mutual_resolution(sensor):
+    profile_resolutions_dict = {}  # a map between a stream type and all of its possible resolutions
+    possible_combinations = []
+    for profile in sensor.get_stream_profiles():
+        stream_type = profile.stream_type()
+        resolution = get_resolution(profile)
+        fps = profile.fps()
+
+        # d[key] = d.get(key, []) + [value] -> adds to the dictionary or appends if it exists
+        profile_resolutions_dict[stream_type] = profile_resolutions_dict.get(stream_type, []) + [resolution]
+
+        if (resolution, fps) not in possible_combinations:
+            possible_combinations.append((resolution, fps))
+
+    possible_combinations.sort(reverse=True)  # sort by resolution first, then by fps, so the best res and fps are first
+
+    # first, try to find a resolution and fps that all profiles have
+    for option in possible_combinations:
+        profiles = get_profiles_by_resolution(sensor, option[0], option[1])
+        if len(profiles) == len(profile_resolutions_dict):
+            return profiles
+
+    # if none found, try to find a resolution that all profiles have, on any fps (fps will be taken as an average later)
+    for option in possible_combinations:
+        profiles = get_profiles_by_resolution(sensor, option[0], None)
+        if len(profiles) == len(profile_resolutions_dict):
+            return profiles
+
+    # if reached here, then we couldn't find a resolution that all profiles have, so we can't test them together :(
+    log.f("Can't run test, sensor", sensor.name, "doesn't have a resolution for all profiles")
+    return []
+
+
+def get_sensors_and_profiles(device):
+    """
+    Returns an array of tuples of a (sensor, profile) for each of its profiles
+    """
+    sensor_profiles_arr = []
+    for sensor in device.query_sensors():
+        profiles = get_mutual_resolution(sensor)
+        for profile in profiles:
+            sensor_profiles_arr.append((sensor, profile))
+
+    return sensor_profiles_arr
+
+
+#####################################################################################################
+
+dev = test.find_first_device_or_exit()
+sensor_profiles_array = get_sensors_and_profiles(dev)
+
+if TEST_ALL_COMBINATIONS:
+    test_modes = [ALL_PERMUTATIONS]
+else:
+    test_modes = [ALL_PAIRS, ALL_SENSORS]
+
+perform_fps_test(sensor_profiles_array, test_modes)
+
+#####################################################################################################
+
+test.print_results_and_exit()

From 0c0f53321c6c92b2f9e67f49f50b0e39db1d3c9a Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Mon, 30 Oct 2023 16:28:11 +0200
Subject: [PATCH 02/14] added locks, added comments

---
 .../live/frames/test-fps-permutations.py      | 54 +++++++++++++------
 1 file changed, 37 insertions(+), 17 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 8087ae26b1..5e7b0a144c 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -1,16 +1,18 @@
 # License: Apache 2.0. See LICENSE file in root directory.
 # Copyright(c) 2023 Intel Corporation. All Rights Reserved.
 
+# test:device D400*
 # test:donotrun:!nightly
 # test:timeout 750
-# timeout - on the worst case, we're testing on D585S, which have 8 sensors, so:
+# timeout - on the worst case, we're testing on D585S, which have 8 streams, so:
 # timeout = (8 choose 2) * 24 + 24 = 696
-# 8 choose 2 tests to do (one for each pair), plus one for all sensors on, each test takes 24 secs
+# 8 choose 2 tests to do (one for each pair), plus one for all streams on, each test takes 24 secs
 
 from rspy import test, log
 import time
 import itertools
 import math
+import threading
 
 
 # To add a new test mode (ex. all singles or triplets) only add it below and on should_test
@@ -25,8 +27,8 @@
 
 # tests parameters
 TEST_ALL_COMBINATIONS = False
-seconds_till_steady_state = 4
-seconds_to_count_frames = 20
+TIME_FOR_STEADY_STATE = 4
+TIME_TO_COUNT_FRAMES = 10
 
 
 ##########################################
@@ -47,7 +49,7 @@ def get_expected_fps(sensor_profiles_dict):
     Return a dictionary between the sensor and its expected fps
     """
     expected_fps_dict = {}
-    # print(sensor_profiles_dict)
+    # log.d(sensor_profiles_dict)
     for key in sensor_profiles_dict:
         avg = 0
         for profile in sensor_profiles_dict[key]:
@@ -59,6 +61,12 @@ def get_expected_fps(sensor_profiles_dict):
 
 
 def should_test(mode, permutation):
+    """
+    Returns true if the given permutation should be tested:
+    If the mode is ALL_PERMUTATIONS returns true (every permutation should be tested)
+    If the mode is ALL_SENSORS return true only if all streams are to be tested
+    If the mode is ALL_PAIRS return true only if there are exactly two streams to be tested
+    """
     return ((mode == ALL_PERMUTATIONS) or
             (mode == ALL_SENSORS and all(v == 1 for v in permutation)) or
             (mode == ALL_PAIRS and permutation.count(1) == 2))
@@ -66,7 +74,7 @@ def should_test(mode, permutation):
 
 def get_dict_for_permutation(sensor_profiles_arr, permutation):
     """
-    Given an array of tuples (sensor, profile) and a permutation of the same length,
+    Given an array of pairs (sensor, profile) and a permutation of the same length,
     return a sensor-profiles dictionary containing the relevant profiles
      - profile will be added only if there's 1 in the corresponding element in the permutation
     """
@@ -82,9 +90,9 @@ def get_time_est_string(num_profiles, modes):
     s = "Estimated time for test:"
     details_str = ""
     total_time = 0
-    global seconds_to_count_frames
-    global seconds_till_steady_state
-    time_per_test = seconds_to_count_frames + seconds_till_steady_state
+    global TIME_TO_COUNT_FRAMES
+    global TIME_FOR_STEADY_STATE
+    time_per_test = TIME_TO_COUNT_FRAMES + TIME_FOR_STEADY_STATE
     for mode in modes:
         test_time = 0
         if mode == ALL_PERMUTATIONS:
@@ -100,6 +108,7 @@ def get_time_est_string(num_profiles, modes):
         details_str += " + "
         total_time += test_time
 
+    # Remove the last " + " for aesthetics
     details_str = details_str[:-3]
     details_str = f"({details_str})"
     details_str += f" * {time_per_test} secs per test"
@@ -112,6 +121,8 @@ def get_tested_profiles_string(sensor_profiles_dict):
     for sensor in sensor_profiles_dict:
         for profile in sensor_profiles_dict[sensor]:
             s += sensor.name + " / " + profile.stream_name() + " + "
+
+    # Remove the last " + " for aesthetics
     s = s[:-3]
     return s
 
@@ -134,12 +145,17 @@ def generate_functions(sensor_profiles_dict):
     Creates callable functions for each sensor to be triggered when a new frame arrives
     Used to count frames received for measuring fps
     """
+    locks = {}
     sensor_function_dict = {}
     for sensor_key in sensor_profiles_dict:
+        new_lock = threading.Lock()
+        locks[sensor_key] = new_lock
+
         def on_frame_received(frame, key=sensor_key):
             global count_frames
             if count_frames:
-                sensor_profiles_dict[key] += 1
+                with locks[key]:
+                    sensor_profiles_dict[key] += 1
 
         sensor_function_dict[sensor_key] = on_frame_received
     return sensor_function_dict
@@ -151,8 +167,8 @@ def measure_fps(sensor_profiles_dict):
     and measure fps
     Return a dictionary of sensors and the fps measured for them
     """
-    global seconds_till_steady_state
-    global seconds_to_count_frames
+    global TIME_FOR_STEADY_STATE
+    global TIME_TO_COUNT_FRAMES
 
     global count_frames
     count_frames = False
@@ -172,14 +188,14 @@ def measure_fps(sensor_profiles_dict):
         sensor.start(funcs_dict[key])
 
     # the core of the test - frames are counted during sleep when count_frames is on
-    time.sleep(seconds_till_steady_state)
+    time.sleep(TIME_FOR_STEADY_STATE)
     count_frames = True  # Start counting frames
-    time.sleep(seconds_to_count_frames)
+    time.sleep(TIME_TO_COUNT_FRAMES)
     count_frames = False  # Stop counting
 
     for key in sensor_profiles_dict:
         sensor_fps_dict[key] /= len(sensor_profiles_dict[key])  # number of profiles on the sensor
-        sensor_fps_dict[key] /= seconds_to_count_frames
+        sensor_fps_dict[key] /= TIME_TO_COUNT_FRAMES
         # now for each sensor we have the average fps received
 
         sensor = key
@@ -210,9 +226,13 @@ def perform_fps_test(sensor_profiles_arr, modes):
     log.d(get_time_est_string(len(sensor_profiles_arr), modes))
 
     for mode in modes:
+        # Generate a list of all possible combinations of 1s and 0s (permutations) in the length of the array
         perms = list(itertools.product([0, 1], repeat=len(sensor_profiles_arr)))
+
+        # Go over every possible permutation and check if we should test it
+        # Each index in a permutation represents a profile, which will be tested only if the value in that index is 1
         for perm in perms:
-            # print(perm)
+            # log.d(perm)
             if all(v == 0 for v in perm):
                 continue
             if should_test(mode, perm):
@@ -276,7 +296,7 @@ def get_mutual_resolution(sensor):
 
 def get_sensors_and_profiles(device):
     """
-    Returns an array of tuples of a (sensor, profile) for each of its profiles
+    Returns an array of pairs of a (sensor, profile) for each of its profiles
     """
     sensor_profiles_arr = []
     for sensor in device.query_sensors():

From 036a1ce8b0642f4d84082bda31466128f80df63f Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Tue, 31 Oct 2023 16:05:50 +0200
Subject: [PATCH 03/14] changed test times, renaming

---
 .../live/frames/test-fps-permutations.py      | 22 +++++++++----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 5e7b0a144c..26ffa5cca0 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -27,8 +27,8 @@
 
 # tests parameters
 TEST_ALL_COMBINATIONS = False
-TIME_FOR_STEADY_STATE = 4
-TIME_TO_COUNT_FRAMES = 10
+TIME_FOR_STEADY_STATE = 1
+TIME_TO_COUNT_FRAMES = 5
 
 
 ##########################################
@@ -133,7 +133,7 @@ def get_tested_profiles_string(sensor_profiles_dict):
 def check_fps(expected_fps, fps_measured):
     all_fps_ok = True
     for key in expected_fps:
-        res = check_fps_pair(expected_fps[key], fps_measured[key])
+        res = check_fps_pair(fps_measured[key], expected_fps[key])
         if not res:
             all_fps_ok = False
             log.e(f"Expected {expected_fps[key]} fps, received {fps_measured[key]} fps in sensor {key.name}")
@@ -251,18 +251,19 @@ def perform_fps_test(sensor_profiles_arr, modes):
 def get_profiles_by_resolution(sensor, resolution, fps=None):
     # resolution is a required parameter because on a sensor all active profiles must have the same resolution
     profiles = []
-    p_streams_added = []
+    stream_types_added = []
     for p in sensor.get_stream_profiles():
         if get_resolution(p) == resolution:
             if fps is None or p.fps() == fps:
-                if p.stream_type() not in p_streams_added:  # can't open same stream twice
+                # to avoid having a long run time, we don't choose the same stream more than once
+                if p.stream_type() not in stream_types_added:
                     profiles.append(p)
-                    p_streams_added.append(p.stream_type())
+                    stream_types_added.append(p.stream_type())
     return profiles
 
 
 def get_mutual_resolution(sensor):
-    profile_resolutions_dict = {}  # a map between a stream type and all of its possible resolutions
+    stream_resolutions_dict = {}  # a map between a stream type and all of its possible resolutions
     possible_combinations = []
     for profile in sensor.get_stream_profiles():
         stream_type = profile.stream_type()
@@ -270,7 +271,7 @@ def get_mutual_resolution(sensor):
         fps = profile.fps()
 
         # d[key] = d.get(key, []) + [value] -> adds to the dictionary or appends if it exists
-        profile_resolutions_dict[stream_type] = profile_resolutions_dict.get(stream_type, []) + [resolution]
+        stream_resolutions_dict[stream_type] = stream_resolutions_dict.get(stream_type, []) + [resolution]
 
         if (resolution, fps) not in possible_combinations:
             possible_combinations.append((resolution, fps))
@@ -280,15 +281,14 @@ def get_mutual_resolution(sensor):
     # first, try to find a resolution and fps that all profiles have
     for option in possible_combinations:
         profiles = get_profiles_by_resolution(sensor, option[0], option[1])
-        if len(profiles) == len(profile_resolutions_dict):
+        if len(profiles) == len(stream_resolutions_dict):
             return profiles
 
     # if none found, try to find a resolution that all profiles have, on any fps (fps will be taken as an average later)
     for option in possible_combinations:
         profiles = get_profiles_by_resolution(sensor, option[0], None)
-        if len(profiles) == len(profile_resolutions_dict):
+        if len(profiles) == len(stream_resolutions_dict):
             return profiles
-
     # if reached here, then we couldn't find a resolution that all profiles have, so we can't test them together :(
     log.f("Can't run test, sensor", sensor.name, "doesn't have a resolution for all profiles")
     return []

From b197b6abdc9da6f011cdd523755cfe648b2a363f Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Wed, 1 Nov 2023 16:36:26 +0200
Subject: [PATCH 04/14] reduced test time

---
 unit-tests/live/frames/test-fps-permutations.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 26ffa5cca0..1f21f9aa07 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -3,10 +3,10 @@
 
 # test:device D400*
 # test:donotrun:!nightly
-# test:timeout 750
+# test:timeout 300
 # timeout - on the worst case, we're testing on D585S, which have 8 streams, so:
-# timeout = (8 choose 2) * 24 + 24 = 696
-# 8 choose 2 tests to do (one for each pair), plus one for all streams on, each test takes 24 secs
+# timeout = ((8 choose 2)+1) * (TIME_FOR_STEADY_STATE + TIME_TO_COUNT_FRAMES)
+# 8 choose 2 tests to do (one for each pair), plus one for all streams on
 
 from rspy import test, log
 import time
@@ -27,7 +27,7 @@
 
 # tests parameters
 TEST_ALL_COMBINATIONS = False
-TIME_FOR_STEADY_STATE = 1
+TIME_FOR_STEADY_STATE = 3
 TIME_TO_COUNT_FRAMES = 5
 
 

From 8b59278ef2f63933fe1b3b13495d8077121f0167 Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Mon, 6 Nov 2023 12:09:07 +0200
Subject: [PATCH 05/14] issues addressed, added auto exposure check

---
 .../live/frames/test-fps-permutations.py      | 36 +++++++++++--------
 1 file changed, 21 insertions(+), 15 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 1f21f9aa07..5fd20eb4c2 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -9,6 +9,7 @@
 # 8 choose 2 tests to do (one for each pair), plus one for all streams on
 
 from rspy import test, log
+import pyrealsense2 as rs
 import time
 import itertools
 import math
@@ -38,9 +39,9 @@ def get_resolution(profile):
     return profile.as_video_stream_profile().width(), profile.as_video_stream_profile().height()
 
 
-def check_fps_pair(fps, expected_fps):
+def check_fps_pair(measured_fps, expected_fps):
     delta_Hz = expected_fps * 0.05  # Validation KPI is 5%
-    return (fps <= (expected_fps + delta_Hz) and fps >= (expected_fps - delta_Hz))
+    return (measured_fps <= (expected_fps + delta_Hz) and measured_fps >= (expected_fps - delta_Hz))
 
 
 def get_expected_fps(sensor_profiles_dict):
@@ -130,13 +131,14 @@ def get_tested_profiles_string(sensor_profiles_dict):
 #############################################
 # ---------- Core Test Functions ---------- #
 #############################################
-def check_fps(expected_fps, fps_measured):
+def check_fps_dict(measured_fps, expected_fps):
     all_fps_ok = True
     for key in expected_fps:
-        res = check_fps_pair(fps_measured[key], expected_fps[key])
+        res = check_fps_pair(measured_fps[key], expected_fps[key])
         if not res:
             all_fps_ok = False
-            log.e(f"Expected {expected_fps[key]} fps, received {fps_measured[key]} fps in sensor {key.name}")
+        log.e(f"Expected {expected_fps[key]} fps, received {measured_fps[key]} fps in sensor {key.name}"
+              f" { '(Pass)' if res else '(Fail)' }")
     return all_fps_ok
 
 
@@ -181,11 +183,9 @@ def measure_fps(sensor_profiles_dict):
     # generate sensor-callable dictionary
     funcs_dict = generate_functions(sensor_fps_dict)
 
-    for key in sensor_profiles_dict:
-        sensor = key
-        profiles = sensor_profiles_dict[key]
+    for sensor, profiles in sensor_profiles_dict.items():
         sensor.open(profiles)
-        sensor.start(funcs_dict[key])
+        sensor.start(funcs_dict[sensor])
 
     # the core of the test - frames are counted during sleep when count_frames is on
     time.sleep(TIME_FOR_STEADY_STATE)
@@ -193,12 +193,11 @@ def measure_fps(sensor_profiles_dict):
     time.sleep(TIME_TO_COUNT_FRAMES)
     count_frames = False  # Stop counting
 
-    for key in sensor_profiles_dict:
-        sensor_fps_dict[key] /= len(sensor_profiles_dict[key])  # number of profiles on the sensor
-        sensor_fps_dict[key] /= TIME_TO_COUNT_FRAMES
+    for sensor, profiles in sensor_profiles_dict.items():
+        sensor_fps_dict[sensor] /= len(profiles)  # number of profiles on the sensor
+        sensor_fps_dict[sensor] /= TIME_TO_COUNT_FRAMES
         # now for each sensor we have the average fps received
 
-        sensor = key
         sensor.stop()
         sensor.close()
 
@@ -217,7 +216,6 @@ def get_test_details_str(sensor_profile_dict, expected_fps_dict):
                   f"{sensor_profile_dict[sensor_key][0].stream_name()} is {expected_fps_dict[sensor_key]}"
                   f" on {get_resolution(sensor_profile_dict[sensor_key][0])}\n")
     s = s.replace("on (0, 0)", "")  # remove no resolution for Motion Module profiles
-    s += "***************"
     return s
 
 
@@ -241,7 +239,7 @@ def perform_fps_test(sensor_profiles_arr, modes):
                 expected_fps = get_expected_fps(partial_dict)
                 log.d(get_test_details_str(partial_dict, expected_fps))
                 fps_dict = measure_fps(partial_dict)
-                test.check(check_fps(expected_fps, fps_dict))
+                test.check(check_fps_dict(fps_dict, expected_fps))
                 test.finish()
 
 
@@ -300,6 +298,14 @@ def get_sensors_and_profiles(device):
     """
     sensor_profiles_arr = []
     for sensor in device.query_sensors():
+        if sensor.is_depth_sensor() and sensor.supports(rs.option.enable_auto_exposure):
+            sensor.set_option(rs.option.enable_auto_exposure, 1)
+        if sensor.is_color_sensor():
+            if sensor.supports(rs.option.enable_auto_exposure):
+                sensor.set_option(rs.option.enable_auto_exposure, 1)
+            if sensor.supports(rs.option.auto_exposure_priority):
+                sensor.set_option(rs.option.auto_exposure_priority, 0)  # AE priority should be 0 for constant FPS
+
         profiles = get_mutual_resolution(sensor)
         for profile in profiles:
             sensor_profiles_arr.append((sensor, profile))

From 987d6044604296aef2a6236c064eb72201a6f896 Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Tue, 7 Nov 2023 16:52:14 +0200
Subject: [PATCH 06/14] log.e -> log.d

---
 unit-tests/live/frames/test-fps-permutations.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 5fd20eb4c2..08844c2053 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -137,7 +137,7 @@ def check_fps_dict(measured_fps, expected_fps):
         res = check_fps_pair(measured_fps[key], expected_fps[key])
         if not res:
             all_fps_ok = False
-        log.e(f"Expected {expected_fps[key]} fps, received {measured_fps[key]} fps in sensor {key.name}"
+        log.d(f"Expected {expected_fps[key]} fps, received {measured_fps[key]} fps in sensor {key.name}"
               f" { '(Pass)' if res else '(Fail)' }")
     return all_fps_ok
 

From 02b7ccbb4ca8f45f733294885a2acf5897c5c4bf Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Sun, 12 Nov 2023 10:34:49 +0200
Subject: [PATCH 07/14] python version required reduced

---
 unit-tests/live/frames/test-fps-permutations.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 08844c2053..a40b44621f 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -87,6 +87,11 @@ def get_dict_for_permutation(sensor_profiles_arr, permutation):
     return partial_dict
 
 
+# To reduce required python version, we implement choose instead of using math.comb
+def choose(n, k):
+    return math.factorial(n)/(math.factorial(k) * math.factorial(n - k))
+
+
 def get_time_est_string(num_profiles, modes):
     s = "Estimated time for test:"
     details_str = ""
@@ -100,8 +105,9 @@ def get_time_est_string(num_profiles, modes):
             test_time = math.factorial(num_profiles) * time_per_test
             details_str += f"{math.factorial(num_profiles)} tests for all permutations"
         elif mode == ALL_PAIRS:
-            test_time = math.comb(num_profiles, 2) * time_per_test
-            details_str += f"{math.comb(num_profiles,2)} tests for all pairs"
+            # test_time = math.comb(num_profiles, 2) * time_per_test
+            test_time = choose(num_profiles, 2) * time_per_test
+            details_str += f"{choose(num_profiles,2)} tests for all pairs"
         elif mode == ALL_SENSORS:
             test_time = time_per_test
             details_str += f"1 test for all sensors on"

From 84969a341df93a12d3bcc4fec50c195c5ea52d38 Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Sun, 12 Nov 2023 15:30:24 +0200
Subject: [PATCH 08/14] now testing per profile and not per sensor

---
 .../live/frames/test-fps-permutations.py      | 92 +++++++++----------
 1 file changed, 43 insertions(+), 49 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index a40b44621f..daba960a28 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -21,7 +21,7 @@
 # run perform_fps_test(sensor_profiles_array, [mode])
 ALL_PERMUTATIONS = 1
 ALL_PAIRS = 2
-ALL_SENSORS = 3
+ALL_STREAMS = 3
 
 # global variable used to count on all the sensors simultaneously
 count_frames = False
@@ -46,18 +46,12 @@ def check_fps_pair(measured_fps, expected_fps):
 
 def get_expected_fps(sensor_profiles_dict):
     """
-    For every sensor, find the expected fps according to its profiles
-    Return a dictionary between the sensor and its expected fps
+    Returns a dictionary between each profile and its expected fps
     """
     expected_fps_dict = {}
-    # log.d(sensor_profiles_dict)
-    for key in sensor_profiles_dict:
-        avg = 0
-        for profile in sensor_profiles_dict[key]:
-            avg += profile.fps()
-
-        avg /= len(sensor_profiles_dict[key])
-        expected_fps_dict[key] = avg
+    for sensor, profiles in sensor_profiles_dict.items():
+        for profile in profiles:
+            expected_fps_dict[profile.stream_name()] = profile.fps()
     return expected_fps_dict
 
 
@@ -69,7 +63,7 @@ def should_test(mode, permutation):
     If the mode is ALL_PAIRS return true only if there are exactly two streams to be tested
     """
     return ((mode == ALL_PERMUTATIONS) or
-            (mode == ALL_SENSORS and all(v == 1 for v in permutation)) or
+            (mode == ALL_STREAMS and all(v == 1 for v in permutation)) or
             (mode == ALL_PAIRS and permutation.count(1) == 2))
 
 
@@ -108,9 +102,9 @@ def get_time_est_string(num_profiles, modes):
             # test_time = math.comb(num_profiles, 2) * time_per_test
             test_time = choose(num_profiles, 2) * time_per_test
             details_str += f"{choose(num_profiles,2)} tests for all pairs"
-        elif mode == ALL_SENSORS:
+        elif mode == ALL_STREAMS:
             test_time = time_per_test
-            details_str += f"1 test for all sensors on"
+            details_str += f"1 test for all streams on"
 
         details_str += " + "
         total_time += test_time
@@ -139,31 +133,29 @@ def get_tested_profiles_string(sensor_profiles_dict):
 #############################################
 def check_fps_dict(measured_fps, expected_fps):
     all_fps_ok = True
-    for key in expected_fps:
-        res = check_fps_pair(measured_fps[key], expected_fps[key])
+    for profile_name in expected_fps:
+        res = check_fps_pair(measured_fps[profile_name], expected_fps[profile_name])
         if not res:
             all_fps_ok = False
-        log.d(f"Expected {expected_fps[key]} fps, received {measured_fps[key]} fps in sensor {key.name}"
+        log.d(f"Expected {expected_fps[profile_name]} fps, received {measured_fps[profile_name]} fps in profile"
+              f" {profile_name}"
               f" { '(Pass)' if res else '(Fail)' }")
     return all_fps_ok
 
 
-def generate_functions(sensor_profiles_dict):
+def generate_functions(sensor_profiles_dict, profile_name_fps_dict, profile_name_lock_dict):
     """
     Creates callable functions for each sensor to be triggered when a new frame arrives
     Used to count frames received for measuring fps
     """
-    locks = {}
     sensor_function_dict = {}
     for sensor_key in sensor_profiles_dict:
-        new_lock = threading.Lock()
-        locks[sensor_key] = new_lock
-
-        def on_frame_received(frame, key=sensor_key):
+        def on_frame_received(frame):  # variables declared on generate_functions should not be used here
             global count_frames
             if count_frames:
-                with locks[key]:
-                    sensor_profiles_dict[key] += 1
+                profile_name = frame.profile.stream_name()
+                with profile_name_lock_dict[profile_name]:  # lock and count frame
+                    profile_name_fps_dict[profile_name] += 1
 
         sensor_function_dict[sensor_key] = on_frame_received
     return sensor_function_dict
@@ -171,9 +163,9 @@ def on_frame_received(frame, key=sensor_key):
 
 def measure_fps(sensor_profiles_dict):
     """
-    Given a dictionary of sensors and profiles to test, activate all sensors on the given profiles
+    Given a dictionary of sensors and profiles to test, activate all streams on the given profiles
     and measure fps
-    Return a dictionary of sensors and the fps measured for them
+    Return a dictionary of profiles and the fps measured for them
     """
     global TIME_FOR_STEADY_STATE
     global TIME_TO_COUNT_FRAMES
@@ -181,13 +173,16 @@ def measure_fps(sensor_profiles_dict):
     global count_frames
     count_frames = False
 
-    # initialize fps dict
-    sensor_fps_dict = {}
-    for key in sensor_profiles_dict:
-        sensor_fps_dict[key] = 0
+    # initialize fps and locks dict
+    profile_name_fps_dict = {}
+    profile_name_lock_dict = {}
+    for sensor, profiles in sensor_profiles_dict.items():
+        for profile in profiles:
+            profile_name_fps_dict[profile.stream_name()] = 0
+            profile_name_lock_dict[profile.stream_name()] = threading.Lock()
 
     # generate sensor-callable dictionary
-    funcs_dict = generate_functions(sensor_fps_dict)
+    funcs_dict = generate_functions(sensor_profiles_dict, profile_name_fps_dict, profile_name_lock_dict)
 
     for sensor, profiles in sensor_profiles_dict.items():
         sensor.open(profiles)
@@ -200,27 +195,23 @@ def measure_fps(sensor_profiles_dict):
     count_frames = False  # Stop counting
 
     for sensor, profiles in sensor_profiles_dict.items():
-        sensor_fps_dict[sensor] /= len(profiles)  # number of profiles on the sensor
-        sensor_fps_dict[sensor] /= TIME_TO_COUNT_FRAMES
-        # now for each sensor we have the average fps received
+        for profile in profiles:
+            profile_name_fps_dict[profile.stream_name()] /= TIME_TO_COUNT_FRAMES
 
         sensor.stop()
         sensor.close()
 
-    return sensor_fps_dict
+    return profile_name_fps_dict
 
 
-def get_test_details_str(sensor_profile_dict, expected_fps_dict):
+def get_test_details_str(sensor_profile_dict):
     s = ""
-    for sensor_key in sensor_profile_dict:
-        if len(sensor_profile_dict[sensor_key]) > 1:
-            s += f"Expected average fps for {sensor_key.name} is {expected_fps_dict[sensor_key]}:\n"
-            for profile in sensor_profile_dict[sensor_key]:
-                s += f"Profile {profile.stream_name()} expects {profile.fps()} fps on {get_resolution(profile)}\n"
-        else:
-            s += (f"Expected fps for sensor {sensor_key.name} on profile "
-                  f"{sensor_profile_dict[sensor_key][0].stream_name()} is {expected_fps_dict[sensor_key]}"
-                  f" on {get_resolution(sensor_profile_dict[sensor_key][0])}\n")
+    for sensor, profiles in sensor_profile_dict.items():
+        for profile in profiles:
+            s += (f"Expected fps for profile {profile.stream_name()} on sensor "
+                  f"{sensor.name} is {profile.fps()} "
+                  f"on {get_resolution(profile)}\n")
+
     s = s.replace("on (0, 0)", "")  # remove no resolution for Motion Module profiles
     return s
 
@@ -243,7 +234,7 @@ def perform_fps_test(sensor_profiles_arr, modes):
                 partial_dict = get_dict_for_permutation(sensor_profiles_arr, perm)
                 test.start("Testing", get_tested_profiles_string(partial_dict))
                 expected_fps = get_expected_fps(partial_dict)
-                log.d(get_test_details_str(partial_dict, expected_fps))
+                log.d(get_test_details_str(partial_dict))
                 fps_dict = measure_fps(partial_dict)
                 test.check(check_fps_dict(fps_dict, expected_fps))
                 test.finish()
@@ -261,6 +252,9 @@ def get_profiles_by_resolution(sensor, resolution, fps=None):
             if fps is None or p.fps() == fps:
                 # to avoid having a long run time, we don't choose the same stream more than once
                 if p.stream_type() not in stream_types_added:
+                    if p.stream_type() == rs.stream.infrared:
+                        if p.stream_index() != 1:
+                            continue  # on some devices, using Infrared 1 seems to have better fps than IR/IR2
                     profiles.append(p)
                     stream_types_added.append(p.stream_type())
     return profiles
@@ -288,7 +282,7 @@ def get_mutual_resolution(sensor):
         if len(profiles) == len(stream_resolutions_dict):
             return profiles
 
-    # if none found, try to find a resolution that all profiles have, on any fps (fps will be taken as an average later)
+    # if none found, try to find a resolution that all profiles have, on any fps
     for option in possible_combinations:
         profiles = get_profiles_by_resolution(sensor, option[0], None)
         if len(profiles) == len(stream_resolutions_dict):
@@ -327,7 +321,7 @@ def get_sensors_and_profiles(device):
 if TEST_ALL_COMBINATIONS:
     test_modes = [ALL_PERMUTATIONS]
 else:
-    test_modes = [ALL_PAIRS, ALL_SENSORS]
+    test_modes = [ALL_PAIRS, ALL_STREAMS]
 
 perform_fps_test(sensor_profiles_array, test_modes)
 

From 23caa53a5afe07968bad160d8f4b95b8d06b7a05 Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Tue, 12 Mar 2024 08:59:48 +0200
Subject: [PATCH 09/14] now trying to use fps-helper

---
 unit-tests/live/frames/fps_helper.py          |   8 +
 .../live/frames/test-fps-permutations.py      | 334 ++----------------
 2 files changed, 46 insertions(+), 296 deletions(-)

diff --git a/unit-tests/live/frames/fps_helper.py b/unit-tests/live/frames/fps_helper.py
index 2534b0d893..9e57f7cd23 100644
--- a/unit-tests/live/frames/fps_helper.py
+++ b/unit-tests/live/frames/fps_helper.py
@@ -74,9 +74,11 @@ def generate_callbacks(sensor_profiles_dict, profile_name_fps_dict):
     """
     def on_frame_received(frame):
         global count_frames
+        log.d(f"frame {frame.profile.stream_name()} #{profile_name_fps_dict[frame.profile.stream_name()] + 1} accepted") # todo remove these
         if count_frames:
             profile_name = frame.profile.stream_name()
             profile_name_fps_dict[profile_name] += 1
+        log.d(f"frame {frame.profile.stream_name()} #{profile_name_fps_dict[frame.profile.stream_name()] + 1} finished")
 
     sensor_function_dict = {sensor_key: on_frame_received for sensor_key in sensor_profiles_dict}
     return sensor_function_dict
@@ -161,3 +163,9 @@ def get_profile(sensor, stream, resolution=None, fps=None):
                 and (resolution is None or get_resolution(profile) == resolution)
                 and (fps is None or profile.fps() == fps)),
                 None)  # return None if no profile found
+
+
+def get_profiles(sensor, stream, resolution=None, fps=None):
+    return iter(profile for profile in sensor.profiles if profile.stream_type() == stream
+                and (resolution is None or get_resolution(profile) == resolution)
+                and (fps is None or profile.fps()))
\ No newline at end of file
diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index daba960a28..cca7bdd45a 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -8,288 +8,14 @@
 # timeout = ((8 choose 2)+1) * (TIME_FOR_STEADY_STATE + TIME_TO_COUNT_FRAMES)
 # 8 choose 2 tests to do (one for each pair), plus one for all streams on
 
-from rspy import test, log
+from rspy import test, log, tests_wrapper
+from itertools import combinations
 import pyrealsense2 as rs
-import time
-import itertools
-import math
-import threading
+import fps_helper
 
 
-# To add a new test mode (ex. all singles or triplets) only add it below and on should_test
-# add it in get_time_est_string for an initial time calculation
-# run perform_fps_test(sensor_profiles_array, [mode])
-ALL_PERMUTATIONS = 1
-ALL_PAIRS = 2
-ALL_STREAMS = 3
-
-# global variable used to count on all the sensors simultaneously
-count_frames = False
-
-# tests parameters
-TEST_ALL_COMBINATIONS = False
-TIME_FOR_STEADY_STATE = 3
-TIME_TO_COUNT_FRAMES = 5
-
-
-##########################################
-# ---------- Helper Functions ---------- #
-##########################################
-def get_resolution(profile):
-    return profile.as_video_stream_profile().width(), profile.as_video_stream_profile().height()
-
-
-def check_fps_pair(measured_fps, expected_fps):
-    delta_Hz = expected_fps * 0.05  # Validation KPI is 5%
-    return (measured_fps <= (expected_fps + delta_Hz) and measured_fps >= (expected_fps - delta_Hz))
-
-
-def get_expected_fps(sensor_profiles_dict):
-    """
-    Returns a dictionary between each profile and its expected fps
-    """
-    expected_fps_dict = {}
-    for sensor, profiles in sensor_profiles_dict.items():
-        for profile in profiles:
-            expected_fps_dict[profile.stream_name()] = profile.fps()
-    return expected_fps_dict
-
-
-def should_test(mode, permutation):
-    """
-    Returns true if the given permutation should be tested:
-    If the mode is ALL_PERMUTATIONS returns true (every permutation should be tested)
-    If the mode is ALL_SENSORS return true only if all streams are to be tested
-    If the mode is ALL_PAIRS return true only if there are exactly two streams to be tested
-    """
-    return ((mode == ALL_PERMUTATIONS) or
-            (mode == ALL_STREAMS and all(v == 1 for v in permutation)) or
-            (mode == ALL_PAIRS and permutation.count(1) == 2))
-
-
-def get_dict_for_permutation(sensor_profiles_arr, permutation):
-    """
-    Given an array of pairs (sensor, profile) and a permutation of the same length,
-    return a sensor-profiles dictionary containing the relevant profiles
-     - profile will be added only if there's 1 in the corresponding element in the permutation
-    """
-    partial_dict = {}
-    for i, j in enumerate(permutation):
-        if j == 1:
-            sensor = sensor_profiles_arr[i][0]
-            partial_dict[sensor] = partial_dict.get(sensor, []) + [sensor_profiles_arr[i][1]]
-    return partial_dict
-
-
-# To reduce required python version, we implement choose instead of using math.comb
-def choose(n, k):
-    return math.factorial(n)/(math.factorial(k) * math.factorial(n - k))
-
-
-def get_time_est_string(num_profiles, modes):
-    s = "Estimated time for test:"
-    details_str = ""
-    total_time = 0
-    global TIME_TO_COUNT_FRAMES
-    global TIME_FOR_STEADY_STATE
-    time_per_test = TIME_TO_COUNT_FRAMES + TIME_FOR_STEADY_STATE
-    for mode in modes:
-        test_time = 0
-        if mode == ALL_PERMUTATIONS:
-            test_time = math.factorial(num_profiles) * time_per_test
-            details_str += f"{math.factorial(num_profiles)} tests for all permutations"
-        elif mode == ALL_PAIRS:
-            # test_time = math.comb(num_profiles, 2) * time_per_test
-            test_time = choose(num_profiles, 2) * time_per_test
-            details_str += f"{choose(num_profiles,2)} tests for all pairs"
-        elif mode == ALL_STREAMS:
-            test_time = time_per_test
-            details_str += f"1 test for all streams on"
-
-        details_str += " + "
-        total_time += test_time
-
-    # Remove the last " + " for aesthetics
-    details_str = details_str[:-3]
-    details_str = f"({details_str})"
-    details_str += f" * {time_per_test} secs per test"
-    s = f"{s} {total_time} secs ({details_str})"
-    return s
-
-
-def get_tested_profiles_string(sensor_profiles_dict):
-    s = ""
-    for sensor in sensor_profiles_dict:
-        for profile in sensor_profiles_dict[sensor]:
-            s += sensor.name + " / " + profile.stream_name() + " + "
-
-    # Remove the last " + " for aesthetics
-    s = s[:-3]
-    return s
-
-
-#############################################
-# ---------- Core Test Functions ---------- #
-#############################################
-def check_fps_dict(measured_fps, expected_fps):
-    all_fps_ok = True
-    for profile_name in expected_fps:
-        res = check_fps_pair(measured_fps[profile_name], expected_fps[profile_name])
-        if not res:
-            all_fps_ok = False
-        log.d(f"Expected {expected_fps[profile_name]} fps, received {measured_fps[profile_name]} fps in profile"
-              f" {profile_name}"
-              f" { '(Pass)' if res else '(Fail)' }")
-    return all_fps_ok
-
-
-def generate_functions(sensor_profiles_dict, profile_name_fps_dict, profile_name_lock_dict):
-    """
-    Creates callable functions for each sensor to be triggered when a new frame arrives
-    Used to count frames received for measuring fps
-    """
-    sensor_function_dict = {}
-    for sensor_key in sensor_profiles_dict:
-        def on_frame_received(frame):  # variables declared on generate_functions should not be used here
-            global count_frames
-            if count_frames:
-                profile_name = frame.profile.stream_name()
-                with profile_name_lock_dict[profile_name]:  # lock and count frame
-                    profile_name_fps_dict[profile_name] += 1
-
-        sensor_function_dict[sensor_key] = on_frame_received
-    return sensor_function_dict
-
-
-def measure_fps(sensor_profiles_dict):
-    """
-    Given a dictionary of sensors and profiles to test, activate all streams on the given profiles
-    and measure fps
-    Return a dictionary of profiles and the fps measured for them
-    """
-    global TIME_FOR_STEADY_STATE
-    global TIME_TO_COUNT_FRAMES
-
-    global count_frames
-    count_frames = False
-
-    # initialize fps and locks dict
-    profile_name_fps_dict = {}
-    profile_name_lock_dict = {}
-    for sensor, profiles in sensor_profiles_dict.items():
-        for profile in profiles:
-            profile_name_fps_dict[profile.stream_name()] = 0
-            profile_name_lock_dict[profile.stream_name()] = threading.Lock()
-
-    # generate sensor-callable dictionary
-    funcs_dict = generate_functions(sensor_profiles_dict, profile_name_fps_dict, profile_name_lock_dict)
-
-    for sensor, profiles in sensor_profiles_dict.items():
-        sensor.open(profiles)
-        sensor.start(funcs_dict[sensor])
-
-    # the core of the test - frames are counted during sleep when count_frames is on
-    time.sleep(TIME_FOR_STEADY_STATE)
-    count_frames = True  # Start counting frames
-    time.sleep(TIME_TO_COUNT_FRAMES)
-    count_frames = False  # Stop counting
-
-    for sensor, profiles in sensor_profiles_dict.items():
-        for profile in profiles:
-            profile_name_fps_dict[profile.stream_name()] /= TIME_TO_COUNT_FRAMES
-
-        sensor.stop()
-        sensor.close()
-
-    return profile_name_fps_dict
-
-
-def get_test_details_str(sensor_profile_dict):
-    s = ""
-    for sensor, profiles in sensor_profile_dict.items():
-        for profile in profiles:
-            s += (f"Expected fps for profile {profile.stream_name()} on sensor "
-                  f"{sensor.name} is {profile.fps()} "
-                  f"on {get_resolution(profile)}\n")
-
-    s = s.replace("on (0, 0)", "")  # remove no resolution for Motion Module profiles
-    return s
-
-
-def perform_fps_test(sensor_profiles_arr, modes):
-
-    log.d(get_time_est_string(len(sensor_profiles_arr), modes))
-
-    for mode in modes:
-        # Generate a list of all possible combinations of 1s and 0s (permutations) in the length of the array
-        perms = list(itertools.product([0, 1], repeat=len(sensor_profiles_arr)))
-
-        # Go over every possible permutation and check if we should test it
-        # Each index in a permutation represents a profile, which will be tested only if the value in that index is 1
-        for perm in perms:
-            # log.d(perm)
-            if all(v == 0 for v in perm):
-                continue
-            if should_test(mode, perm):
-                partial_dict = get_dict_for_permutation(sensor_profiles_arr, perm)
-                test.start("Testing", get_tested_profiles_string(partial_dict))
-                expected_fps = get_expected_fps(partial_dict)
-                log.d(get_test_details_str(partial_dict))
-                fps_dict = measure_fps(partial_dict)
-                test.check(check_fps_dict(fps_dict, expected_fps))
-                test.finish()
-
-
-####################################################
-# ---------- Test Preparation Functions ---------- #
-####################################################
-def get_profiles_by_resolution(sensor, resolution, fps=None):
-    # resolution is a required parameter because on a sensor all active profiles must have the same resolution
-    profiles = []
-    stream_types_added = []
-    for p in sensor.get_stream_profiles():
-        if get_resolution(p) == resolution:
-            if fps is None or p.fps() == fps:
-                # to avoid having a long run time, we don't choose the same stream more than once
-                if p.stream_type() not in stream_types_added:
-                    if p.stream_type() == rs.stream.infrared:
-                        if p.stream_index() != 1:
-                            continue  # on some devices, using Infrared 1 seems to have better fps than IR/IR2
-                    profiles.append(p)
-                    stream_types_added.append(p.stream_type())
-    return profiles
-
-
-def get_mutual_resolution(sensor):
-    stream_resolutions_dict = {}  # a map between a stream type and all of its possible resolutions
-    possible_combinations = []
-    for profile in sensor.get_stream_profiles():
-        stream_type = profile.stream_type()
-        resolution = get_resolution(profile)
-        fps = profile.fps()
-
-        # d[key] = d.get(key, []) + [value] -> adds to the dictionary or appends if it exists
-        stream_resolutions_dict[stream_type] = stream_resolutions_dict.get(stream_type, []) + [resolution]
-
-        if (resolution, fps) not in possible_combinations:
-            possible_combinations.append((resolution, fps))
-
-    possible_combinations.sort(reverse=True)  # sort by resolution first, then by fps, so the best res and fps are first
-
-    # first, try to find a resolution and fps that all profiles have
-    for option in possible_combinations:
-        profiles = get_profiles_by_resolution(sensor, option[0], option[1])
-        if len(profiles) == len(stream_resolutions_dict):
-            return profiles
-
-    # if none found, try to find a resolution that all profiles have, on any fps
-    for option in possible_combinations:
-        profiles = get_profiles_by_resolution(sensor, option[0], None)
-        if len(profiles) == len(stream_resolutions_dict):
-            return profiles
-    # if reached here, then we couldn't find a resolution that all profiles have, so we can't test them together :(
-    log.f("Can't run test, sensor", sensor.name, "doesn't have a resolution for all profiles")
-    return []
+VGA_RESOLUTION = (640, 360)
+HD_RESOLUTION = (1280, 720)
 
 
 def get_sensors_and_profiles(device):
@@ -298,33 +24,49 @@ def get_sensors_and_profiles(device):
     """
     sensor_profiles_arr = []
     for sensor in device.query_sensors():
-        if sensor.is_depth_sensor() and sensor.supports(rs.option.enable_auto_exposure):
-            sensor.set_option(rs.option.enable_auto_exposure, 1)
-        if sensor.is_color_sensor():
+        profile = None
+        if sensor.is_depth_sensor():
+            if sensor.supports(rs.option.enable_auto_exposure):
+                sensor.set_option(rs.option.enable_auto_exposure, 1)
+            depth_resolutions = []
+            for p in sensor.get_stream_profiles():
+                res = fps_helper.get_resolution(p)
+                if res not in depth_resolutions:
+                    depth_resolutions.append(res)
+            for res in depth_resolutions:
+                depth = fps_helper.get_profile(sensor, rs.stream.depth, res)
+                irs = fps_helper.get_profiles(sensor, rs.stream.infrared, res)
+                ir = next(irs)
+                while ir is not None and ir.stream_index() != 1:
+                    ir = next(irs)
+                if ir and depth:
+                    print(ir, depth)
+                    sensor_profiles_arr.append((sensor, depth))
+                    sensor_profiles_arr.append((sensor, ir))
+                    break
+        elif sensor.is_color_sensor():
             if sensor.supports(rs.option.enable_auto_exposure):
                 sensor.set_option(rs.option.enable_auto_exposure, 1)
             if sensor.supports(rs.option.auto_exposure_priority):
                 sensor.set_option(rs.option.auto_exposure_priority, 0)  # AE priority should be 0 for constant FPS
+            profile = fps_helper.get_profile(sensor, rs.stream.color)
+        elif sensor.is_motion_sensor():
+            sensor_profiles_arr.append((sensor, fps_helper.get_profile(sensor, rs.stream.accel)))
+            sensor_profiles_arr.append((sensor, fps_helper.get_profile(sensor, rs.stream.gyro)))
 
-        profiles = get_mutual_resolution(sensor)
-        for profile in profiles:
+        if profile is not None:
             sensor_profiles_arr.append((sensor, profile))
-
     return sensor_profiles_arr
 
 
-#####################################################################################################
-
 dev = test.find_first_device_or_exit()
-sensor_profiles_array = get_sensors_and_profiles(dev)
-
-if TEST_ALL_COMBINATIONS:
-    test_modes = [ALL_PERMUTATIONS]
-else:
-    test_modes = [ALL_PAIRS, ALL_STREAMS]
 
-perform_fps_test(sensor_profiles_array, test_modes)
-
-#####################################################################################################
+sensor_profiles_array = get_sensors_and_profiles(dev)
+# print([profile for _, profile in sensor_profiles_array])
+all_pairs = [[a[1].stream_name(), b[1].stream_name()] for a, b in combinations(sensor_profiles_array, 2)]
+all_streams = [[profile.stream_name() for _, profile in sensor_profiles_array]]
+permutations_to_run = all_pairs + all_streams
+# print(permutations_to_run)
+fps_helper.perform_fps_test(sensor_profiles_array, permutations_to_run)
 
 test.print_results_and_exit()

From ac11751b6340f157c8485666ac8abfcb39cde31e Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Wed, 13 Mar 2024 11:41:06 +0200
Subject: [PATCH 10/14] removed all streams test

---
 unit-tests/live/frames/test-fps-permutations.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index cca7bdd45a..07c3f71d29 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -64,9 +64,8 @@ def get_sensors_and_profiles(device):
 sensor_profiles_array = get_sensors_and_profiles(dev)
 # print([profile for _, profile in sensor_profiles_array])
 all_pairs = [[a[1].stream_name(), b[1].stream_name()] for a, b in combinations(sensor_profiles_array, 2)]
-all_streams = [[profile.stream_name() for _, profile in sensor_profiles_array]]
-permutations_to_run = all_pairs + all_streams
-# print(permutations_to_run)
+# all_streams = [[profile.stream_name() for _, profile in sensor_profiles_array]] # at the moment, this fails on CI
+permutations_to_run = all_pairs #+ all_streams
 fps_helper.perform_fps_test(sensor_profiles_array, permutations_to_run)
 
 test.print_results_and_exit()

From 0ae82ac76118363a4e6c8b55d7d304a6f77bb33d Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Wed, 19 Jun 2024 17:00:00 +0300
Subject: [PATCH 11/14] increase KPI to 10%, exlcude D457

---
 unit-tests/live/frames/fps_helper.py            | 2 +-
 unit-tests/live/frames/test-fps-permutations.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/unit-tests/live/frames/fps_helper.py b/unit-tests/live/frames/fps_helper.py
index 9e57f7cd23..a4a73d2d4d 100644
--- a/unit-tests/live/frames/fps_helper.py
+++ b/unit-tests/live/frames/fps_helper.py
@@ -15,7 +15,7 @@
 # ---------- Helper Functions ---------- #
 ##########################################
 def check_fps_pair(measured_fps, expected_fps):
-    delta_Hz = expected_fps * 0.05  # Validation KPI is 5%
+    delta_Hz = expected_fps * 0.1  # Validation KPI is 10%
     return (measured_fps <= (expected_fps + delta_Hz) and measured_fps >= (expected_fps - delta_Hz))
 
 
diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 07c3f71d29..2ee4baaeb8 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -1,7 +1,8 @@
 # License: Apache 2.0. See LICENSE file in root directory.
 # Copyright(c) 2023 Intel Corporation. All Rights Reserved.
 
-# test:device D400*
+# test:device D400* !D457
+# Currently, we exclude D457 as it's failing
 # test:donotrun:!nightly
 # test:timeout 300
 # timeout - on the worst case, we're testing on D585S, which have 8 streams, so:

From 4a0f4ef034f34bddab390dc5f9995cd7077a9761 Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Mon, 8 Jul 2024 14:20:31 +0300
Subject: [PATCH 12/14] edit comments

---
 unit-tests/live/frames/test-fps-permutations.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/unit-tests/live/frames/test-fps-permutations.py b/unit-tests/live/frames/test-fps-permutations.py
index 2ee4baaeb8..c54bdac779 100644
--- a/unit-tests/live/frames/test-fps-permutations.py
+++ b/unit-tests/live/frames/test-fps-permutations.py
@@ -5,7 +5,7 @@
 # Currently, we exclude D457 as it's failing
 # test:donotrun:!nightly
 # test:timeout 300
-# timeout - on the worst case, we're testing on D585S, which have 8 streams, so:
+# timeout - on the worst case we have 8 streams, so:
 # timeout = ((8 choose 2)+1) * (TIME_FOR_STEADY_STATE + TIME_TO_COUNT_FRAMES)
 # 8 choose 2 tests to do (one for each pair), plus one for all streams on
 
@@ -63,7 +63,6 @@ def get_sensors_and_profiles(device):
 dev = test.find_first_device_or_exit()
 
 sensor_profiles_array = get_sensors_and_profiles(dev)
-# print([profile for _, profile in sensor_profiles_array])
 all_pairs = [[a[1].stream_name(), b[1].stream_name()] for a, b in combinations(sensor_profiles_array, 2)]
 # all_streams = [[profile.stream_name() for _, profile in sensor_profiles_array]] # at the moment, this fails on CI
 permutations_to_run = all_pairs #+ all_streams

From 58a93a974cdadfb58a1703793ca172278ea1b4d5 Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Mon, 15 Jul 2024 09:33:28 +0300
Subject: [PATCH 13/14] test KPI 15%

---
 unit-tests/live/frames/fps_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unit-tests/live/frames/fps_helper.py b/unit-tests/live/frames/fps_helper.py
index a4a73d2d4d..5dce1a8436 100644
--- a/unit-tests/live/frames/fps_helper.py
+++ b/unit-tests/live/frames/fps_helper.py
@@ -15,7 +15,7 @@
 # ---------- Helper Functions ---------- #
 ##########################################
 def check_fps_pair(measured_fps, expected_fps):
-    delta_Hz = expected_fps * 0.1  # Validation KPI is 10%
+    delta_Hz = expected_fps * 0.15  # Validation KPI is 15%
     return (measured_fps <= (expected_fps + delta_Hz) and measured_fps >= (expected_fps - delta_Hz))
 
 

From 1bd56c343a428e56ecfaef4d9b0412529bcdce9d Mon Sep 17 00:00:00 2001
From: Avia Avraham <145359432+AviaAv@users.noreply.github.com>
Date: Mon, 15 Jul 2024 11:37:03 +0300
Subject: [PATCH 14/14] remove comment

---
 unit-tests/live/frames/fps_helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/unit-tests/live/frames/fps_helper.py b/unit-tests/live/frames/fps_helper.py
index 5dce1a8436..b24d4064ef 100644
--- a/unit-tests/live/frames/fps_helper.py
+++ b/unit-tests/live/frames/fps_helper.py
@@ -15,7 +15,7 @@
 # ---------- Helper Functions ---------- #
 ##########################################
 def check_fps_pair(measured_fps, expected_fps):
-    delta_Hz = expected_fps * 0.15  # Validation KPI is 15%
+    delta_Hz = expected_fps * 0.15
     return (measured_fps <= (expected_fps + delta_Hz) and measured_fps >= (expected_fps - delta_Hz))