diff --git a/irobot_benchmark/CMakeLists.txt b/irobot_benchmark/CMakeLists.txt index 63b1e08a..7c938fe7 100644 --- a/irobot_benchmark/CMakeLists.txt +++ b/irobot_benchmark/CMakeLists.txt @@ -39,19 +39,10 @@ install(FILES ${TOPOLOGY_FILES} DESTINATION lib/${PROJECT_NAME}/topology) -set(SCRIPTS - scripts/compare_latency.py - scripts/plot_latency.py - scripts/plot_cpu_memory.py -) - -install(FILES - ${SCRIPTS} - DESTINATION lib/${PROJECT_NAME}/scripts) - -# Install the entire comms_benchmark directory +# Install the entire comms_benchmark directory, preserving file permissions install(DIRECTORY comms_benchmark/ DESTINATION lib/${PROJECT_NAME}/comms_benchmark + USE_SOURCE_PERMISSIONS ) if(BUILD_TESTING) diff --git a/irobot_benchmark/comms_benchmark/profiles/zero-copy-shm.xml b/irobot_benchmark/comms_benchmark/profiles/shared_memory_cyclonedds.xml similarity index 100% rename from irobot_benchmark/comms_benchmark/profiles/zero-copy-shm.xml rename to irobot_benchmark/comms_benchmark/profiles/shared_memory_cyclonedds.xml diff --git a/irobot_benchmark/comms_benchmark/scripts/actions/run_mix_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/actions/run_mix_process_benchmarks.sh deleted file mode 100644 index 1fdd4e00..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/actions/run_mix_process_benchmarks.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -SCALING_GOVERNOR=$(cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor") - -# Check if the governor is set to 'performance' -if [ "$SCALING_GOVERNOR" != "performance" ]; then - echo "Warning: CPU scaling governor is not set to 'performance'. Current value is '$SCALING_GOVERNOR'." - echo "run: 'echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor'" -fi - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/actions" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -MP="${PWD}/actions-results_mix_process/" -rm -rf $MP && mkdir -p $MP - -# Define message sizes and communication types -comms=("ipc_off_fastdds" "ipc_off_cyclonedds") - -# Name of the results folders -results=("10b" "100kb" "1mb" "4mb") - -# Case 1: Define topologies pairs -# topology1=("srv_10b" "srv_100kb" "srv_1mb" "srv_4mb") -# Somehow an external host should synchronize and run also "remote_topology" -# The script "run_single_process_benchmarks could be of help for that" -# remote_topology=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") - - -# Case 2: Define topologies pairs -topology1=("cli_srv_10b" "cli_srv_100kb" "cli_srv_1mb" "cli_srv_4mb") -topology2=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") -# remote_topology=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") - - -# Loop through each index in the topology1 array -for i in "${!topology1[@]}"; do - t1=${topology1[i]} - t2=${topology2[i]} - res=${results[i]} - - for comm in "${comms[@]}"; do - result_folder="$MP/${res}/${comm}" - mkdir -p $result_folder - - # Set environment variables - if [ "$comm" == "ipc_off_fastdds" ]; then - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - else - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - fi - - cli_srv="${topologies_dir}/${t1}.json" - only_client="${topologies_dir}/${t2}.json" - - # Run the command - COMMAND="${irobot_benchmark} $cli_srv $only_client -x 3 --ipc off -t 3 -s 1000 --csv-out on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - # Move log folders - mv *log $result_folder - - # Unset environment variables after running "loaned_fastdds" command - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - unset FASTRTPS_DEFAULT_PROFILES_FILE - unset RMW_FASTRTPS_USE_QOS_FROM_XML - unset CYCLONEDDS_URI - fi - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/actions/run_multi_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/actions/run_multi_process_benchmarks.sh deleted file mode 100644 index da4b6194..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/actions/run_multi_process_benchmarks.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -SCALING_GOVERNOR=$(cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor") - -# Check if the governor is set to 'performance' -if [ "$SCALING_GOVERNOR" != "performance" ]; then - echo "Warning: CPU scaling governor is not set to 'performance'. Current value is '$SCALING_GOVERNOR'." - echo "run: 'echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor'" -fi - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/actions" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -MP="${PWD}/actions-results_multi_process/" -rm -rf $MP && mkdir -p $MP - -# Define communication types -comms=("ipc_off_fastdds" "ipc_off_cyclonedds") - -# Define topologies pairs -results=("10b" "100kb" "1mb" "4mb") -topology1=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") -topology2=("srv_10b" "srv_100kb" "srv_1mb" "srv_4mb") - -# Loop through each index in the topology1 array -for i in "${!topology1[@]}"; do - t1=${topology1[i]} - t2=${topology2[i]} - res=${results[i]} - - for comm in "${comms[@]}"; do - mkdir -p $MP/${res}/${comm} - - # Set environment variables - if [ "$comm" == "ipc_off_fastdds" ]; then - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - else - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - fi - - # Topology - cli_top="${topologies_dir}/${t1}.json" - srv_top="${topologies_dir}/${t2}.json" - - # Results folder - - # Run the command - COMMAND="${irobot_benchmark} $cli_top $srv_top -x 3 --ipc off -t 3 -s 1000 --csv-out on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - # Move log folders - mv *log $MP/${res}/${comm} - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/actions/run_single_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/actions/run_single_process_benchmarks.sh deleted file mode 100644 index c81eb5c1..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/actions/run_single_process_benchmarks.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -SCALING_GOVERNOR=$(cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor") - -# Check if the governor is set to 'performance' -if [ "$SCALING_GOVERNOR" != "performance" ]; then - echo "Warning: CPU scaling governor is not set to 'performance'. Current value is '$SCALING_GOVERNOR'." - echo "run: 'echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor'" -fi - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/actions" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -SP="${PWD}/actions-results_single_process/" -rm -rf $SP && mkdir -p $SP - -# Define communication types -comms=("ipc_on" "ipc_off_fastdds" "ipc_off_cyclonedds") - -topologies=( - "cli_srv_10b" - "cli_srv_100kb" - "cli_srv_1mb" - "cli_srv_4mb" -) - -# Loop through each combination of topology and communication type -for topology in "${topologies[@]}"; do - mkdir -p $SP/${topology} - - for comm in "${comms[@]}"; do - # Set environment variables - if [ "$comm" == "ipc_off_cyclonedds" ]; then - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - else - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - fi - - if [ "$comm" == "ipc_on" ]; then - ipc_option="--ipc on" - else - ipc_option="--ipc off" - fi - - top="${topologies_dir}/${topology}.json" - - # Results folder - result_folder=${comm} - - # Run the command - COMMAND="${irobot_benchmark} $top -x 3 $ipc_option -t 3 -s 1000 --csv-out on --results-dir $result_folder" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - mv $result_folder $SP/${topology} - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_mix_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_mix_process_benchmarks.sh deleted file mode 100644 index 91a634c4..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_mix_process_benchmarks.sh +++ /dev/null @@ -1,75 +0,0 @@ -#!/bin/bash - -SCALING_GOVERNOR=$(cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor") - -# Check if the governor is set to 'performance' -if [ "$SCALING_GOVERNOR" != "performance" ]; then - echo "Warning: CPU scaling governor is not set to 'performance'. Current value is '$SCALING_GOVERNOR'." - echo "run: 'echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor'" -fi - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/cli-srv" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -MP="${PWD}/cli-srv-results_mix_process/" -rm -rf $MP && mkdir -p $MP - -# Define message sizes and communication types -comms=("ipc_off_fastdds" "ipc_off_cyclonedds") - -# Name of the results folders -results=("10b" "100kb" "1mb" "4mb") - -# Case 1: Define topologies pairs -# topology1=("srv_10b" "srv_100kb" "srv_1mb" "srv_4mb") -# Somehow an external host should synchronize and run also "remote_topology" -# The script "run_single_process_benchmarks could be of help for that" -# remote_topology=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") - - -# Case 2: Define topologies pairs -topology1=("cli_srv_10b" "cli_srv_100kb" "cli_srv_1mb" "cli_srv_4mb") -topology2=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") -# remote_topology=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") - - -# Loop through each index in the topology1 array -for i in "${!topology1[@]}"; do - t1=${topology1[i]} - t2=${topology2[i]} - res=${results[i]} - - for comm in "${comms[@]}"; do - result_folder="$MP/${res}/${comm}" - mkdir -p $result_folder - - # Set environment variables - if [ "$comm" == "ipc_off_fastdds" ]; then - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - else - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - fi - - cli_srv="${topologies_dir}/${t1}.json" - only_client="${topologies_dir}/${t2}.json" - - # Run the command - COMMAND="${irobot_benchmark} $cli_srv $only_client -x 3 --ipc off -t 3 -s 1000 --csv-out on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - # Move log folders - mv *log $result_folder - - # Unset environment variables after running "loaned_fastdds" command - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - unset FASTRTPS_DEFAULT_PROFILES_FILE - unset RMW_FASTRTPS_USE_QOS_FROM_XML - unset CYCLONEDDS_URI - fi - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_multi_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_multi_process_benchmarks.sh deleted file mode 100644 index 5b9c970c..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_multi_process_benchmarks.sh +++ /dev/null @@ -1,59 +0,0 @@ -#!/bin/bash - -SCALING_GOVERNOR=$(cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor") - -# Check if the governor is set to 'performance' -if [ "$SCALING_GOVERNOR" != "performance" ]; then - echo "Warning: CPU scaling governor is not set to 'performance'. Current value is '$SCALING_GOVERNOR'." - echo "run: 'echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor'" -fi - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/cli-srv" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -MP="${PWD}/cli-srv-results_multi_process/" -rm -rf $MP && mkdir -p $MP - -# Define communication types -comms=("ipc_off_fastdds" "ipc_off_cyclonedds") - -# Define topologies pairs -results=("10b" "100kb" "1mb" "4mb" "1mb_multi") -topology1=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb" "10_cli_1mb") -topology2=("srv_10b" "srv_100kb" "srv_1mb" "srv_4mb" "srv_1mb") - -# Loop through each index in the topology1 array -for i in "${!topology1[@]}"; do - t1=${topology1[i]} - t2=${topology2[i]} - res=${results[i]} - - for comm in "${comms[@]}"; do - mkdir -p $MP/${res}/${comm} - - # Set environment variables - if [ "$comm" == "ipc_off_fastdds" ]; then - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - else - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - fi - - # Topology - cli_top="${topologies_dir}/${t1}.json" - srv_top="${topologies_dir}/${t2}.json" - - # Results folder - - # Run the command - COMMAND="${irobot_benchmark} $cli_top $srv_top -x 3 --ipc off -t 3 -s 1000 --csv-out on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - # Move log folders - mv *log $MP/${res}/${comm} - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_single_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_single_process_benchmarks.sh deleted file mode 100644 index 11ce69d6..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/cli-srv/run_single_process_benchmarks.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/bin/bash - -SCALING_GOVERNOR=$(cat "/sys/devices/system/cpu/cpufreq/policy0/scaling_governor") - -# Check if the governor is set to 'performance' -if [ "$SCALING_GOVERNOR" != "performance" ]; then - echo "Warning: CPU scaling governor is not set to 'performance'. Current value is '$SCALING_GOVERNOR'." - echo "run: 'echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor'" -fi - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/cli-srv" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -SP="${PWD}/cli-srv-results_single_process/" -rm -rf $SP && mkdir -p $SP - -# Define communication types -comms=("ipc_on" "ipc_off_fastdds" "ipc_off_cyclonedds") - -topologies=( - "cli_srv_10b" - "cli_srv_100kb" - "cli_srv_1mb" - "cli_srv_4mb" - "10_cli_srv_1mb" -) - -# Loop through each combination of topology and communication type -for topology in "${topologies[@]}"; do - mkdir -p $SP/${topology} - - for comm in "${comms[@]}"; do - # Set environment variables - if [ "$comm" == "ipc_off_cyclonedds" ]; then - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - else - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - fi - - if [ "$comm" == "ipc_on" ]; then - ipc_option="--ipc on" - else - ipc_option="--ipc off" - fi - - top="${topologies_dir}/${topology}.json" - - # Results folder - result_folder=${comm} - - # Run the command - COMMAND="${irobot_benchmark} $top -x 3 $ipc_option -t 3 -s 1000 --csv-out on --results-dir $result_folder" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - mv $result_folder $SP/${topology} - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/compare_all_metrics.py b/irobot_benchmark/comms_benchmark/scripts/compare_all_metrics.py new file mode 100755 index 00000000..b78fcc87 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/compare_all_metrics.py @@ -0,0 +1,104 @@ +import sys +import os +import pandas as pd + +def print_help(): + """Print usage and help information.""" + print(""" +Usage: python3 compare_all_metrics.py [--threshold ] + +Arguments: + Path to the first results directory (must contain average_metrics.csv). + Path to the second results directory (must contain average_metrics.csv). + --threshold Optional percentage threshold. Only metrics with differences greater than this threshold or less than the negative of it are displayed in the console. + +Description: + This script compares metrics between two results directories. It calculates the percentage change for each metric and generates a CSV file with the results. If a threshold is specified, metrics exceeding this difference (positively or negatively) are displayed in the console along with the detailed line diff and original values. + +Example: + python3 compare_all_metrics.py results1 results2 + python3 compare_all_metrics.py results1 results2 --threshold 50 + +Output: + - A CSV file named 'comparison_metrics.csv' with all percentage differences. + - Console output of metrics with differences exceeding the specified threshold (if provided). +""") + sys.exit(0) + +def calculate_percentage_change(value1, value2): + """Calculate percentage change from value1 to value2.""" + if value1 == 0: + return "Infinity" if value2 != 0 else "0" + return ((value2 - value1) / value1) * 100 + +# Handle help flag +if "--help" in sys.argv or "-h" in sys.argv: + print_help() + +# Check command-line arguments +if len(sys.argv) < 3 or len(sys.argv) > 5: + print("Error: Invalid arguments.") + print("Use --help or -h for usage instructions.") + sys.exit(1) + +results_dir1 = sys.argv[1] +results_dir2 = sys.argv[2] + +# Optional threshold +threshold = None +if "--threshold" in sys.argv: + try: + threshold_index = sys.argv.index("--threshold") + 1 + threshold = float(sys.argv[threshold_index]) + except (IndexError, ValueError): + print("Error: Invalid threshold value. Provide a valid number after --threshold.") + sys.exit(1) + +# Paths to average_metrics.csv in both directories +metrics_file1 = os.path.join(results_dir1, "average_metrics.csv") +metrics_file2 = os.path.join(results_dir2, "average_metrics.csv") + +if not os.path.exists(metrics_file1) or not os.path.exists(metrics_file2): + print(f"Error: average_metrics.csv not found in one or both directories: {results_dir1}, {results_dir2}") + sys.exit(1) + +# Load the data +df1 = pd.read_csv(metrics_file1) +df2 = pd.read_csv(metrics_file2) + +# Merge the two dataframes on the Directory column +merged = pd.merge(df1, df2, on="Directory", suffixes=("1", "2")) + +# Calculate percentage change for each metric +merged["Average_CPU %"] = merged.apply(lambda row: round(calculate_percentage_change(row["Average_CPU1"], row["Average_CPU2"]), 2), axis=1) +merged["Average_RSS_MB %"] = merged.apply(lambda row: round(calculate_percentage_change(row["Average_RSS_MB1"], row["Average_RSS_MB2"]), 2), axis=1) +merged["Average_VSZ_MB %"] = merged.apply(lambda row: round(calculate_percentage_change(row["Average_VSZ_MB1"], row["Average_VSZ_MB2"]), 2), axis=1) + +# Prepare the final result DataFrame +result = merged[["Directory", "Average_CPU %", "Average_RSS_MB %", "Average_VSZ_MB %"]].copy() + +# Save the result to a CSV file +output_file = "comparison_metrics.csv" +result.to_csv(output_file, index=False) +print(f"Comparison completed. Output saved to {output_file}") + +# Print metrics with differences exceeding the threshold +if threshold is not None: + print(f"\nMetrics with percentage differences exceeding {threshold}% or below {-threshold}%:\n") + filtered = merged[ + (merged["Average_CPU %"] > threshold) | (merged["Average_CPU %"] < -threshold) | + (merged["Average_RSS_MB %"] > threshold) | (merged["Average_RSS_MB %"] < -threshold) | + (merged["Average_VSZ_MB %"] > threshold) | (merged["Average_VSZ_MB %"] < -threshold) + ] + if not filtered.empty: + for _, row in filtered.iterrows(): + print(f"Directory: {row['Directory']}") + if abs(row["Average_CPU %"]) > threshold: + print(f" Average_CPU %: {row['Average_CPU %']}% (Val1={row['Average_CPU1']}, Val2={row['Average_CPU2']})") + if abs(row["Average_RSS_MB %"]) > threshold: + print(f" Average_RSS_MB %: {row['Average_RSS_MB %']}% (Val1={row['Average_RSS_MB1']}, Val2={row['Average_RSS_MB2']})") + if abs(row["Average_VSZ_MB %"]) > threshold: + print(f" Average_VSZ_MB %: {row['Average_VSZ_MB %']}% (Val1={row['Average_VSZ_MB1']}, Val2={row['Average_VSZ_MB2']})") + print("-" * 40) + else: + print("No metrics exceeded the specified threshold.") diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_actions.conf b/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_actions.conf new file mode 100644 index 00000000..c7880f44 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_actions.conf @@ -0,0 +1,17 @@ +# Configuration for ActionClient/ActionServer mix process benchmarks + +OUTPUT_DIR="actions_mix_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/actions" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Results and topologies +RESULTS=("10b" "100kb" "1mb" "4mb") +TOPOLOGY1=("cli_srv_10b" "cli_srv_100kb" "cli_srv_1mb" "cli_srv_4mb") +TOPOLOGY2=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") + +# Communication types +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and LOANED_ENV_VARS for each RMW +COMMS_fastrtps=("ipc_off") +COMMS_cyclonedds=("ipc_off") diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_cli_srv.conf b/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_cli_srv.conf new file mode 100644 index 00000000..b64e9afd --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_cli_srv.conf @@ -0,0 +1,17 @@ +# Configuration for Client/Service mix process benchmarks + +OUTPUT_DIR="cli-srv_mix_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/cli-srv" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Results and topologies +RESULTS=("10b" "100kb" "1mb" "4mb") +TOPOLOGY1=("cli_srv_10b" "cli_srv_100kb" "cli_srv_1mb" "cli_srv_4mb") +TOPOLOGY2=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") + +# Communication types +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and LOANED_ENV_VARS for each RMW +COMMS_fastrtps=("ipc_off") +COMMS_cyclonedds=("ipc_off") diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_pub_sub.conf b/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_pub_sub.conf new file mode 100644 index 00000000..34e4c5fb --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/mix_process_pub_sub.conf @@ -0,0 +1,25 @@ +# Configuration for Pub-Sub Mix-Process benchmarks + +OUTPUT_DIR="pub-sub_mix_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/pub-sub" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Results and topologies +RESULTS=("10b" "100kb" "1mb" "4mb" "sierra_nevada_fixed_size" "white_mountain_fixed_size") +TOPOLOGY1=("pub_sub_10b" "pub_sub_100kb" "pub_sub_1mb" "pub_sub_4mb" "sierra_nevada_fixed_size" "white_mountain_fixed_size") +TOPOLOGY2=("sub_10b" "sub_100kb" "sub_1mb" "sub_4mb" "debug_sierra_nevada_fixed_size" "debug_white_mountain_fixed_size") + +# Communication types +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and LOANED_ENV_VARS for each RMW +COMMS_fastrtps=("ipc_off" "loaned") +LOANED_ENV_VARS_fastrtps=( + "export FASTRTPS_DEFAULT_PROFILES_FILE=${PROFILES_DIR}/shared_memory_fastdds_preallocated_w_realloc.xml" + "export RMW_FASTRTPS_USE_QOS_FROM_XML=1" +) + +COMMS_cyclonedds=("ipc_off") +LOANED_ENV_VARS_cyclonedds=( + "export CYCLONEDDS_URI=${PROFILES_DIR}/shared_memory_cyclonedds.xml" +) diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_actions.conf b/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_actions.conf new file mode 100644 index 00000000..9f5db70f --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_actions.conf @@ -0,0 +1,17 @@ +# Configuration for ActionClient/ActionServer multi process benchmarks + +OUTPUT_DIR="actions_multi_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/actions" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Paired topologies +RESULTS=("10b" "100kb" "1mb" "4mb") +TOPOLOGY1=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb") +TOPOLOGY2=("srv_10b" "srv_100kb" "srv_1mb" "srv_4mb") + +# Communication types +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and LOANED_ENV_VARS for each RMW +COMMS_fastrtps=("ipc_off") +COMMS_cyclonedds=("ipc_off") diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_cli_srv.conf b/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_cli_srv.conf new file mode 100644 index 00000000..a3030b96 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_cli_srv.conf @@ -0,0 +1,17 @@ +# Configuration for Client/Service multi process benchmarks + +OUTPUT_DIR="cli-srv_multi_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/cli-srv" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Paired topologies +RESULTS=("10b" "100kb" "1mb" "4mb" "1mb_multi") +TOPOLOGY1=("cli_10b" "cli_100kb" "cli_1mb" "cli_4mb" "10_cli_1mb") +TOPOLOGY2=("srv_10b" "srv_100kb" "srv_1mb" "srv_4mb" "srv_1mb") + +# Communication types +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and LOANED_ENV_VARS for each RMW +COMMS_fastrtps=("ipc_off") +COMMS_cyclonedds=("ipc_off") diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_pub_sub.conf b/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_pub_sub.conf new file mode 100644 index 00000000..10650dd8 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/multi_process_pub_sub.conf @@ -0,0 +1,25 @@ +# Configuration for Pub-Sub Multi-Process benchmarks + +OUTPUT_DIR="pub-sub_multi_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/pub-sub" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Paired topologies +RESULTS=("10b" "100kb" "1mb" "4mb") +TOPOLOGY1=("pub_10b" "pub_100kb" "pub_1mb" "pub_4mb") +TOPOLOGY2=("sub_10b" "sub_100kb" "sub_1mb" "sub_4mb") + +# Communication types +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and LOANED_ENV_VARS for each RMW +COMMS_fastrtps=("ipc_off" "loaned") +LOANED_ENV_VARS_fastrtps=( + "export FASTRTPS_DEFAULT_PROFILES_FILE=${PROFILES_DIR}/shared_memory_fastdds_preallocated_w_realloc.xml" + "export RMW_FASTRTPS_USE_QOS_FROM_XML=1" +) + +COMMS_cyclonedds=("ipc_off") +LOANED_ENV_VARS_cyclonedds=( + "export CYCLONEDDS_URI=${PROFILES_DIR}/shared_memory_cyclonedds.xml" +) diff --git a/irobot_benchmark/comms_benchmark/scripts/multi-process/run_multi_process_benchmark.sh b/irobot_benchmark/comms_benchmark/scripts/multi-process/run_multi_process_benchmark.sh new file mode 100644 index 00000000..330b9a05 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/multi-process/run_multi_process_benchmark.sh @@ -0,0 +1,102 @@ +#!/bin/bash + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Load the configuration file +CONFIG_FILE=$1 +if [ ! -f "$CONFIG_FILE" ]; then + echo "Configuration file '$CONFIG_FILE' not found!" + exit 1 +fi + +if [ -z "${TEST_DURATION}" ]; then + TEST_DURATION=1 +fi + +# Define SCRIPT_DIR before sourcing the configuration file +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") +export SCRIPT_DIR + +# Source the configuration file +source "$CONFIG_FILE" + +# Load paths +GOVERNOR_SCRIPT="${SCRIPT_DIR}/../set_cpu_governor.sh" +IROBOT_BENCHMARK="${SCRIPT_DIR}/../../../irobot_benchmark" + +# Get the original governor +original_governor=$(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor) +# Set the CPU governor to performance +$GOVERNOR_SCRIPT performance +# Restore the original governor on exit +trap "$GOVERNOR_SCRIPT $original_governor" EXIT + +if [ $? -ne 0 ]; then + echo -e "\033[31m[ERROR] Failed to set CPU governor to performance. Exiting...\033[0m" + exit 1 +fi + +# Create a directory to store log folders +rm -rf "$OUTPUT_DIR" && mkdir -p "$OUTPUT_DIR" + +# Use colors for better visualisation +GREEN="\033[32m" +RESET="\033[0m" + +# Loop through RMW implementations +for RMW in "${RMW_LIST[@]}"; do + echo -e "${GREEN}Processing RMW: $RMW${RESET}" + + # Resolve COMMS and LOANED_ENV_VARS dynamically + declare -n COMMS="COMMS_${RMW}" + declare -n LOANED_ENV_VARS="LOANED_ENV_VARS_${RMW}" + echo -e "${GREEN} COMMS for $RMW: ${COMMS[@]}${RESET}" + echo -e "${GREEN} LOANED_ENV_VARS for $RMW: \n ${LOANED_ENV_VARS[@]}${RESET}" + + for COMM in "${COMMS[@]}"; do + echo -e "${GREEN} Testing COMM: $COMM${RESET}" + + # Set RMW implementation + export RMW_IMPLEMENTATION="rmw_${RMW}_cpp" + + # Handle loaned environment variables + if [[ "$COMM" == "loaned" ]]; then + for VAR in "${LOANED_ENV_VARS[@]}"; do + eval $VAR + done + fi + + # Loop through topology pairs + for i in "${!TOPOLOGY1[@]}"; do + T1="${TOPOLOGY1[i]}" + T2="${TOPOLOGY2[i]}" + RES="${RESULTS[i]}" + + RESULT_FOLDER="$OUTPUT_DIR/${RES}/${COMM}_${RMW}" + mkdir -p "$RESULT_FOLDER" + + # Set topology files + if [[ "$COMM" == "loaned" ]]; then + TOP1="${TOPOLOGIES_DIR}/${T1}_loaned.json" + else + TOP1="${TOPOLOGIES_DIR}/${T1}.json" + fi + TOP2="${TOPOLOGIES_DIR}/${T2}.json" + + # Run the command + COMMAND="${IROBOT_BENCHMARK} $TOP1 $TOP2 -x 3 --ipc off -t $TEST_DURATION -s 1000 --csv-out on" + echo -e "${GREEN}\nCommand: \n$COMMAND\n${RESET}" + eval $COMMAND + if [ $? -ne 0 ]; then + echo -e "\033[31m[ERROR] Command failed: $COMMAND\033[0m" + exit 1 + fi + + # Move log folders + mv *log "$RESULT_FOLDER" + done + done +done diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_latency.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_latency.py old mode 100644 new mode 100755 index 010ed06b..24634890 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_latency.py +++ b/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_latency.py @@ -1,89 +1,242 @@ import os -import sys import csv +import sys +import time import matplotlib.pyplot as plt -def calculate_average_stddev(csv_file): - with open(csv_file, 'r', newline='') as file: - reader = csv.DictReader(file) - mean_values = [] - for row in reader: - mean_values.append(float(row['mean_us'])) - if mean_values: - avg_mean = sum(mean_values) / len(mean_values) - return avg_mean +def get_sorted_files_by_mtime(directory): + # Find all latency_all.txt files in the directory and sort them by modification time + files = [os.path.join(root, file) for root, _, files in os.walk(directory) for file in files if file == 'latency_all.txt'] + return sorted(files, key=lambda f: os.path.getmtime(f)) + +def extract_sections(file_path): + """ + Extract different sections from the latency_all.txt file. + """ + with open(file_path, 'r') as f: + lines = f.readlines() + + sections = { + "subscriptions": [], + "publishers": [], + "clients": [], + "services": [], + "action_clients": [], + "action_servers": [] + } + + current_section = None + + for line in lines: + if "Action Clients stats:" in line: + current_section = "action_clients" + continue + elif "Action Servers stats:" in line: + current_section = "action_servers" + continue + elif "Clients stats:" in line: + current_section = "clients" + continue + elif "Services stats:" in line: + current_section = "services" + continue + elif "Subscriptions stats:" in line: + current_section = "subscriptions" + continue + elif "Publishers stats:" in line: + current_section = "publishers" + continue + + if current_section and line.strip(): + sections[current_section].append(line) + + return sections + +def calculate_average_from_section(lines): + """ + Calculate the average mean_us value from a section's lines. + """ + if not lines: + return None + + reader = csv.DictReader(lines) + mean_values = [] + for row in reader: + mean_us = row.get('mean_us') + if mean_us and mean_us.strip(): + try: + mean_values.append(float(mean_us)) + except ValueError: + print(f"Invalid 'mean_us' value: {mean_us}") else: - return None + print(f"Missing 'mean_us' in row: {row}") + + if mean_values: + return sum(mean_values) / len(mean_values) + return None def process_directory(directory): - pub_duration_csv = os.path.join(directory, "pub_duration.csv") - sub_latency_csv = os.path.join(directory, "sub_latency.csv") + """ + Process all latency_all.txt files in the directory and calculate the averages. + """ + results = [] + # Get sorted latency_all.txt files by modification time + sorted_files = get_sorted_files_by_mtime(directory) + for file_path in sorted_files: + if file_path.endswith('latency_all.txt'): + root = os.path.dirname(file_path) # Get root directory for cleaned path + sections = extract_sections(file_path) - avg_pub_duration = None - avg_sub_latency = None + avg_pub_duration = calculate_average_from_section(sections["publishers"]) + avg_sub_latency = calculate_average_from_section(sections["subscriptions"]) + avg_client_latency = calculate_average_from_section(sections["clients"]) + avg_service_latency = calculate_average_from_section(sections["services"]) + avg_action_client_latency = calculate_average_from_section(sections["action_clients"]) + avg_action_server_latency = calculate_average_from_section(sections["action_servers"]) - if os.path.exists(pub_duration_csv): - avg_pub_duration = calculate_average_stddev(pub_duration_csv) + clean_root = root.lstrip("./") # Remove leading "./" from the directory path + results.append({ + "Directory": clean_root, + "PubDur": avg_pub_duration, + "SubLat": avg_sub_latency, + "CliLat": avg_client_latency, + "SrvLat": avg_service_latency, + "ActionCliLat": avg_action_client_latency, + "ActionSrvLat": avg_action_server_latency + }) + return results - if os.path.exists(sub_latency_csv): - avg_sub_latency = calculate_average_stddev(sub_latency_csv) +def generate_average_latency_csv(results): + """ + Generate a CSV file containing average latency data for all directories and print its content to the console. + """ + with open("average_latency.csv", 'w', newline='') as csvfile: + fieldnames = [ + 'Directory', 'PubDur', 'SubLat', + 'CliLat', 'SrvLat', + 'ActionCliLat', 'ActionSrvLat' + ] + writer = csv.DictWriter(csvfile, fieldnames=fieldnames, delimiter=';') + writer.writeheader() - return avg_pub_duration, avg_sub_latency + # Print the header in the console + print(";".join(fieldnames)) -def generate_average_latency_data(directories): - pub_duration_data = [] - sub_latency_data = [] - for directory in directories: - avg_pub_duration, avg_sub_latency = process_directory(directory) - # Replace None values with zeros - pub_duration_data.append(round(avg_pub_duration, 2) if avg_pub_duration is not None else 0) - sub_latency_data.append(round(avg_sub_latency, 2) if avg_sub_latency is not None else 0) - return pub_duration_data, sub_latency_data - -def plot_publisher_latency(pub_duration_data, directories): - plt.figure(figsize=(8, 5)) - plt.bar(directories, pub_duration_data, color='skyblue') - plt.xlabel('Directory') - plt.ylabel('AvgPubDuration') - plt.title('Average Publishing Duration') - plt.tight_layout() - plt.show() + for result in results: + row = { + 'Directory': result["Directory"], + 'PubDur': round(result["PubDur"], 2) if result["PubDur"] is not None else '', + 'SubLat': round(result["SubLat"], 2) if result["SubLat"] is not None else '', + 'CliLat': round(result["CliLat"], 2) if result["CliLat"] is not None else '', + 'SrvLat': round(result["SrvLat"], 2) if result["SrvLat"] is not None else '', + 'ActionCliLat': round(result["ActionCliLat"], 2) if result["ActionCliLat"] is not None else '', + 'ActionSrvLat': round(result["ActionSrvLat"], 2) if result["ActionSrvLat"] is not None else '' + } + writer.writerow(row) + + # Print the row in the console + print(";".join(str(row[field]) for field in fieldnames)) + +def plot_latency_metrics(results): + """ + Plot the latency metrics for all directories in a single horizontal plot with different symbols. + """ + directories = [result["Directory"] for result in results] + metrics = { + "PubDur": [result["PubDur"] or 0 for result in results], + "SubLat": [result["SubLat"] or 0 for result in results], + "CliLat": [result["CliLat"] or 0 for result in results], + "SrvLat": [result["SrvLat"] or 0 for result in results], + "ActionCliLat": [result["ActionCliLat"] or 0 for result in results], + "ActionSrvLat": [result["ActionSrvLat"] or 0 for result in results], + } -def plot_subscription_latency(sub_latency_data, directories): - plt.figure(figsize=(8, 5)) - plt.bar(directories, sub_latency_data, color='lightgreen') - plt.xlabel('Directory') - plt.ylabel('AvgSubLatency') - plt.title('Average Subscription Latency') + # Assign each metric a unique color and symbol + styles = { + "PubDur": ("blue", "o"), # Circle + "SubLat": ("orange", "*"), # Star + "CliLat": ("green", "s"), # Square + "SrvLat": ("red", "D"), # Diamond + "ActionCliLat": ("purple", "^"), # Triangle Up + "ActionSrvLat": ("brown", "v"), # Triangle Down + } + + plt.figure(figsize=(12, 8)) + + # Plot dots for each metric, skipping zero values + for metric, values in metrics.items(): + color, marker = styles[metric] + filtered_values = [val for val in values if val > 0] + filtered_dirs = [directories[i] for i, val in enumerate(values) if val > 0] + plt.scatter(filtered_values, filtered_dirs, label=metric, color=color, marker=marker, s=50) + + # Add labels and title + plt.xlabel("Latency (us)") + plt.ylabel("Directory") + plt.title("Latency Metrics by Directory") + plt.legend(title="Metrics", loc="upper right") + plt.grid(True, axis="x", linestyle="--", alpha=0.7) plt.tight_layout() plt.show() -def generate_average_latency_csv(directories): - with open("average_latency.csv", 'w', newline='') as csvfile: - fieldnames = ['Directory', 'AvgPubDuration', 'AvgSubLatency'] - writer = csv.DictWriter(csvfile, fieldnames=fieldnames) - writer.writeheader() - for directory in directories: - avg_pub_duration, avg_sub_latency = process_directory(directory) - writer.writerow({ - 'Directory': directory, - 'AvgPubDuration': round(avg_pub_duration, 2) if avg_pub_duration is not None else '', - 'AvgSubLatency': round(avg_sub_latency, 2) if avg_sub_latency is not None else '' - }) + +def print_help(): + """ + Print help and usage instructions. + """ + print(""" +Usage: python compare_latency.py [--plot] + +Arguments: + Root directory to search for 'latency_all.txt' files. + --plot Optional flag to generate latency metric plots. + +Description: + This script processes all 'latency_all.txt' files in the specified directory (and its subdirectories), + calculates average latency metrics (for publishers, subscriptions, clients, services, action clients, and + action servers), and generates a CSV file (average_latency.csv) with these metrics. + If the --plot flag is provided, it also generates plots for the metrics. + +Example: + python compare_latency.py ./my_directory --plot +""") + sys.exit(0) if __name__ == "__main__": + if "--help" in sys.argv or "-h" in sys.argv: + print_help() + if len(sys.argv) < 2: - print("Usage: python script.py directory1 directory2 ...") + print("Error: Missing required arguments.") + print("Use --help or -h for usage instructions.") + sys.exit(1) + + # Extract directory arguments and optional flags + args = sys.argv[1:] + plot = "--plot" in args + directories = [arg for arg in args if arg != "--plot"] + + if not directories: + print("Error: No directories provided.") + sys.exit(1) + + all_results = [] + + for directory in directories: + if not os.path.isdir(directory): + print(f"Warning: '{directory}' is not a valid directory. Skipping.") + continue + results = process_directory(directory) + all_results.extend(results) + + if not all_results: + print("No valid data found to process.") sys.exit(1) - directories = sys.argv[1:] - generate_average_latency_csv(directories) + # Generate CSV + generate_average_latency_csv(all_results) - with open("average_latency.csv", 'r', newline='') as csvfile: - reader = csv.reader(csvfile) - for row in reader: - print(','.join(row)) + # Plot latency metrics if --plot is passed + if plot: + plot_latency_metrics(all_results) - pub_duration_data, sub_latency_data = generate_average_latency_data(directories) - plot_publisher_latency(pub_duration_data, directories) - plot_subscription_latency(sub_latency_data, directories) diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_metrics.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_metrics.py new file mode 100755 index 00000000..54f19adc --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/parse-results/compare_metrics.py @@ -0,0 +1,159 @@ +import os +import sys +import pandas as pd +import time +import numpy as np +import matplotlib.pyplot as plt + +def get_sorted_directories_by_mtime(directories): + # Sort directories by the modification time of resources.txt + return sorted(directories, key=lambda d: os.path.getmtime(os.path.join(d, "resources.txt"))) + +def find_directories_with_resources(root_dirs): + directories = [] + for root_dir in root_dirs: + for root, _, files in os.walk(root_dir): + if "resources.txt" in files: + # Remove ./ prefix from the path + clean_path = root.lstrip("./") + directories.append(clean_path) + return directories + +def process_directory(directory): + file_path = os.path.join(directory, "resources.txt") + + # Read the file + resources_data = pd.read_csv(file_path) + + # Extract CPU percentage, RSS memory, VSZ memory, and Latency + cpu_perc = resources_data['cpu_perc'].astype(float) + rss_KB = resources_data['rss_KB'].astype(float) + vsz_KB = resources_data['vsz_KB'].astype(float) + resources_data['latency_us'] = resources_data['latency_us'].apply(lambda x: 0 if x > 1000000000 else x) + latency_us = resources_data['latency_us'].astype(float) + time_ms = resources_data['time_ms'] + + return time_ms, cpu_perc, rss_KB, vsz_KB, latency_us + +def print_help(): + """ + Print usage instructions for the script. + """ + print(""" +Usage: python compare_metrics.py [ ...] [--plot] + +Arguments: + , , ... Directories to search for subdirectories containing 'resources.txt' files. + --plot Optional flag to generate plots for CPU, RSS, VSZ, and Latency metrics over time. + +Example: + python3 compare_metrics.py ./dir1 ./dir2 --plot +""") + sys.exit(0) + +# Parse command-line arguments for directories and optional plot +if "--help" in sys.argv or "-h" in sys.argv: + print_help() + +if len(sys.argv) < 2: + print("Error: Missing required arguments.") + print("Use --help or -h for usage instructions.") + sys.exit(1) + +args = sys.argv[1:] +plot = "--plot" in args +directories = [arg for arg in args if arg != "--plot"] + +if not directories: + print("Error: No directories provided.") + sys.exit(1) + +# Find directories with resources.txt in the specified directories +directories = find_directories_with_resources(directories) + +if not directories: + print("No directories with 'resources.txt' found in the provided directories.") + exit(1) + +# Sort directories by modification time (oldest to newest) +directories = get_sorted_directories_by_mtime(directories) + +# Lists to store data from all directories +all_time_ms = [] +all_cpu_perc = [] +all_rss_MB = [] +all_vsz_MB = [] +all_latency_us = [] + +# CSV output file +csv_output_file = "average_metrics.csv" + +# Process each directory and collect data +with open(csv_output_file, 'w') as csv_file: + csv_file.write("Directory;CPU;RSS_MB;VSZ_MB;Latency_us\n") + + for directory in directories: + time_ms, cpu_perc, rss_KB, vsz_KB, latency_us = process_directory(directory) + all_time_ms.append(time_ms) + all_cpu_perc.append(cpu_perc) + all_rss_MB.append(rss_KB / 1024) # Convert from KB to MB + all_vsz_MB.append(vsz_KB / 1024) # Convert from KB to MB + all_latency_us.append(latency_us) + + # Compute average metrics for the current directory + average_cpu = round(np.mean(cpu_perc), 2) + average_rss_MB = round(np.mean(rss_KB) / 1024, 2) # Convert from KB to MB + average_vsz_MB = round(np.mean(vsz_KB) / 1024, 2) # Convert from KB to MB + average_latency_us = round(np.mean(latency_us), 2) + + # Print and write average metrics in CSV format + csv_file.write(f"{directory.lstrip('./')};{average_cpu};{average_rss_MB};{average_vsz_MB};{average_latency_us}\n") + if directory == directories[0]: # Print column titles only once + print("Directory;CPU;RSS_MB;VSZ_MB;Latency_us") + print(f"{directory.lstrip('./')};{average_cpu};{average_rss_MB};{average_vsz_MB};{average_latency_us}") + +# Plotting if --plot is passed +if plot: + # Plot CPU usage for all directories + plt.figure(figsize=(8, 5)) + for time_ms, cpu_perc, directory in zip(all_time_ms, all_cpu_perc, directories): + plt.plot(time_ms, cpu_perc, label=f'CPU_{directory}') + plt.xlabel('Time (ms)') + plt.ylabel('CPU Percentage') + plt.title('CPU Usage over Time') + plt.grid(True) + plt.legend() + plt.show() + + # Plot RSS memory for all directories + plt.figure(figsize=(8, 5)) + for time_ms, rss_MB, directory in zip(all_time_ms, all_rss_MB, directories): + plt.plot(time_ms, rss_MB, label=f'RSS_{directory}') + plt.xlabel('Time (ms)') + plt.ylabel('RSS Memory (MB)') + plt.title('RSS Memory over Time') + plt.grid(True) + plt.legend() + plt.show() + + # Plot VSZ memory for all directories + plt.figure(figsize=(8, 5)) + for time_ms, vsz_MB, directory in zip(all_time_ms, all_vsz_MB, directories): + plt.plot(time_ms, vsz_MB, label=f'VSZ_{directory}') + plt.xlabel('Time (ms)') + plt.ylabel('VSZ Memory (MB)') + plt.title('VSZ Memory over Time') + plt.grid(True) + plt.legend() + plt.show() + + # Plot Latency for all directories + plt.figure(figsize=(8, 5)) + for time_ms, latency_us, directory in zip(all_time_ms, all_latency_us, directories): + plt.plot(time_ms, latency_us, label=f'Latency_{directory}') + plt.xlabel('Time (ms)') + plt.ylabel('Latency (us)') + plt.title('Latency over Time') + plt.grid(True) + plt.legend() + plt.show() diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_cpu_multi_process.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_cpu_multi_process.py deleted file mode 100755 index 58895e3f..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_cpu_multi_process.py +++ /dev/null @@ -1,115 +0,0 @@ -import pandas as pd -import sys - -def process_csv(csv_file): - # Read the CSV file - df = pd.read_csv(csv_file) - - # Initialize dictionary to store results - output_data_cpu = {} - output_data_rss = {} - output_data_vsz = {} - - # Iterate over each row - for index, row in df.iterrows(): - directory = row['Directory'] - size, comm, topology = directory.split('/')[:3] - key = f"{size}/{comm}" - pub_cpu = 0 - sub_cpu = 0 - pub_sub_cpu = 0 - debug_cpu = 0 - main_cpu = 0 - pub_rss = 0 - sub_rss = 0 - pub_sub_rss = 0 - debug_rss = 0 - main_rss = 0 - pub_vsz = 0 - sub_vsz = 0 - pub_sub_vsz = 0 - debug_vsz = 0 - main_vsz = 0 - - if 'debug' in topology: - debug_cpu = row['Average_CPU'] - debug_rss = row['Average_RSS_MB'] - debug_vsz = row['Average_VSZ_MB'] - elif 'pub' in topology: - if 'pub_sub' in topology: - pub_sub_cpu = row['Average_CPU'] - pub_sub_rss = row['Average_RSS_MB'] - pub_sub_vsz = row['Average_VSZ_MB'] - else: - pub_cpu = row['Average_CPU'] - pub_rss = row['Average_RSS_MB'] - pub_vsz = row['Average_VSZ_MB'] - elif 'sub' in topology: - sub_cpu = row['Average_CPU'] - sub_rss = row['Average_RSS_MB'] - sub_vsz = row['Average_VSZ_MB'] - else: - main_cpu = row['Average_CPU'] - main_rss = row['Average_RSS_MB'] - main_vsz = row['Average_VSZ_MB'] - - if key not in output_data_cpu: - output_data_cpu[key] = {'pub_cpu': 0, 'sub_cpu': 0, 'pub_sub_cpu': 0, 'debug_cpu': 0, 'main_cpu': 0} - output_data_cpu[key]['pub_cpu'] += pub_cpu - output_data_cpu[key]['sub_cpu'] += sub_cpu - output_data_cpu[key]['pub_sub_cpu'] += pub_sub_cpu - output_data_cpu[key]['debug_cpu'] += debug_cpu - output_data_cpu[key]['main_cpu'] += main_cpu - - if key not in output_data_rss: - output_data_rss[key] = {'pub_rss': 0, 'sub_rss': 0, 'pub_sub_rss': 0, 'debug_rss': 0, 'main_rss': 0} - output_data_rss[key]['pub_rss'] += pub_rss - output_data_rss[key]['sub_rss'] += sub_rss - output_data_rss[key]['pub_sub_rss'] += pub_sub_rss - output_data_rss[key]['debug_rss'] += debug_rss - output_data_rss[key]['main_rss'] += main_rss - - if key not in output_data_vsz: - output_data_vsz[key] = {'pub_vsz': 0, 'sub_vsz': 0, 'pub_sub_vsz': 0, 'debug_vsz': 0, 'main_vsz': 0} - output_data_vsz[key]['pub_vsz'] += pub_vsz - output_data_vsz[key]['sub_vsz'] += sub_vsz - output_data_vsz[key]['pub_sub_vsz'] += pub_sub_vsz - output_data_vsz[key]['debug_vsz'] += debug_vsz - output_data_vsz[key]['main_vsz'] += main_vsz - - # Initialize lists to store combined data for CPU, RSS, and VSZ - combined_data_cpu = [] - combined_data_rss = [] - combined_data_vsz = [] - - # Combine data into lists for CPU, RSS, and VSZ - for key, value in output_data_cpu.items(): - total_cpu = value['pub_cpu'] + value['sub_cpu'] + value['pub_sub_cpu'] + value['debug_cpu'] + value['main_cpu'] - combined_data_cpu.append([key, value['pub_cpu'], value['sub_cpu'], value['pub_sub_cpu'], value['debug_cpu'], value['main_cpu'], total_cpu]) - - for key, value in output_data_rss.items(): - total_rss = value['pub_rss'] + value['sub_rss'] + value['pub_sub_rss'] + value['debug_rss'] + value['main_rss'] - combined_data_rss.append([key, value['pub_rss'], value['sub_rss'], value['pub_sub_rss'], value['debug_rss'], value['main_rss'], total_rss]) - - for key, value in output_data_vsz.items(): - total_vsz = value['pub_vsz'] + value['sub_vsz'] + value['pub_sub_vsz'] + value['debug_vsz'] + value['main_vsz'] - combined_data_vsz.append([key, value['pub_vsz'], value['sub_vsz'], value['pub_sub_vsz'], value['debug_vsz'], value['main_vsz'], total_vsz]) - - # Create DataFrames for the output CSVs for CPU, RSS, and VSZ - output_df_cpu = pd.DataFrame(combined_data_cpu, columns=['Dir', 'pub_cpu', 'sub_cpu', 'pub_sub_cpu', 'debug_cpu', 'main_cpu', 'total']) - output_df_rss = pd.DataFrame(combined_data_rss, columns=['Dir', 'pub_rss', 'sub_rss', 'pub_sub_rss', 'debug_rss', 'main_rss', 'total']) - output_df_vsz = pd.DataFrame(combined_data_vsz, columns=['Dir', 'pub_vsz', 'sub_vsz', 'pub_sub_vsz', 'debug_vsz', 'main_vsz', 'total']) - - # Print the DataFrames as comma-separated values - print(output_df_cpu.to_csv(index=False)) - print(output_df_rss.to_csv(index=False)) - print(output_df_vsz.to_csv(index=False)) - - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python script.py ") - sys.exit(1) - - input_csv_file = sys.argv[1] - process_csv(input_csv_file) diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_cpu_single_process.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_cpu_single_process.py deleted file mode 100755 index bf805335..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_cpu_single_process.py +++ /dev/null @@ -1,41 +0,0 @@ -import csv -import sys - -def process_csv(input_file): - data = {} - with open(input_file, newline='') as csvfile: - reader = csv.DictReader(csvfile) - for row in reader: - directory = row['Directory'] - name, comm = directory.split('/')[-2:] - average_cpu = row['Average_CPU'] - average_rss_mb = row['Average_RSS_MB'] - average_vsz_mb = row['Average_VSZ_MB'] - if name not in data: - data[name] = {} - data[name][comm] = {'CPU': average_cpu, 'RSS': average_rss_mb, 'VSZ': average_vsz_mb} - - return data - -def print_output(data): - name_list = sorted(data.keys()) - comm_set = set() - for name in name_list: - for comm in data[name]: - comm_set.add(comm) - - for metric in ['CPU', 'RSS', 'VSZ']: - print('Topology,' + ','.join(sorted(comm_set))) - for name in name_list: - row = [metric + '_' + name] + [data[name][comm].get(metric, '') for comm in sorted(comm_set)] - print(','.join(row)) - print() # Print an empty line between different metrics - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python parse_averages_single_process.py average_metrics.csv") - sys.exit(1) - - input_file = sys.argv[1] - data = process_csv(input_file) - print_output(data) diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_latency_multi_process.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_latency_multi_process.py deleted file mode 100755 index 800cfab9..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_latency_multi_process.py +++ /dev/null @@ -1,25 +0,0 @@ -import pandas as pd -import sys - -def generate_summary(csv_file): - # Read the CSV file into a DataFrame - df = pd.read_csv(csv_file) - - # Extracting relevant columns - df['Directory'] = df['Directory'].str.split('/').str[0] - df['Category'] = df['Directory'].str.split('_').str[-1] - df['Sub'] = df['AvgSubLatency'] - df = df[['Directory', 'Sub', 'AvgPubDuration']] - - # Pivot the DataFrame - df_pivot = pd.pivot_table(df, values=['Sub', 'AvgPubDuration'], index='Directory', aggfunc='first') - - # Print the new DataFrame - print(df_pivot.fillna(0)) - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python script.py input_csv_file") - else: - csv_file = sys.argv[1] - generate_summary(csv_file) diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_latency_single_process.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_latency_single_process.py deleted file mode 100755 index 94a7ce64..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/parse_latency_single_process.py +++ /dev/null @@ -1,33 +0,0 @@ -import pandas as pd -import sys - -def generate_csv_tables(vcs_file): - # Read the VCS file into a DataFrame - df = pd.read_csv(vcs_file) - - # Split the 'Directory' column to extract topology and type - df[['Topology', 'Type']] = df['Directory'].str.split('/', expand=True) - - # Pivot the DataFrame to rearrange data for AvgSubLatency - avg_sub_latency_df = df.pivot(index='Type', columns='Topology', values='AvgSubLatency') - - # Pivot the DataFrame to rearrange data for AvgPubDuration - avg_pub_duration_df = df.pivot(index='Type', columns='Topology', values='AvgPubDuration') - - # Transpose the DataFrames - transposed_avg_sub_latency_df = avg_sub_latency_df.T - transposed_avg_pub_duration_df = avg_pub_duration_df.T - - # Print the transposed CSV tables to console - print("Transposed AvgSubLatency Table:") - print(transposed_avg_sub_latency_df.to_csv(sep=",")) - print("\nTransposed AvgPubDuration Table:") - print(transposed_avg_pub_duration_df.to_csv(sep=",")) - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python script.py input.vcs") - sys.exit(1) - - vcs_file = sys.argv[1] - generate_csv_tables(vcs_file) diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/plot_cpu_memory.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/plot_cpu_memory.py deleted file mode 100644 index 98f45602..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/plot_cpu_memory.py +++ /dev/null @@ -1,89 +0,0 @@ -import sys -import os -import pandas as pd -import numpy as np -import matplotlib.pyplot as plt - -def process_directory(directory): - file_path = os.path.join(directory, "resources.txt") - - # Read the file - resources_data = pd.read_csv(file_path) - - # Extract CPU percentage, RSS memory, and VSZ memory - cpu_perc = resources_data['cpu_perc'].astype(float) - rss_KB = resources_data['rss_KB'].astype(float) - vsz_KB = resources_data['vsz_KB'].astype(float) - time_ms = resources_data['time_ms'] - - return time_ms, cpu_perc, rss_KB, vsz_KB - -if len(sys.argv) < 2: - print("Usage: python3 plot_metrics.py ...") - sys.exit(1) - -directories = sys.argv[1:] - -# Lists to store data from all directories -all_time_ms = [] -all_cpu_perc = [] -all_rss_MB = [] -all_vsz_MB = [] - -# CSV output file -csv_output_file = "average_metrics.csv" - -# Process each directory and collect data -with open(csv_output_file, 'w') as csv_file: - csv_file.write("Directory,Average_CPU,Average_RSS_MB,Average_VSZ_MB\n") - - for directory in directories: - time_ms, cpu_perc, rss_KB, vsz_KB = process_directory(directory) - all_time_ms.append(time_ms) - all_cpu_perc.append(cpu_perc) - all_rss_MB.append(rss_KB / 1024) # Convert from KB to MB - all_vsz_MB.append(vsz_KB / 1024) # Convert from KB to MB - - # Compute average metrics for the current directory - average_cpu = round(np.mean(cpu_perc), 2) - average_rss_MB = round(np.mean(rss_KB) / 1024, 2) # Convert from KB to MB - average_vsz_MB = round(np.mean(vsz_KB) / 1024, 2) # Convert from KB to MB - - # Print and write average metrics in CSV format - csv_file.write(f"{directory},{average_cpu},{average_rss_MB},{average_vsz_MB}\n") - if directory == directories[0]: # Print column titles only once - print("Directory,Average_CPU,Average_RSS_MB,Average_VSZ_MB") - print(f"{directory},{average_cpu},{average_rss_MB},{average_vsz_MB}") - -# Plot CPU usage for all directories -plt.figure(figsize=(8, 5)) -for time_ms, cpu_perc, directory in zip(all_time_ms, all_cpu_perc, directories): - plt.plot(time_ms.values, cpu_perc, label=f'CPU_{directory}') -plt.xlabel('Time (ms)') -plt.ylabel('CPU Percentage') -plt.title('CPU Usage over Time') -plt.grid(True) -plt.legend() -plt.show() - -# Plot RSS memory for all directories -plt.figure(figsize=(8, 5)) -for time_ms, rss_MB, directory in zip(all_time_ms, all_rss_MB, directories): - plt.plot(time_ms.values, rss_MB, label=f'RSS_{directory}') -plt.xlabel('Time (ms)') -plt.ylabel('RSS Memory (MB)') -plt.title('RSS Memory over Time') -plt.grid(True) -plt.legend() -plt.show() - -# Plot VSZ memory for all directories -plt.figure(figsize=(8, 5)) -for time_ms, vsz_MB, directory in zip(all_time_ms, all_vsz_MB, directories): - plt.plot(time_ms.values, vsz_MB, label=f'VSZ_{directory}') -plt.xlabel('Time (ms)') -plt.ylabel('VSZ Memory (MB)') -plt.title('VSZ Memory over Time') -plt.grid(True) -plt.legend() -plt.show() diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/plot_latency.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/plot_latency.py deleted file mode 100644 index f8dc84f7..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/plot_latency.py +++ /dev/null @@ -1,132 +0,0 @@ -import sys -import os -import pandas as pd -import numpy as np -import matplotlib.pyplot as plt - -def process_directory(directory): - file_path = os.path.join(directory, "latency_all.txt") - - # Read the file - with open(file_path, "r") as file: - lines = file.readlines() - - # Find the start index of subscription and publisher sections - sub_start = lines.index("Subscriptions stats:\n") + 1 - pub_start = lines.index("Publishers stats:\n") + 1 - - # Find the end index of subscription and publisher sections - sub_end = next((i for i, line in enumerate(lines[sub_start:], start=sub_start) if line.startswith("Publishers stats:") or line == "\n"), len(lines)) - pub_end = len(lines) - - # Read subscriptions data into a pandas DataFrame - subscriptions_data = pd.read_csv( - file_path, - skiprows=sub_start, - nrows=sub_end - sub_start - 1, # Skip the last row containing "Publishers stats:" or the next section - skipinitialspace=True - ) - - # Read publishers data into a pandas DataFrame - publishers_data = pd.read_csv( - file_path, - skiprows=pub_start, - nrows=pub_end - pub_start - 1, # Exclude the last line containing a newline character - skipinitialspace=True - ) - - # Find the position of "mean_us" and "sd_us" in subscription data - sub_mean_col_index = subscriptions_data.columns.get_loc("mean_us") - sub_std_col_index = subscriptions_data.columns.get_loc("sd_us") - - # Extract mean_us and sd_us values from subscriptions data - sub_mean_values = subscriptions_data.iloc[:, sub_mean_col_index].tolist() - sub_std_values = subscriptions_data.iloc[:, sub_std_col_index].tolist() - sub_topics = subscriptions_data['topic'].tolist() - - # Calculate average latency and standard deviation for subscriptions - sub_avg_latency = np.nanmean(sub_mean_values) - sub_avg_std_dev = np.nanmean(sub_std_values) - - # Find the position of "mean_us" and "sd_us" in publisher data - pub_mean_col_index = publishers_data.columns.get_loc("mean_us") - pub_std_col_index = publishers_data.columns.get_loc("sd_us") - - # Extract mean_us and sd_us values from publisher data - pub_mean_values = publishers_data.iloc[:, pub_mean_col_index].tolist() - pub_std_values = publishers_data.iloc[:, pub_std_col_index].tolist() - pub_topics = publishers_data['topic'].tolist() - - # Calculate average latency and standard deviation for publishers - pub_avg_latency = np.nanmean(pub_mean_values) - pub_avg_std_dev = np.nanmean(pub_std_values) - - return sub_mean_values, sub_std_values, sub_topics, sub_avg_latency, sub_avg_std_dev, pub_mean_values, pub_std_values, pub_topics, pub_avg_latency, pub_avg_std_dev - - -if len(sys.argv) < 2: - print("Usage: python3 plot_latency.py ...") - sys.exit(1) - -directories = sys.argv[1:] - -sub_mean_values_combined = [] -sub_std_values_combined = [] -sub_topics_combined = [] -pub_mean_values_combined = [] -pub_std_values_combined = [] -pub_topics_combined = [] - -# Lists to store average values -avg_sub_latencies = [] -avg_pub_latencies = [] - -# Process each directory -for directory in directories: - sub_mean_values, sub_std_values, sub_topics, sub_avg_latency, sub_avg_std_dev, pub_mean_values, pub_std_values, pub_topics, pub_avg_latency, pub_avg_std_dev = process_directory(directory) - - sub_mean_values_combined.append(sub_mean_values) - sub_std_values_combined.append(sub_std_values) - sub_topics_combined.append(sub_topics) - pub_mean_values_combined.append(pub_mean_values) - pub_std_values_combined.append(pub_std_values) - pub_topics_combined.append(pub_topics) - - # Append average latency values - avg_sub_latencies.append(sub_avg_latency) - avg_pub_latencies.append(pub_avg_latency) - - # Compute and print average metrics for the current directory - print(f"Directory: {directory}") - print("Average Latency for Subscriptions:", sub_avg_latency) - print("Average Latency for Publishers:", pub_avg_latency) - print() - -# Print combined statistics in CSV format -print("Directory,Pub_avg_latency,Sub_avg_latency") -for directory, avg_pub_latency, avg_sub_latency in zip(directories, avg_pub_latencies, avg_sub_latencies): - print(f"{directory},{avg_pub_latency},{avg_sub_latency}") - -# Plot combined data -plt.figure(figsize=(10, 5)) -for i, (sub_mean_values, sub_std_values, sub_topics) in enumerate(zip(sub_mean_values_combined, sub_std_values_combined, sub_topics_combined)): - plt.errorbar(range(len(sub_mean_values)), sub_mean_values, yerr=sub_std_values, fmt='o', label=f'Subscriptions {directories[i]}') - - # Calculate and plot horizontal line for the average of mean_us - # avg_mean_us_sub = np.mean(sub_mean_values) - # plt.axhline(y=avg_mean_us_sub, color='r', linestyle='--', label=f'Avg Mean {directories[i]}: {avg_mean_us_sub:.2f} us') - -for i, (pub_mean_values, pub_std_values, pub_topics) in enumerate(zip(pub_mean_values_combined, pub_std_values_combined, pub_topics_combined)): - plt.errorbar(range(len(pub_mean_values)), pub_mean_values, yerr=pub_std_values, fmt='o', label=f'Publishers {directories[i]}') - - # Calculate and plot horizontal line for the average of mean_us - # avg_mean_us_pub = np.mean(pub_mean_values) - # plt.axhline(y=avg_mean_us_pub, color='g', linestyle='--', label=f'Avg Mean {directories[i]}: {avg_mean_us_pub:.2f} us') - -plt.xticks(range(max(len(sub_mean_values_combined[0]), len(pub_mean_values_combined[0]))), sub_topics_combined[0], rotation=45, ha='right') -plt.xlabel('Index') -plt.ylabel('Latency (us)') -plt.title('Combined Latency Stats') -plt.legend() -plt.grid(True) -plt.show() diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/single_process_benchmarks_plot_results.sh b/irobot_benchmark/comms_benchmark/scripts/parse-results/single_process_benchmarks_plot_results.sh deleted file mode 100755 index 601c432d..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/single_process_benchmarks_plot_results.sh +++ /dev/null @@ -1,17 +0,0 @@ -#!/bin/bash - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -scripts="${script_dir}/../../scripts" - -topologies=("pub_sub_10b" "pub_sub_100kb" "pub_sub_1mb" "pub_sub_4mb" "sierra_nevada_fixed_size" "white_mountain_fixed_size") -comms=("ipc_on" "ipc_off" "loaned") - -for topology in "${topologies[@]}"; do - python_args=() - for comm in "${comms[@]}"; do - python_args+=("../results_single_process/${topology}/${comm}") - done - python3 $scripts/plot_cpu.py "${python_args[@]}" - python3 $scripts/plot_latency.py "${python_args[@]}" -done diff --git a/irobot_benchmark/comms_benchmark/scripts/parse-results/split_latency_all.py b/irobot_benchmark/comms_benchmark/scripts/parse-results/split_latency_all.py deleted file mode 100755 index aed4a825..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/parse-results/split_latency_all.py +++ /dev/null @@ -1,53 +0,0 @@ -import os -import sys - -def extract_sections(file_path): - with open(file_path, 'r') as f: - lines = f.readlines() - - subscriptions_started = False - publishers_started = False - sub_lines = [] - pub_lines = [] - - for line in lines: - if "Subscriptions stats:" in line: - subscriptions_started = True - continue - elif "Publishers stats:" in line: - publishers_started = True - subscriptions_started = False - continue - elif subscriptions_started: - if line.strip(): - sub_lines.append(line) - elif publishers_started: - if line.strip(): - pub_lines.append(line) - - return sub_lines, pub_lines - -def process_directory(directory): - for root, dirs, files in os.walk(directory): - for file in files: - if file == 'latency_all.txt': - file_path = os.path.join(root, file) - sub_lines, pub_lines = extract_sections(file_path) - save_to_csv(sub_lines, os.path.join(root, "sub_latency.csv")) - save_to_csv(pub_lines, os.path.join(root, "pub_duration.csv")) - -def save_to_csv(lines, filename): - with open(filename, 'w') as f: - f.writelines(lines) - -if __name__ == "__main__": - if len(sys.argv) != 2: - print("Usage: python script.py ") - sys.exit(1) - - directory = sys.argv[1] - if not os.path.isdir(directory): - print("Error: Directory '{}' not found!".format(directory)) - sys.exit(1) - - process_directory(directory) diff --git a/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_mix_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_mix_process_benchmarks.sh deleted file mode 100644 index de1b393c..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_mix_process_benchmarks.sh +++ /dev/null @@ -1,89 +0,0 @@ -#!/bin/bash - -# Set governor to performance -echo "echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor" -echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/pub-sub" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -MP="${PWD}/results_mix_process" -rm -rf $MP && mkdir -p $MP - -# Define message sizes and communication types -comms=("ipc_off" "loaned_fastdds" "loaned_cyclone") - -# Name of the results folders -results=("10b" "100kb" "1mb" "4mb" "sierra_nevada_fixed_size" "white_mountain_fixed_size") -# Define topologies pairs -topology1=("pub_sub_10b" "pub_sub_100kb" "pub_sub_1mb" "pub_sub_4mb" "sierra_nevada_fixed_size" "white_mountain_fixed_size") -topology2=("sub_10b" "sub_100kb" "sub_1mb" "sub_4mb" "debug_sierra_nevada_fixed_size" "debug_white_mountain_fixed_size") - -# Somehow an external host should synchronize and run also "topology 2" -# topology2=("sub_10b" "sub_100kb" "sub_1mb" "sub_4mb" "debug_sierra_nevada_fixed_size" "debug_white_mountain_fixed_size") -# The script "run_single_process_benchmarks could be of help for that" - -# Check if iox-roudi is running, needed for CycloneDDS zero-copy -if ! pgrep -x "iox-roudi" > /dev/null -then - echo "CycloneDDS: iox-roudi is NOT running. Run as: ./iox-roudi -c roudi_config.toml" -fi - -# Loop through each index in the topology1 array -for i in "${!topology1[@]}"; do - t1=${topology1[i]} - t2=${topology2[i]} - res=${results[i]} - - for comm in "${comms[@]}"; do - result_folder="$MP/${res}/${comm}" - mkdir -p $result_folder - - # Set environment variables - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - - if [ "$comm" == "loaned_fastdds" ]; then - echo "Using FastDDS config: shared_memory_fastdds_preallocated_w_realloc.xml" - export FASTRTPS_DEFAULT_PROFILES_FILE="${profiles_dir}/shared_memory_fastdds_preallocated_w_realloc.xml" - export RMW_FASTRTPS_USE_QOS_FROM_XML=1 - fi - - # Set environment variables for "loaned" communication type - if [ "$comm" == "loaned_cyclone" ]; then - echo "Using CycloneDDS config: zero-copy-shm.xml" - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - export CYCLONEDDS_URI="${profiles_dir}/zero-copy-shm.xml" - fi - - # Set right topology for shared memory - if [ "$comm" == "loaned_fastdds" ]; then - top="${topologies_dir}/${t1}_loaned.json" - else - top="${topologies_dir}/${t1}.json" - fi - - debug_topology="${topologies_dir}/${t2}.json" - - # Results folder - - # Run the command - COMMAND="${irobot_benchmark} $top $debug_topology -x 3 --ipc off -t 60 -s 1000 --csv-out on" - # COMMAND="${irobot_benchmark} $top $debug_topology -x 3 --ipc off -t 60 -s 1000 --csv-out on --timers-separate-thread on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - # Move log folders - mv *log $result_folder - - # Unset environment variables after running "loaned_fastdds" command - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - unset FASTRTPS_DEFAULT_PROFILES_FILE - unset RMW_FASTRTPS_USE_QOS_FROM_XML - unset CYCLONEDDS_URI - fi - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_multi_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_multi_process_benchmarks.sh deleted file mode 100644 index 7c7c95ec..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_multi_process_benchmarks.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Set governor to performance -echo "echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor" -echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/pub-sub" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -MP="${PWD}/results_multi_process" -rm -rf $MP && mkdir -p $MP - -# Define message sizes and communication types -comms=("ipc_off" "loaned_fastdds" "loaned_cyclone") - -# Define topologies pairs -results=("10b" "100kb" "1mb" "4mb") -topology1=("pub_10b" "pub_100kb" "pub_1mb" "pub_4mb") -topology2=("sub_10b" "sub_100kb" "sub_1mb" "sub_4mb") - -# Check if iox-roudi is running, needed for CycloneDDS zero-copy -if ! pgrep -x "iox-roudi" > /dev/null -then - echo "CycloneDDS: iox-roudi is NOT running. Run as: ./iox-roudi -c roudi_config.toml" -fi - -# Loop through each index in the topology1 array -for i in "${!topology1[@]}"; do - t1=${topology1[i]} - t2=${topology2[i]} - res=${results[i]} - - for comm in "${comms[@]}"; do - mkdir -p $MP/${res}/${comm} - - # Set environment variables for "loaned" communication type - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - - if [ "$comm" == "loaned_fastdds" ]; then - export FASTRTPS_DEFAULT_PROFILES_FILE="${profiles_dir}/shared_memory_fastdds_preallocated_w_realloc.xml" - export RMW_FASTRTPS_USE_QOS_FROM_XML=1 - fi - - # Set environment variables for "loaned" communication type - if [ "$comm" == "loaned_cyclone" ]; then - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - export CYCLONEDDS_URI="${profiles_dir}/zero-copy-shm.xml" - fi - - - # Topology - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - top="${topologies_dir}/${t1}_loaned.json" - else - top="${topologies_dir}/${t1}.json" - fi - - debug_topology="${topologies_dir}/${t2}.json" - - # Results folder - - # Run the command - COMMAND="${irobot_benchmark} $top $debug_topology -x 3 --ipc off -t 60 -s 1000 --csv-out on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - # Move log folders - mv *log $MP/${res}/${comm} - - # Unset environment variables after running "loaned" command - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - unset FASTRTPS_DEFAULT_PROFILES_FILE - unset RMW_FASTRTPS_USE_QOS_FROM_XML - fi - done -done diff --git a/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_single_process_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_single_process_benchmarks.sh deleted file mode 100644 index dcbc6cb7..00000000 --- a/irobot_benchmark/comms_benchmark/scripts/pub-sub/run_single_process_benchmarks.sh +++ /dev/null @@ -1,87 +0,0 @@ -#!/bin/bash - -# Set governor to performance -echo "echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor" -echo performance > /sys/devices/system/cpu/cpufreq/policy0/scaling_governor - -# Get the directory where the script is located -script_dir=$(dirname "$(readlink -f "$0")") -irobot_benchmark="${script_dir}/../../../irobot_benchmark" -topologies_dir="${script_dir}/../../topologies/pub-sub" -profiles_dir="${script_dir}/../../profiles" - -# Create a directory to store log folders -SP="${PWD}/results_single_process/" -rm -rf $SP && mkdir -p $SP - -# Define message sizes and communication types -comms=("ipc_on" "ipc_off" "loaned_fastdds" "loaned_cyclone") - -topologies=( - "pub_sub_10b" - "pub_sub_100kb" - "pub_sub_1mb" - "pub_sub_4mb" - "sierra_nevada_fixed_size" - "white_mountain_fixed_size" -) - -# Check if iox-roudi is running, needed for CycloneDDS zero-copy -if ! pgrep -x "iox-roudi" > /dev/null -then - echo "CycloneDDS: iox-roudi is NOT running. Run as: ./iox-roudi -c roudi_config.toml" -fi - -# Loop through each combination of topology and communication type -for topology in "${topologies[@]}"; do - mkdir -p $SP/${topology} - - for comm in "${comms[@]}"; do - # Set environment variables - export RMW_IMPLEMENTATION=rmw_fastrtps_cpp - - if [ "$comm" == "loaned_fastdds" ]; then - export FASTRTPS_DEFAULT_PROFILES_FILE="${profiles_dir}/shared_memory_fastdds_dynamic_reusable.xml" - export RMW_FASTRTPS_USE_QOS_FROM_XML=1 - fi - - # Set environment variables for "loaned" communication type - if [ "$comm" == "loaned_cyclone" ]; then - export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp - export CYCLONEDDS_URI="${profiles_dir}/zero-copy-shm.xml" - fi - - # Construct the command based on communication type - if [ "$comm" == "ipc_on" ]; then - ipc_option="--ipc on" - else - ipc_option="--ipc off" - fi - - # Set right topology for shared memory - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - top="${topologies_dir}/${topology}_loaned.json" - else - top="${topologies_dir}/${topology}.json" - fi - - # Results folder - result_folder=${comm} - - # Run the command - COMMAND="${irobot_benchmark} $top -x 3 $ipc_option -t 30 -s 1000 --csv-out on --results-dir $result_folder" - # COMMAND="${irobot_benchmark} $top -x 3 $ipc_option -t 30 -s 1000 --csv-out on --results-dir $result_folder --timers-separate-thread on" - echo -e "\nCommand: \n$COMMAND\n" - eval $COMMAND - - mv $result_folder $SP/${topology} - - # Unset environment variables - if [[ "$comm" == "loaned_fastdds" || "$comm" == "loaned_cyclone" ]]; then - unset FASTRTPS_DEFAULT_PROFILES_FILE - unset RMW_FASTRTPS_USE_QOS_FROM_XML - unset CYCLONEDDS_URI - fi - done -done - diff --git a/irobot_benchmark/comms_benchmark/scripts/requirements.sh b/irobot_benchmark/comms_benchmark/scripts/requirements.sh new file mode 100755 index 00000000..7b481379 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/requirements.sh @@ -0,0 +1,172 @@ +#!/bin/bash + +# Display help/usage instructions +print_help() { + echo "Usage: $0 [FOLDER]" + echo + echo "This script analyzes benchmark requirements using CSV files from the specified folder." + echo "It is work in progress - not finished yet." + echo + echo "Arguments:" + echo " FOLDER Path to the folder containing 'average_latency.csv' and 'average_metrics.csv'." + echo + echo "Description:" + echo " The script verifies multiple requirements by extracting and analyzing data from the provided CSV files." + echo + echo "Examples:" + echo " $0 ./results_folder Analyze data from the 'results_folder'." + echo + exit 0 +} + +# Check for help flag or no arguments +if [[ "$1" == "--help" || "$1" == "-h" || -z "$1" ]]; then + print_help +fi + +# Validate the folder argument +folder="$1" +if [[ ! -d "$folder" ]]; then + echo -e "\033[31m[ERROR] The folder '$folder' does not exist or is not accessible.\033[0m" + exit 1 +fi + +# Ensure required CSV files exist in the folder +latency_csv="${folder}/average_latency.csv" +metrics_csv="${folder}/average_metrics.csv" +if [[ ! -f "$latency_csv" || ! -f "$metrics_csv" ]]; then + echo -e "\033[31m[ERROR] Missing 'average_latency.csv' or 'average_metrics.csv' in folder '$folder'.\033[0m" + exit 1 +fi + +echo "Analyzing CSV files from folder: $folder" + +# ------------------------------------------------------------------------------------------------------------ +echo "### 1. Latency Consistency Across Message Sizes" +echo -e "Requirement: CPU and latency should not depend on message size for single-process applications. \n" +output=$(cat "$latency_csv" | grep pub-sub | grep single | grep loaned | grep fast) +echo -e "rclcpp intra-process on: \n" +head -n 1 "$latency_csv" +echo -e "$output \n" +output=$(cat "$latency_csv" | grep pub-sub | grep single | grep ipc_off | grep fast) +echo -e "Shared Memory (Loaned messages): \n" +head -n 1 "$latency_csv" +echo -e "$output \n" +output=$(cat "$latency_csv" | grep pub-sub | grep single | grep ipc_on | grep fast) +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 2. Latency and CPU Comparison: Multi-Process vs Single-Process \n" +echo -e "Requirement: Multi-process latency/CPU should be ≈ single-process for POD message types. \n" +echo -e "Multi-Process (Shared Memory): \n" +latency_output=$(cat "$latency_csv" | grep pub-sub | grep multi | grep loaned | grep fast) +echo -e "Latency: \n" +head -n 1 "$latency_csv" +echo -e "$latency_output \n" +metrics_output=$(cat "$metrics_csv" | grep pub-sub | grep multi | grep loaned | grep fast) +echo -e "Metrics: \n" +head -n 1 "$metrics_csv" +echo -e "$metrics_output \n" +echo -e "Single-Process (Shared Memory): \n" +latency_output=$(cat "$latency_csv" | grep pub-sub | grep single | grep loaned | grep fast) +echo -e "Latency: \n" +head -n 1 "$latency_csv" +echo -e "$latency_output \n" +metrics_output=$(cat "$metrics_csv" | grep pub-sub | grep single | grep loaned | grep fast) +echo -e "Metrics: \n" +head -n 1 "$metrics_csv" +echo -e "$metrics_output \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 3. Long-Running Stability \n" +echo -e "Requirement: CPU, latency, and RAM must remain constant over long durations (>10 minutes). \n" +latency_output=$(cat average_latency.csv | grep white_mountain_fixed_size | grep loaned | grep fast) +echo -e "Latency: \n" +head -n 1 average_latency.csv +echo -e "$latency_output \n" +metrics_output=$(cat average_metrics.csv | grep white_mountain_fixed_size | grep loaned | grep fast) +echo -e "Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 4. Multi-Transport Communication \n" +echo -e "Requirement: Combining multiple transports should not degrade performance. \n" +latency_output=$(cat average_latency.csv | grep pub-sub | grep mix_process | grep loaned | grep fast) +echo -e "Latency: \n" +head -n 1 average_latency.csv +echo -e "$latency_output \n" +metrics_output=$(cat average_metrics.csv | grep pub-sub | grep mix_process | grep loaned | grep fast) +echo -e "Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 5. Scalability with Number of Clients (Services) \n" +echo -e "Requirement: Service performance depends only on active clients, not the total number. \n" +latency_output=$(cat average_latency.csv | grep cli-srv | grep multi | grep loaned | grep fast) +echo -e "Latency: \n" +head -n 1 average_latency.csv +echo -e "$latency_output \n" +metrics_output=$(cat average_metrics.csv | grep cli-srv | grep multi | grep loaned | grep fast) +echo -e "Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 6. Scalability with Number of Clients (Actions) \n" +echo -e "Requirement: Action performance depends only on active clients, not the total number. \n" +latency_output=$(cat average_latency.csv | grep action | grep multi | grep loaned | grep fast) +echo -e "Latency: \n" +head -n 1 average_latency.csv +echo -e "$latency_output \n" +metrics_output=$(cat average_metrics.csv | grep action | grep multi | grep loaned | grep fast) +echo -e "Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 7. CPU Overhead with Remote Subscribers \n" +echo -e "Requirement: Adding remote subscribers should add <10% CPU overhead. \n" +metrics_output_1mb=$(cat average_metrics.csv | grep pub-sub | grep mix_process | grep loaned | grep fast | grep 1mb) +echo -e "1MB Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output_1mb \n" +metrics_output_4mb=$(cat average_metrics.csv | grep pub-sub | grep mix_process | grep loaned | grep fast | grep 4mb) +echo -e "4MB Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output_4mb \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 8. Scalability of RAM Usage \n" +echo -e "Requirement: RAM usage scales linearly with processes or entities. \n" +metrics_output_pubsub=$(cat average_metrics.csv | grep pub-sub | grep multi | grep loaned | grep fast) +echo -e "Pub-Sub Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output_pubsub \n" +metrics_output_clisrv=$(cat average_metrics.csv | grep cli-srv | grep multi | grep loaned | grep fast) +echo -e "CLI-SRV Metrics: \n" +head -n 1 average_metrics.csv +echo -e "$metrics_output_clisrv \n" +# ------------------------------------------------------------------------------------------------------------ + +# ------------------------------------------------------------------------------------------------------------ +echo -e "### 9. Lifecycle Mechanism Efficiency \n" +echo -e "Requirement: Autocore lifecycle mechanisms must have less CPU overhead than callback-based 'early return'. \n" +echo -e ".. no command for this yet .. \n" + +echo -e "### 10. Flash Size Limit \n" +echo -e "Requirement: Total flash size of Autocore RMW and dependencies must be <4MB. \n" +echo -e ".. no command for this yet .. \n" + +echo -e "### 11. Baseline Comparison \n" +echo -e "Requirement: Autocore RMW must show no regressions compared to Fast-DDS. \n" +echo -e "Run all relevant commands for both 'fast' and 'cyclone' entries. \n" +echo -e ".. no automated script yet .. \n" +# ------------------------------------------------------------------------------------------------------------ diff --git a/irobot_benchmark/comms_benchmark/scripts/run_all_benchmarks.sh b/irobot_benchmark/comms_benchmark/scripts/run_all_benchmarks.sh new file mode 100755 index 00000000..88fb788b --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/run_all_benchmarks.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# Display help/usage instructions +print_help() { + echo "Usage: $0 [OPTIONS] [TEST_DURATION]" + echo + echo "This script automates running benchmarks for single-process, multi-process," + echo "and mixed-process configurations, organizes the results, and processes them for analysis." + echo + echo "Options:" + echo " --help, -h Display this help message." + echo + echo "Arguments:" + echo " TEST_DURATION Time in seconds to run each benchmark. Defaults to 1 if not provided." + echo + echo "Description:" + echo " 1. Benchmarks are categorized as:" + echo " - Single-process benchmarks [pub_sub, cli_srv, actions]." + echo " - Multi-process benchmarks [pub_sub, cli_srv, actions]" + echo " - Mix-process benchmarks [pub_sub, cli_srv, actions]" + echo + echo " 2. Results are stored in a timestamped directory (e.g., results_29_11_24_18h51)." + echo + echo " 3. Parsing includes: (Uses python3 & dependencies)" + echo " - Comparing CPU, Memory & Latency (compare_metrics.py)." + echo " - Comparing Latency metrics (compare_latency.py)." + echo + echo "Example:" + echo " sudo env LD_LIBRARY_PATH=\$LD_LIBRARY_PATH $0 10 Run benchmarks with TEST_DURATION=10 seconds." + echo + echo "Needed because sudo resets LD_LIBRARY_PATH, breaking shared library access." + echo + exit 0 +} + +# Check for help flag +if [[ "$1" == "--help" || "$1" == "-h" ]]; then + print_help +fi + +# Check if the script is run as root +if [[ $EUID -ne 0 ]]; then + echo "You must run this script with sudo or as root." + echo "Usage: sudo env LD_LIBRARY_PATH=\$LD_LIBRARY_PATH $0 [arguments]" + exit 1 +fi + +# Set TEST_DURATION +TEST_DURATION=${1:-1} # Default to 1 second if not provided + +# Create the results directory +current_date=$(date +"%d_%m_%y_%Hh%M") +results_dir="results_${current_date}" +mkdir -p "$results_dir" +echo "Storing results in $results_dir" + +run_benchmark() { + local script=$1 + local config=$2 + + echo "Running: $script with config: $config and test duration=$TEST_DURATION" + TEST_DURATION="$TEST_DURATION" bash "$script" "$config" + local exit_code=$? + + if [ $exit_code -ne 0 ]; then + echo -e "\033[31m[ERROR] $script $config failed with exit code $exit_code\033[0m" + exit $exit_code + else + echo -e "\033[32m[SUCCESS] $script $config completed successfully\033[0m" + fi +} + +# Single-process benchmarks +echo "Starting single-process benchmarks..." +cd single-process +run_benchmark "run_single_process_benchmark.sh" "pub_sub.conf" +run_benchmark "run_single_process_benchmark.sh" "cli_srv.conf" +run_benchmark "run_single_process_benchmark.sh" "actions.conf" +cd - + +# Multi-process benchmarks +echo "Starting multi-process benchmarks..." +cd multi-process +run_benchmark "run_multi_process_benchmark.sh" "multi_process_pub_sub.conf" +run_benchmark "run_multi_process_benchmark.sh" "multi_process_cli_srv.conf" +run_benchmark "run_multi_process_benchmark.sh" "multi_process_actions.conf" + +# Mix-process benchmarks +echo "Starting mix-process benchmarks..." +run_benchmark "run_multi_process_benchmark.sh" "mix_process_pub_sub.conf" +run_benchmark "run_multi_process_benchmark.sh" "mix_process_cli_srv.conf" +run_benchmark "run_multi_process_benchmark.sh" "mix_process_actions.conf" +cd - + +# Move results to the results directory +mv single-process/pub-sub* "$results_dir" +mv single-process/cli-srv* "$results_dir" +mv single-process/actions* "$results_dir" + +mv multi-process/pub-sub* "$results_dir" +mv multi-process/cli-srv* "$results_dir" +mv multi-process/actions* "$results_dir" + +# Parse results +echo +echo "To parse results run:" +echo " python3 parse-results/compare_metrics.py $results_dir --plot" +echo " python3 parse-results/compare_latency.py $results_dir --plot" +echo +echo "Remove --plot if no plot is needed." +echo "If not already done, run 'pip3 install pandas numpy matplotlib' to install dependencies" diff --git a/irobot_benchmark/comms_benchmark/scripts/set_cpu_governor.sh b/irobot_benchmark/comms_benchmark/scripts/set_cpu_governor.sh new file mode 100755 index 00000000..1690aea1 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/set_cpu_governor.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# Function to display help +print_help() { + echo "Usage: $0 " + echo + echo "This script sets the CPU governor to the specified value for all policies." + echo + echo "Examples:" + echo " $0 performance # Set the CPU governor to 'performance'." + echo " $0 powersave # Set the CPU governor to 'powersave'." + echo + exit 0 +} + +# Check if arguments are provided +if [[ $# -ne 1 || "$1" == "--help" || "$1" == "-h" ]]; then + print_help +fi + +# Get the desired governor from the argument +desired_governor="$1" + +# Ensure the CPU frequency scaling directory exists +if [ ! -d "/sys/devices/system/cpu/cpufreq/" ]; then + echo -e "\033[31m[ERROR] CPU frequency scaling not supported on this system.\033[0m" + exit 1 +fi + +# Apply the specified governor to all policies +echo "Setting CPU governor to '$desired_governor'..." +for policy in /sys/devices/system/cpu/cpufreq/policy*/scaling_governor; do + if ! sudo echo "$desired_governor" > "$policy" 2>/dev/null; then + echo -e "\033[31m[ERROR] Permission denied while setting governor for $policy - run with sudo.\033[0m" + exit 1 + fi +done + +# Verify the change +current_governor=$(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor) +if [ "$current_governor" != "$desired_governor" ]; then + echo -e "\033[31m[ERROR] Failed to set CPU governor to '$desired_governor'. Current value: '$current_governor'.\033[0m" + exit 1 +fi + +echo -e "\033[32m[SUCCESS] CPU governor set to '$desired_governor'.\033[0m" diff --git a/irobot_benchmark/comms_benchmark/scripts/single-process/actions.conf b/irobot_benchmark/comms_benchmark/scripts/single-process/actions.conf new file mode 100644 index 00000000..713930a2 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/single-process/actions.conf @@ -0,0 +1,13 @@ +# Configuration for ActionClient/ActionServer benchmarks + +OUTPUT_DIR="actions_single_process" +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/actions" +TOPOLOGIES=("cli_srv_10b" "cli_srv_100kb" "cli_srv_1mb" "cli_srv_4mb") + +# Define RMWs to iterate over +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and env vars for each RMW +COMMS_fastrtps=("ipc_on" "ipc_off") +COMMS_cyclonedds=("ipc_off") +# COMMS_example=() diff --git a/irobot_benchmark/comms_benchmark/scripts/single-process/cli_srv.conf b/irobot_benchmark/comms_benchmark/scripts/single-process/cli_srv.conf new file mode 100644 index 00000000..9853bd7d --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/single-process/cli_srv.conf @@ -0,0 +1,14 @@ +# Configuration for Client/Service benchmarks + +OUTPUT_DIR="cli-srv_single_process" +TOPOLOGIES=("cli_srv_10b" "cli_srv_100kb" "cli_srv_1mb" "cli_srv_4mb" "10_cli_srv_1mb") +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/cli-srv" + +# Define RMWs to iterate over +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and env vars for each RMW +COMMS_fastrtps=("ipc_on" "ipc_off") +COMMS_cyclonedds=("ipc_off") + +# COMMS_example=() diff --git a/irobot_benchmark/comms_benchmark/scripts/single-process/pub_sub.conf b/irobot_benchmark/comms_benchmark/scripts/single-process/pub_sub.conf new file mode 100644 index 00000000..e4982487 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/single-process/pub_sub.conf @@ -0,0 +1,25 @@ +# Configuration for Pub/Sub benchmarks + +OUTPUT_DIR="pub-sub_single_process" +TOPOLOGIES=("pub_sub_10b" "pub_sub_100kb" "pub_sub_1mb" "pub_sub_4mb" "sierra_nevada_fixed_size" "white_mountain_fixed_size") + +TOPOLOGIES_DIR="${SCRIPT_DIR}/../../topologies/pub-sub" +PROFILES_DIR="${SCRIPT_DIR}/../../profiles" + +# Define RMWs to iterate over +RMW_LIST=("fastrtps" "cyclonedds") + +# Define COMMS and env vars for each RMW +COMMS_fastrtps=("ipc_on" "ipc_off" "loaned") +LOANED_ENV_VARS_fastrtps=( + "export FASTRTPS_DEFAULT_PROFILES_FILE=${PROFILES_DIR}/shared_memory_fastdds_preallocated_w_realloc.xml" + "export RMW_FASTRTPS_USE_QOS_FROM_XML=1" +) + +COMMS_cyclonedds=("ipc_off") +LOANED_ENV_VARS_cyclonedds=( + "export CYCLONEDDS_URI=${PROFILES_DIR}/shared_memory_cyclonedds.xml" +) + +# COMMS_example=() +# LOANED_ENV_VARS_example=() diff --git a/irobot_benchmark/comms_benchmark/scripts/single-process/run_single_process_benchmark.sh b/irobot_benchmark/comms_benchmark/scripts/single-process/run_single_process_benchmark.sh new file mode 100644 index 00000000..0581c1a6 --- /dev/null +++ b/irobot_benchmark/comms_benchmark/scripts/single-process/run_single_process_benchmark.sh @@ -0,0 +1,104 @@ +#!/bin/bash + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + exit 1 +fi + +# Define SCRIPT_DIR before sourcing the configuration file +SCRIPT_DIR=$(dirname "$(readlink -f "$0")") + +# Load the configuration file +CONFIG_FILE=$1 +if [ ! -f "$CONFIG_FILE" ]; then + echo -e "\033[31m[ERROR] Configuration file '$CONFIG_FILE' not found!\033[0m" + exit 1 +fi + +if [ -z "${TEST_DURATION}" ]; then + TEST_DURATION=1 +fi + +# Export SCRIPT_DIR for use in the configuration file +export SCRIPT_DIR + +source "$CONFIG_FILE" + +# Load paths +GOVERNOR_SCRIPT="${SCRIPT_DIR}/../set_cpu_governor.sh" +IROBOT_BENCHMARK="${SCRIPT_DIR}/../../../irobot_benchmark" + +# Get the original governor +original_governor=$(cat /sys/devices/system/cpu/cpufreq/policy0/scaling_governor) +# Set the CPU governor to performance +$GOVERNOR_SCRIPT performance +# Restore the original governor on exit +trap "$GOVERNOR_SCRIPT $original_governor" EXIT + +if [ $? -ne 0 ]; then + echo -e "\033[31m[ERROR] Failed to set CPU governor to performance. Exiting...\033[0m" + exit 1 +fi + +# Create a directory to store log folders +SP="${PWD}/${OUTPUT_DIR}" +rm -rf "$SP" && mkdir -p "$SP" + +for RMW in "${RMW_LIST[@]}"; do + echo -e "\033[32mProcessing RMW: $RMW\033[0m" + + # Set RMW implementation + export RMW_IMPLEMENTATION="rmw_${RMW}_cpp" + + # Resolve COMMS and LOANED_ENV_VARS dynamically + declare -n COMMS="COMMS_${RMW}" + declare -n LOANED_ENV_VARS="LOANED_ENV_VARS_${RMW}" + echo -e "\033[32m COMMS for $RMW: ${COMMS[@]}\033[0m" + echo -e "\033[32m LOANED_ENV_VARS for $RMW: \n ${LOANED_ENV_VARS[@]}\033[0m" + + for COMM in "${COMMS[@]}"; do + echo -e "\033[32m Testing COMM: $COMM\033[0m" + + # Handle loaned environment variables + if [[ "$COMM" == "loaned" ]]; then + for VAR in "${LOANED_ENV_VARS[@]}"; do + eval $VAR + done + fi + + IPC_OPTION="--ipc off" + if [[ "$COMM" == "ipc_on" ]]; then + IPC_OPTION="--ipc on" + fi + + for TOPOLOGY in "${TOPOLOGIES[@]}"; do + mkdir -p "$SP/${TOPOLOGY}/${RMW}_${COMM}" + + # Set topology file dynamically from configuration + if [[ "$COMM" == "loaned" ]]; then + TOP="${TOPOLOGIES_DIR}/${TOPOLOGY}_loaned.json" + else + TOP="${TOPOLOGIES_DIR}/${TOPOLOGY}.json" + fi + + # Results folder + RESULT_FOLDER="${RMW}_${COMM}" + + # Run the command + COMMAND="${IROBOT_BENCHMARK} $TOP -x 3 $IPC_OPTION -t $TEST_DURATION -s 1000 --csv-out on --results-dir $RESULT_FOLDER" + echo -e "\033[32m\nCommand: \n$COMMAND\n\033[0m" + eval $COMMAND + if [ $? -ne 0 ]; then + echo -e "\033[31m[ERROR] Command failed: $COMMAND\033[0m" + exit 1 + fi + + mv "$RESULT_FOLDER" "$SP/${TOPOLOGY}/" + done + + # Unset loaned environment variables if applicable + unset FASTRTPS_DEFAULT_PROFILES_FILE + unset RMW_FASTRTPS_USE_QOS_FROM_XML + unset CYCLONEDDS_URI + done +done diff --git a/irobot_benchmark/src/irobot_benchmark.cpp b/irobot_benchmark/src/irobot_benchmark.cpp index fdf57baf..9407bc5b 100644 --- a/irobot_benchmark/src/irobot_benchmark.cpp +++ b/irobot_benchmark/src/irobot_benchmark.cpp @@ -50,7 +50,7 @@ create_result_directory(const std::string & topology_json, const std::string & r { std::string result_dir_name; - if (result_dir != "") { + if (!result_dir.empty()) { result_dir_name = result_dir; } else { size_t last_slash = topology_json.find_last_of("/"); diff --git a/irobot_benchmark/topology/sierra_nevada.json b/irobot_benchmark/topology/sierra_nevada.json index 67a9d152..5a72a28b 100644 --- a/irobot_benchmark/topology/sierra_nevada.json +++ b/irobot_benchmark/topology/sierra_nevada.json @@ -39,7 +39,7 @@ {"topic_name":"parana", "msg_type":"stamped3_float32"} ], "publishers": [ - {"topic_name": "salween", "msg_type": "stamped12_float32", "period_ms": 100, "msg_pass_by":"shared_ptr"} + {"topic_name": "salween", "msg_type": "stamped12_float32", "period_ms": 100, "msg_pass_by":"loaned_msg"} ] }, diff --git a/performance_test_factory/src/factory.cpp b/performance_test_factory/src/factory.cpp index fb735880..6fb3e402 100644 --- a/performance_test_factory/src/factory.cpp +++ b/performance_test_factory/src/factory.cpp @@ -638,7 +638,7 @@ void TemplateFactory::add_periodic_action_client_from_json( } else if (action_client_json.find("period_ms") != action_client_json.end()) { period_ms = action_client_json["period_ms"]; } else { - std::cout << "Error! Action Clients must set period_ms or freq_hz in json file" << std::endl; + assert(0 && "Error! Action Clients must set period_ms or freq_hz in json file"); } auto period = std::chrono::microseconds(static_cast(period_ms * 1000));