fix(perf): adjust cli flags for perf runner for testing purposes (#275)

Co-authored-by: Max Inden <mail@max-inden.de>

perf/README.md

@@ -15,11 +15,11 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp
 3. Wait for action run to finish and to push a commit to your branch.
 4. Visualize results on https://observablehq.com/@libp2p-workspace/performance-dashboard.
 
-## Running with Terraform on AWS manually
+## Running manually
 
 ### Prerequisites
 
-- Terraform 1.5.5 or later
+- Terraform 1.5.4 or later
 - Node.js 18 or later
 - [an AWS IAM user](https://docs.aws.amazon.com/IAM/latest/UserGuide/id_users.html)
 
@@ -34,7 +34,6 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp
 6. `SERVER_IP=$(terraform output -raw server_ip)`
 
 **Notes**
-- You may need to reset the infrastructure if you encounter any errors, you can do that by running `terraform destroy` and then `terraform apply`.
 - While running terraform you may encounter the following error:
   ```bash
     Error: collecting instance settings: reading EC2 Launch Template versions: couldn't find resource
@@ -43,7 +42,7 @@ Benchmark results can be visualized with https://observablehq.com/@libp2p-worksp
     │   on ../../modules/short_lived/main.tf line 15, in resource "aws_instance" "perf":
     │   15: resource "aws_instance" "perf" {
   ```
-- If you set *TF_VAR* [`long_lived_enabled`](./terraform/configs/local/terraform.tf#L42) env variable to default to **true** terraform should spin up the long-lived resources that are required for the short-lived resources to be created.
+- This implies that you haven't deployed the long-lived infrastructure on your AWS account. To do so along with each short-lived deployment, you can set *TF_VAR* [`long_lived_enabled`](./terraform/configs/local/terraform.tf#L42) env variable to default to `true`. Terraform should then spin up the long-lived resources that are required for the short-lived resources to be created.
 
 - It's best to destroy the infrastructure after you're done with your testing, you can do that by running `terraform destroy`.
 
@@ -80,7 +79,7 @@ Given you have provisioned your infrastructure, you can now build and run the li
             - `--download-bytes` number of bytes to download per stream.
           - Output
             - Logging MUST go to `stderr`.
-            - Measurement output is printed to **stdout** as JSON in the form of:
+            - Measurement output is printed to `stdout` as JSON in the form of:
               ```json
               {"latency": 0.246442851}
               ```

perf/runner/src/index.ts

@@ -4,9 +4,9 @@ import yargs from 'yargs';
 import fs from 'fs';
 import { BenchmarkResults, Benchmark, Result, IperfResults, PingResults, ResultValue } from './benchmark-result-type';
 
-async function main(clientPublicIP: string, serverPublicIP: string, iterations: number) {
-    const pings = runPing(clientPublicIP, serverPublicIP);
-    const iperf = runIPerf(clientPublicIP, serverPublicIP);
+async function main(clientPublicIP: string, serverPublicIP: string, testing: boolean) {
+    const pings = runPing(clientPublicIP, serverPublicIP, testing);
+    const iperf = runIPerf(clientPublicIP, serverPublicIP, testing);
 
     copyAndBuildPerfImplementations(serverPublicIP);
     copyAndBuildPerfImplementations(clientPublicIP);
@@ -19,7 +19,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations:
                  uploadBytes: 100 << 20,
                  downloadBytes: 0,
                  unit: "bit/s",
-                 iterations,
+                 iterations: testing ? 1 : 10,
              }),
              runBenchmarkAcrossVersions({
                  name: "Single Connection throughput – Download 100 MiB",
@@ -28,7 +28,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations:
                  uploadBytes: 0,
                  downloadBytes: 100 << 20,
                  unit: "bit/s",
-                 iterations,
+                 iterations: testing ? 1 : 10,
              }),
              runBenchmarkAcrossVersions({
                  name: "Connection establishment + 1 byte round trip latencies",
@@ -37,7 +37,7 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations:
                  uploadBytes: 1,
                  downloadBytes: 1,
                  unit: "s",
-                 iterations: iterations * 10,
+                 iterations: testing ? 1 : 100,
              }),
     ];
 
@@ -53,10 +53,11 @@ async function main(clientPublicIP: string, serverPublicIP: string, iterations:
     console.error("== done");
 }
 
-function runPing(clientPublicIP: string, serverPublicIP: string): PingResults {
-    console.error(`= run 100 pings from client to server`);
+function runPing(clientPublicIP: string, serverPublicIP: string, testing: boolean): PingResults {
+    const pingCount = testing ? 1 : 100;
+    console.error(`= run ${pingCount} pings from client to server`);
 
-    const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'ping -c 100 ${serverPublicIP}'`;
+    const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'ping -c ${pingCount} ${serverPublicIP}'`;
     const stdout = execCommand(cmd).toString();
 
     // Extract the time from each ping
@@ -71,9 +72,9 @@ function runPing(clientPublicIP: string, serverPublicIP: string): PingResults {
     return { unit: "s", results: times }
 }
 
-function runIPerf(clientPublicIP: string, serverPublicIP: string): IperfResults {
-    const iterations = 60;
-    console.error(`= run ${iterations} iPerf TCP from client to server`);
+function runIPerf(clientPublicIP: string, serverPublicIP: string, testing: boolean): IperfResults {
+    const iPerfIterations = testing ? 1 : 60;
+    console.error(`= run ${iPerfIterations} iPerf TCP from client to server`);
 
     const killCMD = `ssh -o StrictHostKeyChecking=no ec2-user@${serverPublicIP} 'kill $(cat pidfile); rm pidfile; rm server.log || true'`;
     const killSTDOUT = execCommand(killCMD);
@@ -83,7 +84,7 @@ function runIPerf(clientPublicIP: string, serverPublicIP: string): IperfResults
     const serverSTDOUT = execCommand(serverCMD);
     console.error(serverSTDOUT);
 
-    const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'iperf3 -c ${serverPublicIP} -b 25g -t ${iterations}'`;
+    const cmd = `ssh -o StrictHostKeyChecking=no ec2-user@${clientPublicIP} 'iperf3 -c ${serverPublicIP} -b 25g -t ${iPerfIterations}'`;
     const stdout = execSync(cmd).toString();
 
     // Extract the bitrate from each relevant line
@@ -232,14 +233,14 @@ const argv = yargs
             demandOption: true,
             description: 'Server public IP address',
         },
-        'iterations': {
-            type: 'number',
-            default: 10,
-            description: 'Number of iterations to run',
+        'testing': {
+            type: 'boolean',
+            default: false,
+            description: 'Run in testing mode',
             demandOption: false,
         }
     })
     .command('help', 'Print usage information', yargs.help)
     .parseSync();
 
-main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['iterations'] as number);
+main(argv['client-public-ip'] as string, argv['server-public-ip'] as string, argv['testing'] as boolean);