From 789d0ae918cdf0b6292909e4865461ca4a9ca038 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Sun, 25 Feb 2024 10:04:33 -0500 Subject: [PATCH 1/6] Process Raw Latency Data in netperf --- scripts/secnetperf-helpers.psm1 | 58 ++++++++++++++++++++++++++++++--- scripts/secnetperf.ps1 | 1 + 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 9a87f5f914..b83ef9ed4a 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -282,6 +282,7 @@ function Start-LocalTest { $pinfo.FileName = $FullPath $pinfo.Arguments = $FullArgs } else { + # We use bash to execute the test so we can collect core dumps. $pinfo.FileName = "bash" $pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''`"" $pinfo.WorkingDirectory = $OutputDir @@ -300,7 +301,9 @@ function Wait-LocalTest { param ($Process, $OutputDir, $testKernel, $TimeoutMs) $StdOut = $Process.StandardOutput.ReadToEndAsync() $StdError = $Process.StandardError.ReadToEndAsync() + # Wait for the process to exit. if (!$Process.WaitForExit($TimeoutMs)) { + # We timed out waiting for completion. Collect a dump of the current state. if ($testKernel) { Collect-LiveKD $OutputDir "client" } Collect-LocalDump $Process $OutputDir try { $Process.Kill() } catch { } @@ -311,6 +314,7 @@ function Wait-LocalTest { } catch {} throw "secnetperf: Client timed out!" } + # Verify the process cleanly exitted. if ($Process.ExitCode -ne 0) { try { [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError)) @@ -319,6 +323,7 @@ function Wait-LocalTest { } catch {} throw "secnetperf: Nonzero exit code: $($Process.ExitCode)" } + # Wait for the output streams to flush. [System.Threading.Tasks.Task]::WaitAll(@($StdOut, $StdError)) $consoleTxt = $StdOut.Result.Trim() if ($consoleTxt.Length -eq 0) { @@ -377,6 +382,43 @@ function Get-TestOutput { } } +function Get-LatencyOutput { + param ($FilePath) + # The data is in the format of: + # + # Value Percentile TotalCount 1/(1-Percentile) + # + # 93.000 0.000000 1 1.00 + # 118.000 0.100000 18203 1.11 + # 120.000 0.200000 29488 1.25 + # ... ... ... ... + # 9847.000 0.999992 139772 131072.00 + # 41151.000 0.999993 139773 145635.56 + # 41151.000 1.000000 139773 inf + ##[Mean = 142.875, StdDeviation = 137.790] + ##[Max = 41151.000, Total count = 139773] + ##[Buckets = 6, SubBuckets = 2048] + + $contents = Get-Content $FilePath + Remove-Item $FilePath + + # Parse through the data and extract the Value and Percentile columns and + # convert to an array. Ignore the trailing data. + $data = @() + foreach ($line in $contents) { + if ($line -match "^\s*(\d+\.\d+)\s+(\d+\.\d+)") { + $value = [int]$matches[1] + $percentile = [double]$matches[2] + $data += New-Object PSObject -Property @{ + "Value" = $value + "Percentile" = $percentile + } + } + } + + return $data +} + # Invokes secnetperf with the given arguments for both TCP and QUIC. function Invoke-Secnetperf { param ($Session, $RemoteName, $RemoteDir, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter) @@ -413,12 +455,16 @@ function Invoke-Secnetperf { $serverArgs += " -stats:1" $clientArgs += " -pconn:1" } - if ($testId.Contains("rps")) { - $latencyDir = Repo-Path "latency.txt" + + $latency = $null + $extraOutput = $null + if ($metric -eq "latency") { + $latency = @(@(), @()) + $extraOutput = Repo-Path "latency.txt" if (!$isWindows) { - chmod +rw "$latencyDir" + chmod +rw "$extraOutput" } - $clientArgs += " -extraOutputFile:$latencyDir" + $clientArgs += " -extraOutputFile:$extraOutput" } if (!(Check-TestFilter $clientArgs $Filter)) { @@ -473,6 +519,9 @@ function Invoke-Secnetperf { $rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000 Write-Host $rawOutput $values[$tcp] += Get-TestOutput $rawOutput $metric + if ($extraOutput) { + $latency[$tcp] = Get-LatencyOutput $extraOutput + } $rawOutput | Add-Content $clientOut $successCount++ } catch { @@ -523,6 +572,7 @@ function Invoke-Secnetperf { return [pscustomobject]@{ Metric = $metric Values = $values + Latency = $latency HasFailures = $hasFailures } } diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 917187afc0..03fe404478 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -284,6 +284,7 @@ VALUES ("$TestId-tcp-$tcp", "$MsQuicCommit", $env, $env, $item, NULL, "$io", "$t "@ } } elseif ($Test.Metric -eq "latency") { + $json["$testId-$transport-lat"] = $Test.Latency[$tcp] # Test.Values[...] is a flattened 1D array of the form: [ first run + RPS, second run + RPS, third run + RPS..... ], ie. if each run has 8 values + RPS, then the array has 27 elements (8*3 + 3) for ($offset = 0; $offset -lt $Test.Values[$tcp].Length; $offset += 9) { $SQL += @" From ca2f2bb6a49e4f559707bf178674c4a5fff11609 Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Sun, 25 Feb 2024 10:09:50 -0500 Subject: [PATCH 2/6] Better data format --- scripts/secnetperf-helpers.psm1 | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index b83ef9ed4a..69dcfdf2e5 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -404,19 +404,19 @@ function Get-LatencyOutput { # Parse through the data and extract the Value and Percentile columns and # convert to an array. Ignore the trailing data. - $data = @() + $values = @() + $percentiles = @() foreach ($line in $contents) { if ($line -match "^\s*(\d+\.\d+)\s+(\d+\.\d+)") { - $value = [int]$matches[1] - $percentile = [double]$matches[2] - $data += New-Object PSObject -Property @{ - "Value" = $value - "Percentile" = $percentile - } + $values += [int]$matches[1] + $percentiles += [double]$matches[2] } } - return $data + return [pscustomobject]@{ + Values = $values + Percentiles = $percentiles + } } # Invokes secnetperf with the given arguments for both TCP and QUIC. From dffe2220aa1a2152cf783e9621379a1c24cc858c Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Sun, 25 Feb 2024 10:30:20 -0500 Subject: [PATCH 3/6] Fix bug --- scripts/secnetperf-helpers.psm1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index 69dcfdf2e5..dcde7e91ea 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -520,7 +520,7 @@ function Invoke-Secnetperf { Write-Host $rawOutput $values[$tcp] += Get-TestOutput $rawOutput $metric if ($extraOutput) { - $latency[$tcp] = Get-LatencyOutput $extraOutput + $latency[$tcp] += Get-LatencyOutput $extraOutput } $rawOutput | Add-Content $clientOut $successCount++ From 15ee1afc281f61ea42cca38914356df743ee2e9b Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Sun, 25 Feb 2024 11:47:43 -0500 Subject: [PATCH 4/6] Fix overwrite --- scripts/secnetperf-helpers.psm1 | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1 index dcde7e91ea..82eb63e5d6 100644 --- a/scripts/secnetperf-helpers.psm1 +++ b/scripts/secnetperf-helpers.psm1 @@ -423,16 +423,23 @@ function Get-LatencyOutput { function Invoke-Secnetperf { param ($Session, $RemoteName, $RemoteDir, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter) + $values = @(@(), @()) + $latency = $null + $extraOutput = $null + $hasFailures = $false + $tcpSupported = ($io -ne "xdp" -and $io -ne "qtip" -and $io -ne "wsk") ? 1 : 0 $metric = "throughput" if ($exeArgs.Contains("plat:1")) { $metric = "latency" + $latency = @(@(), @()) + $extraOutput = Repo-Path "latency.txt" + if (!$isWindows) { + chmod +rw "$extraOutput" + } } elseif ($exeArgs.Contains("prate:1")) { $metric = "hps" } - $values = @(@(), @()) - $hasFailures = $false - $tcpSupported = ($io -ne "xdp" -and $io -ne "qtip" -and $io -ne "wsk") ? 1 : 0 for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) { # Set up all the parameters and paths for running the test. @@ -455,15 +462,7 @@ function Invoke-Secnetperf { $serverArgs += " -stats:1" $clientArgs += " -pconn:1" } - - $latency = $null - $extraOutput = $null - if ($metric -eq "latency") { - $latency = @(@(), @()) - $extraOutput = Repo-Path "latency.txt" - if (!$isWindows) { - chmod +rw "$extraOutput" - } + if ($extraOutput) { $clientArgs += " -extraOutputFile:$extraOutput" } From eb47092a6adcbc6dfb348bf375f6788615667f7f Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Mon, 26 Feb 2024 11:51:01 -0500 Subject: [PATCH 5/6] more depth --- scripts/secnetperf.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index 03fe404478..ce95c8df39 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -329,7 +329,7 @@ Write-Host "Tests complete!" Write-Host "`Writing test-results-$environment-$plat-$os-$arch-$tls-$io.sql" $SQL | Set-Content -Path "test-results-$environment-$plat-$os-$arch-$tls-$io.sql" Write-Host "`Writing json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" - $json | ConvertTo-Json | Set-Content -Path "json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" + $json | ConvertTo-Json -Depth 4 | Set-Content -Path "json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" } # Clear out any exit codes from previous commands. From afea67f52908dc63ab5c72e6cfe32382c79604fa Mon Sep 17 00:00:00 2001 From: Nick Banks Date: Mon, 26 Feb 2024 12:18:16 -0500 Subject: [PATCH 6/6] Remove plat --- scripts/secnetperf.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1 index ce95c8df39..926cea3b79 100644 --- a/scripts/secnetperf.ps1 +++ b/scripts/secnetperf.ps1 @@ -326,10 +326,10 @@ Write-Host "Tests complete!" } catch { } # Save the test results (sql and json). - Write-Host "`Writing test-results-$environment-$plat-$os-$arch-$tls-$io.sql" - $SQL | Set-Content -Path "test-results-$environment-$plat-$os-$arch-$tls-$io.sql" - Write-Host "`Writing json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" - $json | ConvertTo-Json -Depth 4 | Set-Content -Path "json-test-results-$environment-$plat-$os-$arch-$tls-$io.json" + Write-Host "`Writing test-results-$environment-$os-$arch-$tls-$io.sql" + $SQL | Set-Content -Path "test-results-$environment-$os-$arch-$tls-$io.sql" + Write-Host "`Writing json-test-results-$environment-$os-$arch-$tls-$io.json" + $json | ConvertTo-Json -Depth 4 | Set-Content -Path "json-test-results-$environment-$os-$arch-$tls-$io.json" } # Clear out any exit codes from previous commands.