Merge branch 'main' into dev/daiki/winkernel_merging_exp

scripts/secnetperf-helpers.psm1

@@ -540,7 +540,7 @@ function Get-LatencyOutput {
 
 # Invokes secnetperf with the given arguments for both TCP and QUIC.
 function Invoke-Secnetperf {
-    param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $TestId, $ExeArgs, $io, $Filter, $Environment, $RunId, $SyncerSecret)
+    param ($Session, $RemoteName, $RemoteDir, $UserName, $SecNetPerfPath, $LogProfile, $Scenario, $io, $Filter, $Environment, $RunId, $SyncerSecret)
 
     $values = @(@(), @())
     $latency = $null
@@ -552,26 +552,25 @@ function Invoke-Secnetperf {
         $tcpSupported = 0
     }
     $metric = "throughput"
-    if ($exeArgs.Contains("conns:16cpu")) { # TODO: figure out a better way to detect max RPS tests
+    if ($Scenario.Contains("rps")) {
         $metric = "rps"
-    } elseif ($exeArgs.Contains("plat:1")) {
+    } elseif ($Scenario.Contains("latency")) {
         $metric = "latency"
         $latency = @(@(), @())
         $extraOutput = Repo-Path "latency.txt"
         if (!$isWindows) {
             chmod +rw "$extraOutput"
         }
-    } elseif ($exeArgs.Contains("prate:1")) {
+    } elseif ($Scenario.Contains("hps")) {
         $metric = "hps"
     }
 
     for ($tcp = 0; $tcp -le $tcpSupported; $tcp++) {
 
     # Set up all the parameters and paths for running the test.
-    $execMode = $ExeArgs.Substring(0, $ExeArgs.IndexOf(" ")) # First arg is the exec mode
     $clientPath = Repo-Path $SecNetPerfPath
-    $serverArgs = "$execMode -io:$io"
-    $clientArgs = "-target:$RemoteName $ExeArgs -tcp:$tcp -trimout -watchdog:25000"
+    $serverArgs = "-scenario:$Scenario -io:$io"
+    $clientArgs = "-target:$RemoteName -scenario:$Scenario -io:$io -tcp:$tcp -trimout -watchdog:25000"
     if ($io -eq "xdp" -or $io -eq "qtip") {
         $serverArgs += " -pollidle:10000"
         $clientArgs += " -pollidle:10000"
@@ -600,9 +599,9 @@ function Invoke-Secnetperf {
     $useSudo = (!$isWindows -and $io -eq "xdp")
 
     if ($tcp -eq 0) {
-        $artifactName = "$TestId-quic"
+        $artifactName = "$Scenario-quic"
     } else {
-        $artifactName = "$TestId-tcp"
+        $artifactName = "$Scenario-tcp"
     }
     New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null
     $artifactDir = Repo-Path "artifacts/logs/$artifactName"
@@ -731,14 +730,14 @@ function Invoke-Secnetperf {
     }
 }
 
-function CheckRegressionResult($values, $testid, $transport, $regressionJson, $envStr) {
+function CheckRegressionResult($values, $scenario, $transport, $regressionJson, $envStr) {
 
     $sum = 0
     foreach ($item in $values) {
         $sum += $item
     }
     $avg = $sum / $values.Length
-    $Testid = "$testid-$transport"
+    $Scenario = "$scenario-$transport"
 
     $res = @{
         Baseline = "N/A"
@@ -750,14 +749,14 @@ function CheckRegressionResult($values, $testid, $transport, $regressionJson, $e
     }
 
     try {
-        $res.Baseline = $regressionJson.$Testid.$envStr.baseline
-        $res.BestResult = $regressionJson.$Testid.$envStr.BestResult
-        $res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit
+        $res.Baseline = $regressionJson.$Scenario.$envStr.baseline
+        $res.BestResult = $regressionJson.$Scenario.$envStr.BestResult
+        $res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit
         $res.CumulativeResult = $avg
         $res.AggregateFunction = "AVG"
 
         if ($avg -lt $res.Baseline) {
-            Write-GHError "Regression detected in $Testid for $envStr. See summary table for details."
+            Write-GHError "Regression detected in $Scenario for $envStr. See summary table for details."
             $res.HasRegression = $true
         }
     } catch {
@@ -767,7 +766,7 @@ function CheckRegressionResult($values, $testid, $transport, $regressionJson, $e
     return $res
 }
 
-function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envStr) {
+function CheckRegressionLat($values, $regressionJson, $scenario, $transport, $envStr) {
 
     # TODO: Right now, we are not using a watermark based method for regression detection of latency percentile values because we don't know how to determine a "Best Ever" distribution.
     #       (we are just looking at P0, P50, P99 columns, and computing the baseline for each percentile as the mean - 2 * std of the last 20 runs. )
@@ -780,7 +779,7 @@ function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envS
     }
 
     $RpsAvg /= $NumRuns
-    $Testid = "$testid-$transport"
+    $Scenario = "$scenario-$transport"
 
     $res = @{
         Baseline = "N/A"
@@ -792,14 +791,14 @@ function CheckRegressionLat($values, $regressionJson, $testid, $transport, $envS
     }
 
     try {
-        $res.Baseline = $regressionJson.$Testid.$envStr.baseline
-        $res.BestResult = $regressionJson.$Testid.$envStr.BestResult
-        $res.BestResultCommit = $regressionJson.$Testid.$envStr.BestResultCommit
+        $res.Baseline = $regressionJson.$Scenario.$envStr.baseline
+        $res.BestResult = $regressionJson.$Scenario.$envStr.BestResult
+        $res.BestResultCommit = $regressionJson.$Scenario.$envStr.BestResultCommit
         $res.CumulativeResult = $RpsAvg
         $res.AggregateFunction = "AVG"
 
         if ($RpsAvg -lt $res.Baseline) {
-            Write-GHError "RPS Regression detected in $Testid for $envStr. See summary table for details."
+            Write-GHError "RPS Regression detected in $Scenario for $envStr. See summary table for details."
             $res.HasRegression = $true
         }
     } catch {

scripts/secnetperf.ps1

@@ -251,17 +251,11 @@ if ($isWindows) {
     $kernelVersion = bash -c "uname -r"
     $json["os_version"] = "$osName $kernelVersion"
 }
-$allTests = [System.Collections.Specialized.OrderedDictionary]::new()
 
-# > All tests:
-$allTests["tput-up"] = "-exec:maxtput -up:12s -ptput:1"
-$allTests["tput-down"] = "-exec:maxtput -down:12s -ptput:1"
-$allTests["hps-conns-100"] = "-exec:maxtput -rconn:1 -share:1 -conns:100 -run:12s -prate:1"
-$allTests["rps-up-512-down-4000"] = "-exec:lowlat -rstream:1 -up:512 -down:4000 -run:20s -plat:1"
-$allTests["max-rps-up-512-down-4000"] = "-exec:lowlat -conns:16cpu -streams:10 -rstream:1 -up:512 -down:4000 -run:20s -plat:1"
+# Test all supported scenarios.
+$allScenarios = @("upload", "download", "hps", "rps", "rps-multi", "latency")
 
 $hasFailures = $false
-$json["run_args"] = $allTests
 
 try {
 
@@ -336,9 +330,8 @@ $regressionJson = Get-Content -Raw -Path "watermark_regression.json" | ConvertFr
 
 # Run all the test cases.
 Write-Host "Setup complete! Running all tests"
-foreach ($testId in $allTests.Keys) {
-    $ExeArgs = $allTests[$testId] + " -io:$io"
-    $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $testId $ExeArgs $io $filter $environment $RunId $SyncerSecret
+foreach ($scenario in $allScenarios) {
+    $Output = Invoke-Secnetperf $Session $RemoteName $RemoteDir $UserName $SecNetPerfPath $LogProfile $scenario $io $filter $environment $RunId $SyncerSecret
     $Test = $Output[-1]
     if ($Test.HasFailures) { $hasFailures = $true }
 
@@ -349,15 +342,15 @@ foreach ($testId in $allTests.Keys) {
         } else {
             $transport = "quic"
         }
-        $json["$testId-$transport"] = $Test.Values[$tcp]
+        $json["$scenario-$transport"] = $Test.Values[$tcp]
 
         if ($Test.Metric -eq "latency") {
-            $json["$testId-$transport-lat"] = $Test.Latency[$tcp]
-            $LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $testId $transport "$os-$arch-$environment-$io-$tls"
-            $json["$testId-$transport-regression"] = $LatencyRegression
+            $json["$scenario-$transport-lat"] = $Test.Latency[$tcp]
+            $LatencyRegression = CheckRegressionLat $Test.Values[$tcp] $regressionJson $scenario $transport "$os-$arch-$environment-$io-$tls"
+            $json["$scenario-$transport-regression"] = $LatencyRegression
         } else {
-            $ResultRegression = CheckRegressionResult $Test.Values[$tcp] $testId $transport $regressionJson "$os-$arch-$environment-$io-$tls"
-            $json["$testId-$transport-regression"] = $ResultRegression
+            $ResultRegression = CheckRegressionResult $Test.Values[$tcp] $scenario $transport $regressionJson "$os-$arch-$environment-$io-$tls"
+            $json["$scenario-$transport-regression"] = $ResultRegression
         }
     }
 }

src/inc/msquichelper.h

@@ -304,7 +304,7 @@ IsValue(
     _In_z_ const char* toTestAgainst
     )
 {
-    return _strnicmp(name, toTestAgainst, CXPLAT_MIN(strlen(name), strlen(toTestAgainst))) == 0;
+    return _strnicmp(name, toTestAgainst, strlen(toTestAgainst)) == 0;
 }
 
 inline

src/perf/lib/PerfClient.cpp

@@ -108,6 +108,52 @@ PerfClient::Init(
 
     CountMult[0] = CxPlatProcCount();
 
+    //
+    // Scenario profile sets new defauls for values below, that may then be
+    // further overridden by command line arguments.
+    //
+    const char* ScenarioStr = GetValue(argc, argv, "scenario");
+    if (ScenarioStr != nullptr) {
+        if (IsValue(ScenarioStr, "upload")) {
+            Upload = S_TO_US(12); // 12 seconds
+            Timed = TRUE;
+            PrintThroughput = TRUE;
+        } else if (IsValue(ScenarioStr, "download")) {
+            Download = S_TO_US(12); // 12 seconds
+            Timed = TRUE;
+            PrintThroughput = TRUE;
+        } else if (IsValue(ScenarioStr, "hps")) {
+            ConnectionCount = 16 * CxPlatProcCount();
+            RunTime = S_TO_US(12); // 12 seconds
+            RepeatConnections = TRUE;
+            PrintIoRate = TRUE;
+        } else if (IsValue(ScenarioStr, "rps-multi")) {
+            Upload = 512;
+            Download = 4000;
+            ConnectionCount = 16 * CxPlatProcCount();
+            StreamCount = 100;
+            RunTime = S_TO_US(20); // 20 seconds
+            RepeatStreams = TRUE;
+            PrintLatency = TRUE;
+        } else if (IsValue(ScenarioStr, "rps")) {
+            Upload = 512;
+            Download = 4000;
+            StreamCount = 100;
+            RunTime = S_TO_US(20); // 20 seconds
+            RepeatStreams = TRUE;
+            PrintLatency = TRUE;
+        } else if (IsValue(ScenarioStr, "latency")) {
+            Upload = 512;
+            Download = 4000;
+            RunTime = S_TO_US(20); // 20 seconds
+            RepeatStreams = TRUE;
+            PrintLatency = TRUE;
+        } else {
+            WriteOutput("Failed to parse scenario profile[%s]!\n", ScenarioStr);
+            return QUIC_STATUS_INVALID_PARAMETER;
+        }
+    }
+
     //
     // Remote target/server options
     //

src/perf/lib/SecNetPerfMain.cpp

@@ -33,6 +33,36 @@ uint8_t PerfDefaultAffinitizeThreads = false;
 #ifdef _KERNEL_MODE
 volatile int BufferCurrent;
 char Buffer[BufferLength];
+static inline LONG _strtol(const CHAR* nptr, CHAR** endptr, int base) {
+    UNREFERENCED_PARAMETER(base);
+    ULONG temp;
+    RtlCharToInteger(nptr, base, &temp);
+    if (endptr != NULL) {
+        const CHAR* ptr = nptr;
+        while (*ptr >= '0' && *ptr <= '9') {
+            ptr++;
+        }
+        *endptr = (CHAR*)ptr;
+    }
+    return (LONG)temp;
+}
+
+static inline ULONG _strtoul(const CHAR* nptr, CHAR** endptr, int base) {
+    UNREFERENCED_PARAMETER(base);
+    ULONG temp;
+    RtlCharToInteger(nptr, base, &temp);
+    if (endptr != NULL) {
+        const CHAR* ptr = nptr;
+        while (*ptr >= '0' && *ptr <= '9') {
+            ptr++;
+        }
+        *endptr = (CHAR*)ptr;
+    }
+    return temp;
+}
+#else
+#define _strtol strtol
+#define _strtoul strtoul
 #endif
 
 static
@@ -78,6 +108,8 @@ PrintHelp(
         "  -platency<0/1>           Print latency statistics. (def:0)\n"
         "\n"
         "  Scenario options:\n"
+        "  -scenario:<profile>      Scenario profile to use.\n"
+        "                            - {upload, download, hps, rps, rps-multi, latency}.\n"
         "  -conns:<####>            The number of connections to use. (def:1)\n"
         "  -streams:<####>          The number of streams to send on at a time. (def:0)\n"
         "  -upload:<####>[unit]     The length of bytes to send on each stream, with an optional (time or length) unit. (def:0)\n"
@@ -96,13 +128,16 @@ PrintHelp(
         "  -pollidle:<time_us>      Amount of time to poll while idle before sleeping (default: 0).\n"
         "  -ecn:<0/1>               Enables/disables sender-side ECN support. (def:0)\n"
         "  -qeo:<0/1>               Allows/disallowes QUIC encryption offload. (def:0)\n"
-#ifndef _KERNEL_MODE
+#ifdef _KERNEL_MODE
         "  -io:<mode>               Configures a requested network IO model to be used.\n"
         "                            - {iocp, rio, xdp, qtip, wsk, epoll, kqueue}\n"
+#else
+        "  -io:<mode>               Configures a requested network IO model to be used.\n"
+        "                            - {xdp}\n"
+#endif // _KERNEL_MODE
         "  -cpu:<cpu_index>         Specify the processor(s) to use.\n"
         "  -cipher:<value>          Decimal value of 1 or more QUIC_ALLOWED_CIPHER_SUITE_FLAGS.\n"
         "  -highpri:<0/1>           Configures MsQuic to run threads at high priority. (def:0)\n"
-#endif // _KERNEL_MODE
         "\n",
         PERF_DEFAULT_PORT,
         PERF_DEFAULT_PORT
@@ -142,10 +177,9 @@ QuicMainStart(
     QUIC_EXECUTION_CONFIG* Config = (QUIC_EXECUTION_CONFIG*)RawConfig;
     Config->PollingIdleTimeoutUs = 0; // Default to no polling.
     bool SetConfig = false;
-
-#ifndef _KERNEL_MODE
     const char* IoMode = GetValue(argc, argv, "io");
 
+#ifndef _KERNEL_MODE
     if (IoMode && IsValue(IoMode, "qtip")) {
         Config->Flags |= QUIC_EXECUTION_CONFIG_FLAG_QTIP;
         SetConfig = true;
@@ -156,6 +190,8 @@ QuicMainStart(
         SetConfig = true;
     }
 
+#endif // _KERNEL_MODE
+
     if (IoMode && IsValue(IoMode, "xdp")) {
         Config->Flags |= QUIC_EXECUTION_CONFIG_FLAG_XDP;
         SetConfig = true;
@@ -164,15 +200,15 @@ QuicMainStart(
     const char* CpuStr;
     if ((CpuStr = GetValue(argc, argv, "cpu")) != nullptr) {
         SetConfig = true;
-        if (strtol(CpuStr, nullptr, 10) == -1) {
+        if (_strtol(CpuStr, nullptr, 10) == -1) {
             for (uint32_t i = 0; i < CxPlatProcCount() && Config->ProcessorCount < 256; ++i) {
                 Config->ProcessorList[Config->ProcessorCount++] = (uint16_t)i;
             }
         } else {
             do {
                 if (*CpuStr == ',') CpuStr++;
                 Config->ProcessorList[Config->ProcessorCount++] =
-                    (uint16_t)strtoul(CpuStr, (char**)&CpuStr, 10);
+                    (uint16_t)_strtoul(CpuStr, (char**)&CpuStr, 10);
             } while (*CpuStr && Config->ProcessorCount < 256);
         }
     }
@@ -188,7 +224,6 @@ QuicMainStart(
         Config->Flags |= QUIC_EXECUTION_CONFIG_FLAG_AFFINITIZE;
         SetConfig = true;
     }
-#endif // _KERNEL_MODE
 
     if (TryGetValue(argc, argv, "pollidle", &Config->PollingIdleTimeoutUs)) {
         SetConfig = true;
@@ -206,6 +241,25 @@ QuicMainStart(
         return Status;
     }
 
+    const char* ScenarioStr = GetValue(argc, argv, "scenario");
+    if (ScenarioStr != nullptr) {
+        if (IsValue(ScenarioStr, "upload") ||
+            IsValue(ScenarioStr, "download") ||
+            IsValue(ScenarioStr, "hps")) {
+            PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_MAX_THROUGHPUT;
+            TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_MAX_THROUGHPUT;
+        } else if (
+            IsValue(ScenarioStr, "rps") ||
+            IsValue(ScenarioStr, "rps-multi") ||
+            IsValue(ScenarioStr, "latency")) {
+            PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_LOW_LATENCY;
+            TcpDefaultExecutionProfile = TCP_EXECUTION_PROFILE_LOW_LATENCY;
+        } else {
+            WriteOutput("Failed to parse scenario profile[%s]!\n", ScenarioStr);
+            return QUIC_STATUS_INVALID_PARAMETER;
+        }
+    }
+
     const char* ExecStr = GetValue(argc, argv, "exec");
     if (ExecStr != nullptr) {
         if (IsValue(ExecStr, "lowlat")) {
@@ -219,7 +273,8 @@ QuicMainStart(
         } else if (IsValue(ExecStr, "realtime")) {
             PerfDefaultExecutionProfile = QUIC_EXECUTION_PROFILE_TYPE_REAL_TIME;
         } else {
-            WriteOutput("Failed to parse execution profile[%s], use lowlat as default for QUIC, lowlat as default for TCP.\n", ExecStr);
+            WriteOutput("Failed to parse execution profile[%s]!\n", ExecStr);
+            return QUIC_STATUS_INVALID_PARAMETER;
         }
     }
 

src/test/MsQuicTests.h

@@ -712,6 +712,7 @@ static const GUID QUIC_TEST_DEVICE_INSTANCE =
 
 typedef struct {
     BOOLEAN UseDuoNic;
+    QUIC_EXECUTION_CONFIG Config;
     char CurrentDirectory[MAX_PATH];
 } QUIC_TEST_CONFIGURATION_PARAMS;