Merge branch 'main' into dev/daiki/ubuntu22_test_resume

microsoft · Jun 7, 2024 · e5b512a · e5b512a
2 parents 0c9b5a6 + 55e05ee
commit e5b512a
Show file tree

Hide file tree

Showing 10 changed files with 188 additions and 28 deletions.
diff --git a/.docker/ubuntu-24.04/Dockerfile b/.docker/ubuntu-24.04/Dockerfile
@@ -0,0 +1,106 @@
+# Stage 1: Base image
+FROM ubuntu:24.04@sha256:e3f92abc0967a6c19d0dfa2d55838833e947b9d74edbcb0113e48535ad4be12a AS base
+
+LABEL org.opencontainers.image.source https://github.com/microsoft/msquic
+
+ENV DEBIAN_FRONTEND=noninteractive
+
+SHELL ["/bin/bash", "-c"]
+
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    apt-transport-https \
+    ca-certificates \
+    gnupg \
+    software-properties-common \
+    wget && \
+    wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null | gpg --dearmor - | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null && \
+    apt-add-repository 'deb https://apt.kitware.com/ubuntu/ jammy main' && \
+    apt-add-repository ppa:lttng/stable-2.13 && \
+    apt-get update && \
+    apt-get install -y tzdata \
+    build-essential \
+    cmake \
+    clang \
+    git \
+    make \
+    sudo \
+    lttng-tools \
+    perl \
+    nasm \
+    ruby \
+    ruby-dev \
+    rpm \
+    cppcheck \
+    clang-tidy \
+    gdb \
+    liblttng-ust-dev \
+    libssl-dev \
+    libnuma-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN gem install fpm
+RUN wget -q https://launchpad.net/ubuntu/+archive/primary/+files/libicu72_72.1-3ubuntu3_amd64.deb \
+    && dpkg -i ./libicu72_72.1-3ubuntu3_amd64.deb \
+    && rm ./libicu72_72.1-3ubuntu3_amd64.deb \
+    && wget -q https://packages.microsoft.com/config/ubuntu/18.04/packages-microsoft-prod.deb -O packages-microsoft-prod.deb && \
+    sudo dpkg -i packages-microsoft-prod.deb && \
+    sudo add-apt-repository universe && \
+    sudo apt-get update -y && \
+    sudo apt-get install -y \
+        # INSTALL POWERSHELl
+        powershell \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN git config --global safe.directory '*'
+
+# Stage 2: dependencies for cross-build.
+#          ubuntu 24.04 can install both xdp and arm dependencies
+FROM base AS cross-build
+
+RUN apt-get update -y && apt-get install --no-install-recommends -y \
+    libnl-3-dev \
+    libnl-genl-3-dev \
+    libnl-route-3-dev \
+    zlib1g-dev \
+    zlib1g \
+    pkg-config \
+    m4 \
+    libpcap-dev \
+    libelf-dev \
+    libc6-dev-i386 \
+    libxdp-dev \
+    libbpf-dev \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN apt-get update && apt-get install -y \
+    gcc-arm-linux-gnueabihf \
+    g++-arm-linux-gnueabihf \
+    binutils-arm-linux-gnueabihf \
+    gcc-aarch64-linux-gnu \
+    g++-aarch64-linux-gnu \
+    binutils-aarch64-linux-gnu \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN dpkg --add-architecture arm64 && \
+    dpkg --add-architecture armhf
+
+RUN mv /etc/apt/sources.list /etc/apt/sources.list.int && \
+    cat /etc/apt/sources.list.int | grep "^deb" | sed 's/deb /deb [arch=amd64] /g' > /etc/apt/sources.list && \
+    rm /etc/apt/sources.list.int && \
+    rm /etc/apt/sources.list.d/ubuntu.sources
+
+RUN echo $' \n\
+deb [arch=armhf,arm64] http://ports.ubuntu.com/ noble main restricted universe multiverse \n\
+deb [arch=armhf,arm64] http://ports.ubuntu.com/ noble-updates main restricted universe multiverse \n\
+deb [arch=armhf,arm64] http://ports.ubuntu.com/ noble-backports main restricted universe multiverse \n\
+' > /etc/apt/sources.list.d/arm-cross-compile-sources.list
+
+RUN apt-get update \
+    && apt-get --no-install-recommends install -y \
+    liblttng-ust-dev:arm64 \
+    liblttng-ust-dev:armhf \
+    libssl-dev:arm64 \
+    libssl-dev:armhf \
+    libnuma-dev:arm64 \
+    libnuma-dev:armhf \
+    && rm -rf /var/lib/apt/lists/*
diff --git a/.github/workflows/docker-publish-xcomp.yml b/.github/workflows/docker-publish-xcomp.yml
@@ -7,12 +7,14 @@ on:
     - .github/workflows/docker-publish-xcomp.yml
     - .docker/ubuntu-20.04/*
     - .docker/ubuntu-22.04/*
+    - .docker/ubuntu-24.04/*
   pull_request:
     branches: [ main ]
     paths:
     - .github/workflows/docker-publish-xcomp.yml
     - .docker/ubuntu-20.04/*
     - .docker/ubuntu-22.04/*
+    - .docker/ubuntu-24.04/*
 
 permissions: read-all
 
@@ -27,12 +29,15 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version: ['20.04', '22.04']
+        version: ['20.04', '22.04', '24.04']
         target: ['xdp', 'cross']
         exclude:
-        # Exclude xdp build for ubuntu 20.04
-        - version: '20.04'
-          target: 'xdp'
+          # 20.04 doesn't support XDP, 24.04's XDP is included in cross
+          - version: '20.04'
+            target: 'xdp'
+          - version: '24.04'
+            target: 'xdp'
+
     name: Build
     runs-on: ubuntu-latest
     permissions:

diff --git a/docs/InteropServerSetup.md b/docs/InteropServerSetup.md
@@ -91,7 +91,7 @@ Feel free to update `$OutputDir` to whatever local directory you wish.
 
 The following commands (run as root) should configure core dumps to be created in the local directory:
 
-```
+```sh
 # Enable core dumps for the system.
 sudo sh -c "echo 'root soft core unlimited' >> /etc/security/limits.conf"
 sudo sh -c "echo 'root hard core unlimited' >> /etc/security/limits.conf"
@@ -101,3 +101,12 @@ sudo sh -c "echo '* hard core unlimited' >> /etc/security/limits.conf"
 # Set the core dump pattern.
 sudo sh -c "echo -n '%e.%p.%t.core' > /proc/sys/kernel/core_pattern"
 ```
+
+The following are sudo basd run especially for using XDP
+```sh
+# Increase the number of file descriptors.
+sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf"
+sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf"
+sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf"
+sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf"
+```
diff --git a/scripts/prepare-machine.ps1 b/scripts/prepare-machine.ps1
@@ -558,6 +558,11 @@ if ($IsLinux) {
         sudo sh -c "echo 'root hard core unlimited' >> /etc/security/limits.conf"
         sudo sh -c "echo '* soft core unlimited' >> /etc/security/limits.conf"
         sudo sh -c "echo '* hard core unlimited' >> /etc/security/limits.conf"
+        # Increase the number of file descriptors.
+        sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf"
         #sudo cat /etc/security/limits.conf
 
         # Set the core dump pattern.

diff --git a/scripts/run-gtest.ps1 b/scripts/run-gtest.ps1
@@ -241,15 +241,15 @@ $FinalResultsPath = "$($LogDir)-results.xml"
 # Base XML results data.
 $XmlResults = [xml]@"
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="0" failures="0" disabled="0" errors="0" time="0" timestamp="date" name="AllTests">
+<testsuites tests="0" failures="0" retried="0" disabled="0" errors="0" time="0" timestamp="date" name="AllTests">
 </testsuites>
 "@
 $XmlResults.testsuites.timestamp = Get-Date -UFormat "%Y-%m-%dT%T"
 
 # XML for creating new (failure) result data.
 $FailXmlText = @"
 <?xml version="1.0" encoding="UTF-8"?>
-<testsuites tests="1" failures="1" disabled="0" errors="0" time="0" name="AllTests">
+<testsuites tests="1" failures="1" retried="0" disabled="0" errors="0" time="0" name="AllTests">
   <testsuite name="TestSuiteName" tests="1" failures="1" disabled="0" errors="0" timestamp="date" time="0" >
     <testcase name="TestCaseName" status="run" result="completed" time="0" timestamp="date" classname="TestSuiteName">
       <failure message="Application Crashed" type=""><![CDATA[Application Crashed]]></failure>
@@ -287,6 +287,7 @@ function Add-XmlResults($TestCase) {
     }
 
     $IsFailure = $NewXmlResults.testsuites.failures -eq 1
+    $IsRetry = $false
     $Time = $NewXmlResults.testsuites.testsuite.testcase.time -as [Decimal]
 
     $Node = $null
@@ -295,6 +296,7 @@ function Add-XmlResults($TestCase) {
         $Node = $XmlResults.testsuites.testsuite | Where-Object { $_.Name -eq $TestSuiteName }
     }
     if ($null -ne $Node) {
+        $IsRetry = $Node.testcase | Where-Object { $_.Name -eq $TestCaseName }
         # Already has a matching test suite. Add the test case to it.
         $Node.tests = ($Node.tests -as [Int]) + 1
         if ($IsFailure) {
@@ -311,6 +313,9 @@ function Add-XmlResults($TestCase) {
 
     # Update the top level test and failure counts.
     $XmlResults.testsuites.tests = ($XmlResults.testsuites.tests -as [Int]) + 1
+    if ($IsRetry) {
+        $XmlResults.testsuites.retried = ($XmlResults.testsuites.retried -as [Int]) + 1
+    }
     if ($IsFailure) {
         $XmlResults.testsuites.failures = ($XmlResults.testsuites.failures -as [Int]) + 1
     }
@@ -372,10 +377,10 @@ function Start-TestExecutable([String]$Arguments, [String]$OutputDir) {
 }
 
 # Asynchronously starts a single msquictest test case running.
-function Start-TestCase([String]$Name) {
+function Start-TestCase([String]$Name, [int]$Trial = 1) {
 
     $InstanceName = $Name.Replace("/", "_")
-    $LocalLogDir = Join-Path $LogDir $InstanceName
+    $LocalLogDir = Join-Path $LogDir ($InstanceName + "_$Trial")
     mkdir $LocalLogDir | Out-Null
 
     if ($LogProfile -ne "None") {
@@ -678,6 +683,8 @@ function Wait-TestCase($TestCase) {
             Remove-Item $TestCase.LogDir -Recurse -Force | Out-Null
         }
     }
+
+    return $AnyTestFailed
 }
 
 # Runs the test executable to query all available test cases, parses the console
@@ -836,7 +843,10 @@ try {
     } else {
         # Run the test cases individually.
         for ($i = 0; $i -lt $TestCount; $i++) {
-            Wait-TestCase (Start-TestCase ($TestCases -as [String[]])[$i])
+            $AnyTestFailed = Wait-TestCase (Start-TestCase ($TestCases -as [String[]])[$i] 1)
+            if ($AnyTestFailed) {
+                Wait-TestCase (Start-TestCase ($TestCases -as [String[]])[$i] 2)
+            }
             if (!$NoProgress) {
                 Write-Progress -Activity "Running tests" -Status "Progress:" -PercentComplete ($i/$TestCount*100)
             }
@@ -890,6 +900,7 @@ try {
 
     $TestCount = $XmlResults.testsuites.tests -as [Int]
     $TestsFailed = $XmlResults.testsuites.failures -as [Int]
+    $TestsRetried = $XmlResults.testsuites.retried -as [Int]
 
     # Uninstall the kernel mode test driver and revert the msquic driver.
     if ($Kernel -ne "") {
@@ -913,11 +924,17 @@ try {
     Log "$($TestCount) test(s) run."
     if ($KeepOutputOnSuccess -or ($TestsFailed -ne 0) -or ($global:CrashedProcessCount -ne 0)) {
         Log "Output can be found in $($LogDir)"
-        if ($ErrorsAsWarnings) {
-            Write-Warning "$($TestsFailed) test(s) failed."
-            Write-Warning "$($TestsFailed) test(s) failed, $($global:CrashedProcessCount) test(s) crashed."
+        $ConsistentFailure = $TestsFailed - $TestsRetried
+        $PassedWithRetry = $TestsRetried - $ConsistentFailure
+        if ($ErrorsAsWarnings -or
+            (($IsolationMode -eq "Isolated") -and ($TestsFailed -ne 0) -and ($TestsFailed -eq $TestsRetried))) {
+            if ($TestsFailed -eq $TestsRetried) {
+                Write-Warning "$($TestsRetried) test(s) passed with retry, $($global:CrashedProcessCount) test(s) crashed."
+            } else { # for $ErrorsAsWarning
+                Write-Warning "$($ConsistentFailure) test(s) failed with retry, $($PassedWithRetry) test(s) passed with retry, $($global:CrashedProcessCount) test(s) crashed."
+            }
         } else {
-            Write-Error "$($TestsFailed) test(s) failed, $($global:CrashedProcessCount) test(s) crashed."
+            Write-Error "$($ConsistentFailure) test(s) failed with retry, $($PassedWithRetry) test(s) passed with retry, $($global:CrashedProcessCount) test(s) crashed."
             $LastExitCode = 1
         }
     } elseif ($AZP -and $TestCount -eq 0) {

diff --git a/scripts/secnetperf-helpers.psm1 b/scripts/secnetperf-helpers.psm1
@@ -254,9 +254,14 @@ function Cleanup-State {
 # Waits for a remote job to be ready based on looking for a particular string in
 # the output.
 function Start-RemoteServer {
-    param ($Session, $Command)
+    param ($Session, $Command, $ServerArgs, $UseSudo)
     # Start the server on the remote in an async job.
-    $job = Invoke-Command -Session $Session -ScriptBlock { iex $Using:Command } -AsJob
+
+    if ($UseSudo) {
+        $job = Invoke-Command -Session $Session -ScriptBlock { iex "sudo LD_LIBRARY_PATH=$(Split-Path $Using:Command -Parent)  $Using:Command $Using:ServerArgs" } -AsJob
+    } else {
+        $job = Invoke-Command -Session $Session -ScriptBlock { iex "$Using:Command $Using:ServerArgs"} -AsJob
+    }
     # Poll the job for 10 seconds to see if it started.
     $stopWatch = [system.diagnostics.stopwatch]::StartNew()
     while ($stopWatch.ElapsedMilliseconds -lt 10000) {
@@ -303,15 +308,22 @@ function Stop-RemoteServer {
 
 # Creates a new local process to asynchronously run the test.
 function Start-LocalTest {
-    param ($FullPath, $FullArgs, $OutputDir)
+    param ($FullPath, $FullArgs, $OutputDir, $UseSudo)
     $pinfo = New-Object System.Diagnostics.ProcessStartInfo
     if ($IsWindows) {
         $pinfo.FileName = $FullPath
         $pinfo.Arguments = $FullArgs
     } else {
         # We use bash to execute the test so we can collect core dumps.
-        $pinfo.FileName = "bash"
-        $pinfo.Arguments = "-c `"ulimit -c unlimited && LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''`""
+        $NOFILE = Invoke-Expression "bash -c 'ulimit -n'"
+        $CommonCommand = "ulimit -n $NOFILE && ulimit -c unlimited && LD_LIBRARY_PATH=$(Split-Path $FullPath -Parent) LSAN_OPTIONS=report_objects=1 ASAN_OPTIONS=disable_coredump=0:abort_on_error=1 UBSAN_OPTIONS=halt_on_error=1:print_stacktrace=1 $FullPath $FullArgs && echo ''"
+        if ($UseSudo) {
+            $pinfo.FileName = "/usr/bin/sudo"
+            $pinfo.Arguments = "/usr/bin/bash -c `"$CommonCommand`""
+        } else {
+            $pinfo.FileName = "bash"
+            $pinfo.Arguments = "-c `"$CommonCommand`""
+        }
         $pinfo.WorkingDirectory = $OutputDir
     }
     $pinfo.RedirectStandardOutput = $true
@@ -499,14 +511,13 @@ function Invoke-Secnetperf {
     }
 
     # These scenarios are currently broken! TODO - Figure out why and fix them.
-    if (($io -eq "wsk" -and $metric -eq "hps") -or
-        (!$isWindows -and $io -eq "xdp" -and $metric -eq "hps")) {
+    if ($io -eq "wsk" -and $metric -eq "hps") {
         Write-Host "> secnetperf $clientArgs BROKEN!"
         continue
     }
 
      # Linux XDP requires sudo for now
-    $sudo = (!$IsWindows -and $io -eq "xdp") ? "sudo -E LD_LIBRARY_PATH=$RemoteDir/$(Split-Path $SecNetPerfPath -Parent) " : ""
+    $useSudo = (!$IsWindows -and $io -eq "xdp")
 
     $artifactName = $tcp -eq 0 ? "$TestId-quic" : "$TestId-tcp"
     New-Item -ItemType Directory "artifacts/logs/$artifactName" -ErrorAction Ignore | Out-Null
@@ -536,7 +547,7 @@ function Invoke-Secnetperf {
 
     # Start the server running.
     "> secnetperf $serverArgs" | Add-Content $serverOut
-    $job = Start-RemoteServer $Session "$sudo$RemoteDir/$SecNetPerfPath $serverArgs"
+    $job = Start-RemoteServer $Session "$RemoteDir/$SecNetPerfPath" $serverArgs $useSudo
 
     # Run the test multiple times, failing (for now) only if all tries fail.
     # TODO: Once all failures have been fixed, consider all errors fatal.
@@ -546,12 +557,12 @@ function Invoke-Secnetperf {
         Write-Host "==============================`nRUN $($try+1):"
         "> secnetperf $clientArgs" | Add-Content $clientOut
         try {
-            $process = Start-LocalTest "$sudo$clientPath" $clientArgs $artifactDir
+            $process = Start-LocalTest "$clientPath" $clientArgs $artifactDir $useSudo
             $rawOutput = Wait-LocalTest $process $artifactDir ($io -eq "wsk") 30000
             Write-Host $rawOutput
             $values[$tcp] += Get-TestOutput $rawOutput $metric
             if ($extraOutput) {
-                if ($sudo -ne "") {
+                if ($useSudo) {
                     sudo chown $UserName $extraOutput
                 }
                 $latency[$tcp] += Get-LatencyOutput $extraOutput

diff --git a/scripts/secnetperf.ps1 b/scripts/secnetperf.ps1
@@ -273,6 +273,11 @@ if (!$isWindows) {
         sudo sh -c "echo "root hard core unlimited" >> /etc/security/limits.conf"
         sudo sh -c "echo "* soft core unlimited" >> /etc/security/limits.conf"
         sudo sh -c "echo "* hard core unlimited" >> /etc/security/limits.conf"
+        # Increase the number of file descriptors.
+        sudo sh -c "echo 'root soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo 'root hard nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* soft nofile 1048576' >> /etc/security/limits.conf"
+        sudo sh -c "echo '* hard nofile 1048576' >> /etc/security/limits.conf"
     }
 
     # Set the core dump pattern.

diff --git a/scripts/spin.ps1 b/scripts/spin.ps1
@@ -188,7 +188,8 @@ if (![string]::IsNullOrWhiteSpace($ExtraArtifactDir)) {
 
 # Run the script.
 if ($IsLinux -and $UseXdp) {
-    Invoke-Expression ('sudo pwsh -c "$RunExecutable $Arguments"')
+    $NOFILE = Invoke-Expression "bash -c 'ulimit -n'"
+    Invoke-Expression ('/usr/bin/sudo bash -c "ulimit -n $NOFILE && pwsh $RunExecutable $Arguments"')
 } else {
     Invoke-Expression ($RunExecutable + " " + $Arguments)
 }