RunTests.ps1

[CmdletBinding()]
param(
    [switch]$NoRun,
    [switch]$NoDeploy,
    [switch]$ForceDeploy,
    [switch]$LogVerbose,
    [switch]$LogSuperVerbose,
    [switch]$Release,
    [Alias("L")][switch]$List,
    [Alias("CC")][switch]$CodeCoverage,
    [String[]]$Name,
    [String[]]$xName,
    [String]$Flavor = "Debug",
    [String]$Platform = "x86",
    [String]$MinVersion,
    [switch]$dllOnly,
    [switch]$appxOnly,
    [switch]$waitForDebugger,
    [switch]$waitForAppDebugger,
    [int]$Iterations = 1,
    [string]$LogFile = "TestResults.out",
    [String]$OutputPath = "$PSScriptRoot\CodeCoverageOutput",
    [String]$TestTimeout,
    [String]$MagellanInstallPath = "$env:USERPROFILE\.nuget\packages\microsoft.internal.magellan\5.4.170227001-pkges"
)

#
# NOTE: This file currently relies on TShell to execute commands on a remote machine, which is not available externally.
#       Use Open-Device to connect to a remote machine, then run this script to run tests on that machine.
#       TODO 19757865: Change this so that it no longer has a dependency on TShell.
#

function Build-TaefAppXIfNeeded
{
    param(
        [System.IO.FileSystemInfo]$MuxDllFile,
        [string]$ProjectName,
        [string]$Platform,
        [string]$Flavor
    )

    $projectFileName = "$($ProjectName).TAEF"

    Build-AppXIfNeeded -MuxDllFile $MuxDllFile -ProjectName $ProjectName -Platform $Platform -Flavor $Flavor -AppXPath "$projectFileName\AppPackages\$($ProjectName)_Test\$ProjectName.appx" -ExePath "$projectFileName\$ProjectName.exe" -ProjectPath "$($ProjectName)\TAEF\$projectFileName.csproj"
}

function Build-AppXIfNeeded
{
    param(
        [System.IO.FileSystemInfo]$MuxDllFile,
        [string]$AppXPath,
        [string]$ExePath,
        [string]$ProjectPath,
        [string]$ProjectName,
        [string]$Platform,
        [string]$Flavor
    )

    $testAppxFile = Get-Item "$PSScriptRoot\BuildOutput\$Flavor\$Platform\$AppXPath" -ErrorAction Ignore
    $testExeFile = Get-Item "$PSScriptRoot\BuildOutput\$Flavor\$Platform\$ExePath" -ErrorAction Ignore

    if ((!$testAppxFile) -or (!$testExeFile) -or ($testExeFile.LastWriteTime -gt $testAppxFile.LastWriteTime) -or ($muxDllFile.LastWriteTime -gt $testAppxFile.LastWriteTime))
    {
	    Write-Host ""
	    Write-Host "$ProjectName AppX does not exist or is not up-to-date. Running build to generate..." -foregroundcolor Magenta
	    Write-Host ""
	    $buildCmd = "$PSScriptRoot\build.cmd $($Platform.ToLower()) $($Flavor.ToLower()) /project `"$PSScriptRoot\test\$ProjectPath`""
	    Write-Host $buildCmd
	    Invoke-Expression $buildCmd
	    Write-Host ""
	    Write-Host "Rebuild complete." -foregroundcolor Magenta
    }
}

if (!(test-path variable:\DeviceAddress))
{
    Write-Host "";
    Write-Host "DeviceAddress variable is not set -- are you connected to a device?" -foregroundcolor red
    Write-Host "";
    return;
}

$deviceDir = "c:\data\test\bin"
$ignoredOutput = cmdd if not exist $deviceDir mkdir $deviceDir

$muxDllFile = Get-Item "$PSScriptRoot\BuildOutput\$Flavor\$Platform\Microsoft.UI.Xaml\Microsoft.UI.Xaml.dll"

if(!$Release)
{
    Build-TaefAppXIfNeeded -MuxDllFile $muxDllFile -ProjectName "MUXControlsTestApp" -Platform $Platform -Flavor $Flavor
    Build-TaefAppXIfNeeded -MuxDllFile $muxDllFile -ProjectName "IXMPTestApp" -Platform $Platform -Flavor $Flavor
    Build-AppXIfNeeded -MuxDllFile $muxDllFile -ProjectName "MUXControlsTestAppWPFPackage" -Platform $Platform -Flavor $Flavor -AppXPath "MUXControlsTestAppWPFPackage\AppPackages\MUXControlsTestAppWPFPackage_Test\MUXControlsTestAppWPFPackage.appx" -ExePath "MUXControlsTestAppWPF\MUXControlsTestAppWPF.exe" -ProjectPath "MUXControlsTestAppWPFPackage\MUXControlsTestAppWPFPackage.wapproj"
}
else
{
    Build-AppXIfNeeded -MuxDllFile $muxDllFile -ProjectName "NugetPackageTestApp" -Platform $Platform -Flavor $Flavor -AppXPath "NugetPackageTestApp\AppPackages\NugetPackageTestApp_Test\NugetPackageTestApp.appx" -ExePath "NugetPackageTestApp\NugetPackageTestApp.exe" -ProjectPath "MUXControlsReleaseTest\NugetPackageTestApp\NugetPackageTestApp.csproj"
    Build-AppXIfNeeded -MuxDllFile $muxDllFile -ProjectName "NugetPackageTestAppCX" -Platform $Platform -Flavor $Flavor -AppXPath "NugetPackageTestAppCX\AppPackages\NugetPackageTestAppCX_Test\NugetPackageTestAppCX.appx" -ExePath "NugetPackageTestAppCX\NugetPackageTestAppCX.exe" -ProjectPath "MUXControlsReleaseTest\NugetPackageTestAppCX\NugetPackageTestAppCX.csproj"
}

Write-Host ""

#
# Deploy TAEF, .NETCoreApp, and Microsoft.Windows.Apps.Test. These are the ones that we grabbed off nuget
# so that they match the ones we linked against/packaged in our appx.
#

if (!$NoDeploy)
{
    $nugetPackagesDir = "$env:USERPROFILE\.nuget\packages"
    $binplaceDir = "C:\data\test\bin\"

    putd "$nugetPackagesDir\microsoft.windows.apps.test\1.0.181203002\lib\netcoreapp2.1\*.dll" $binplaceDir
    putd "$nugetPackagesDir\taef.redist.wlk\10.31.180822002\build\Binaries\$platform\*" $binplaceDir
    putd "$nugetPackagesDir\taef.redist.wlk\10.31.180822002\build\Binaries\$platform\CoreClr\*" $binplaceDir
    putd "$nugetPackagesDir\runtime.win-$platform.microsoft.netcore.app\2.1.0\runtimes\win-$platform\lib\netcoreapp2.1\*" $binplaceDir
    putd "$nugetPackagesDir\runtime.win-$platform.microsoft.netcore.app\2.1.0\runtimes\win-$platform\native\*" $binplaceDir
    putd "$nugetPackagesDir\runtime.win-$platform.microsoft.netcore.app\2.1.0\runtimes\win-$platform\lib\netcoreapp2.1\*" "$binplaceDir\.NETCoreApp2.1\"
    putd "$nugetPackagesDir\runtime.win-$platform.microsoft.netcore.app\2.1.0\runtimes\win-$platform\native\*" "$binplaceDir\.NETCoreApp2.1\"
}

# If we're doing a code coverage run, let's instrument the binaries now.
if ($CodeCoverage)
{
    Invoke-Expression "$PSScriptRoot\tools\CodeCoverage\InstrumentBinaries.ps1 -BuildPlatform $Platform -BuildConfiguration $Flavor"
}

# Always copy over the test files.

$repoDirectory = Split-Path -Parent $script:MyInvocation.MyCommand.Path
if(!$Release)
{
    $testDllOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\AnyCPU\MUXControls.Test.TAEF"
    $testAppOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\$platform\MUXControlsTestApp.TAEF"
    $ixmpAppOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\$platform\IXMPTestApp.TAEF"
    $testAppWPFXamlIslandsOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\$platform\MUXControlsTestAppWPFPackage"

    $appxPath = Join-Path $testAppOutputDir "AppPackages\MUXControlsTestApp_Test"
    $dependenciesPath = Join-Path $testAppOutputDir "AppPackages\MUXControlsTestApp_Test\Dependencies\$platform"

    $putDOutput += putd (Join-Path $testDllOutputDir "MUXControls.Test.dll") $deviceDir
    $putDOutput += putd "$appxPath\*" $deviceDir
    $putDOutput += putd "$dependenciesPath\MUXControlsTestApp*" $deviceDir
    $putDOutput += putd (Join-Path $ixmpAppOutputDir "AppPackages\IXMPTestApp_Test\IXMPTestApp*") $deviceDir
    $putDOutput += putd (Join-Path $ixmpAppOutputDir "AppPackages\IXMPTestApp_Test\Dependencies\$platform\*") $deviceDir

    $putDOutput += putd (Join-Path $testAppWPFXamlIslandsOutputDir "AppPackages\MUXControlsTestAppWPFPackage_Test\MUXControlsTestAppWPFPackage*") $deviceDir
    # app package dependencies are missing. Following up on email
    #$putDOutput += putd (Join-Path $testAppWPFXamlIslandsOutputDir "AppPackages\MUXControlsTestAppWPFPackage_Test\Dependencies\$platform\*") $deviceDir
}
else
{
    $testDllOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\AnyCPU\MUXControls.Test.TAEF"
    $releaseTestDllOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\AnyCPU\MUXControls.ReleaseTest.TAEF"
    $testAppOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\$platform\NugetPackageTestApp"
    $testAppCxOutputDir = Join-Path $repoDirectory "BuildOutput\$flavor\$platform\NugetPackageTestAppCX"

    $putDOutput += putd (Join-Path $testDllOutputDir "MUXControls.Test.dll") $deviceDir
    $putDOutput += putd (Join-Path $releaseTestDllOutputDir "MUXControls.ReleaseTest.dll") $deviceDir
    $putDOutput += putd (Join-Path $testAppOutputDir "AppPackages\NugetPackageTestApp_Test\NugetPackageTestApp*") $deviceDir
    $putDOutput += putd (Join-Path $testAppOutputDir "AppPackages\NugetPackageTestApp_Test\Dependencies\$platform\*") $deviceDir
    $putDOutput += putd (Join-Path $testAppCxOutputDir "AppPackages\NugetPackageTestAppCX_Test\NugetPackageTestAppCX*") $deviceDir
    $putDOutput += putd (Join-Path $testAppCxOutputDir "AppPackages\NugetPackageTestAppCX_Test\Dependencies\$platform\*") $deviceDir
}

# Make watson keep dumps offline locally. We will check after the tests run to see if there were any dumps.
$regdOutput += regd add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v DisableArchive /t REG_DWORD /d 0
$regdOutput += regd add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v Disabled /t REG_DWORD /d 0
$regdOutput += regd add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v BypassDataThrottling /t REG_DWORD /d 1
$regdOutput += regd add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\Windows Error Reporting" /v ConfigureArchive /t REG_DWORD /d 2
$regdOutput += regd add "HKEY_LOCAL_MACHINE\Software\Microsoft\Windows\Windows Error Reporting\LocalDumps" /v DumpType /t REG_DWORD /d 2
$regdOutput += regd add "HKEY_LOCAL_MACHINE\SOFTWARE\Policies\Microsoft\Windows\DataCollection" /v AllowTelemetry /t REG_DWORD /d 3

# Get a snapshot of the dump folder before we start running tests
$WERArchiveBeforeRun = ((cmdd dir /s /b "%programdata%\microsoft\windows\wer\reportarchive\*.zip" -HideOutput).Output -split "\r\n")

cdd $deviceDir
if (!$NoRun)
{
    if ($CodeCoverage)
    {
        Write-Host "Starting code coverage..."
        execd cmd -AsUser -Arguments "/c %SystemRoot%\system32\WindowsPowerShell\v1.0\powershell.exe -ExecutionPolicy Bypass -File $deviceDir\StartCodeCoverage.ps1 -MagellanInstallPath $deviceDir\Magellan"
    }

    $teCmd = "execd ""$deviceDir\TE.exe"" -DeviceLogPath $LogFile -Arguments """
    $selectQuery = @()

    if (!$appxOnly)
    {
        if(!$Release)
        {
            $teCmd += " MUXControls.Test.dll"
        }
        else
        {
            $teCmd += " MUXControls.ReleaseTest.dll"
        }
    }

    if (!$dllOnly)
    {
        if(!$Release)
        {
            $teCmd += " MUXControlsTestApp.appx /APPX:CertificateFileName=MUXControlsTestApp.cer";
            $teCmd += " IXMPTestApp.appx /APPX:CertificateFileName=IXMPTestApp.cer";
        }
    }

    if ($Name)
    {
        $nameCondition = ($Name | % { "@Name='*$_*'" }) -join " OR "
        $selectQuery += "$nameCondition"
    }

    if ($MinVersion)
    {
        $selectQuery += "@MinVersion='$MinVersion' OR not(@MinVersion=*)"
    }

    if($selectQuery)
    {
        $teCmd += " /select:""""$(($selectQuery | % { "($_)" }) -join " AND ")"""""
    }

    if ($LogVerbose -or $LogSuperVerbose)
    {
        $teCmd += " /logOutput:High /p:LogVerbose";
    }

    if ($LogSuperVerbose)
    {
        $teCmd += " /p:LogSuperVerbose";
    }

    if ($List)
    {
        $teCmd += " /list";
    }

    if ($waitForDebugger)
    {
        $teCmd += " /p:WaitForDebugger";
    }

    if ($waitForAppDebugger)
    {
        $teCmd += " /p:WaitForAppDebugger";
    }
    
    if ($TestTimeout)
    {
        $teCmd += " /testtimeout:$TestTimeout";
    }
    else
    {
        # Don't set a timeout if someone is attaching a debugger. We don't want the test to be killed while 
        # it is in the middle of being debugged.
        if(!$waitForDebugger -and !$waitForAppDebugger)
        {
            $teCmd += " /testtimeout:0:01";
        }
    }
    
    if ($Iterations -ne 1)
    {
        $teCmd += " /testmode:loop /looptest:$Iterations"
    }

    $teCmd += " /screenCaptureOnError""";

    Write-Host
    Write-Host "Running '$teCmd'";
    Write-Host
    Invoke-Expression $teCmd | Set-Variable TEOutput;
    if ($TEOutput.ExitCode -ne 0)
    {
        Write-Host ("ERROR: TE.exe returned error code {0}" -f $TEOutput.ExitCode);
    }
    
    if ($CodeCoverage)
    {
        Write-Host "Stopping code coverage..."
        execd cmd -AsUser -Arguments "/c %SystemRoot%\system32\WindowsPowerShell\v1.0\powershell.exe -ExecutionPolicy Bypass -File $deviceDir\StopCodeCoverage.ps1 -TestSuite Integration -MagellanInstallPath $deviceDir\Magellan" 1> $null
        Write-Host

        if ($TEOutput.ExitCode -eq 0)
        {
            Write-Host "Gathering code coverage data..."

            if (-not (Test-Path "$OutputPath"))
            {
                New-Item "$OutputPath" -ItemType Directory
            }

            Copy-Item "BuildOutput\$Flavor\$platform\Microsoft.UI.Xaml\instr\*.covsym" "$OutputPath\" 1> $null
            getd "C:\ProgramData\Coverage\microsoft.ui.xaml.dll\*.covdata" "$OutputPath\" 1> $null
        
            Write-Host "Creating code coverage report..."

            Invoke-Expression "$PSScriptRoot\tools\CodeCoverage\CreateCodeCoverageReport.ps1 -MagellanInstallPath `"$MagellanInstallPath`" -covSymPath `"$OutputPath`" -covDataPath `"$OutputPath`" -coverageSummaryOutputPath `"$OutputPath\Summary`" -coverageReportOutputPath `"$OutputPath\Report`" -isLocalRun" 1> $null

            if (Test-Path $OutputPath\Report\index.html)
            {
                Write-Host "Code coverage report written to $OutputPath\Report."

                $shouldOpenReport = ""
                while ($shouldOpenReport -inotlike "y" -and $shouldOpenReport -inotlike "n")
                {
                    $shouldOpenReport = Read-Host "Open report? (y/n)"
                }

                if ($shouldOpenReport -ilike "y")
                {
                    Start-Process -FilePath "$OutputPath\Report\index.html"
                }
            }
        }
        else
        {
            Write-Host "Test failures detected. Skipping code coverage data gathering."
        }
    }
}

# If we were doing a code coverage run, let's put back the old test appx now.
if ($CodeCoverage -and (Test-Path "$testAppOutputDir\AppPackages\MUXControlsTestApp_Test\MUXControlsTestApp.appx.orig"))
{
    Remove-Item "$testAppOutputDir\AppPackages\MUXControlsTestApp_Test\MUXControlsTestApp.appx"
    Rename-Item "$testAppOutputDir\AppPackages\MUXControlsTestApp_Test\MUXControlsTestApp.appx.orig" "$testAppOutputDir\AppPackages\MUXControlsTestApp_Test\MUXControlsTestApp.appx"
}

# Check for dumps
$WERArchiveAfterRun = ((cmdd dir /s /b "%programdata%\microsoft\windows\wer\reportarchive\*.zip" -HideOutput).Output -split "\r\n")
$NewCabs = $WERArchiveAfterRun | Where-Object { -not $WERArchiveBeforeRun.Contains($_) }
Write-Verbose "Before cabs: $WERArchiveBeforeRun"
Write-Verbose "After cabs: $WERArchiveAfterRun"
Write-Verbose "New cabs = $NewCabs"
if ($NewCabs)
{
    Write-Host "Error: Dumps found during test run" -foregroundcolor Red
    $LocalDumpPath = "$PSScriptRoot\DumpFiles"
    if (-not (Test-Path $LocalDumpPath)) { New-Item -Force $LocalDumpPath -ItemType Directory }
    foreach ($NewCab in $NewCabs)
    {
        # Make the filename the archive directory name (instead of "report.zip")
        $fileName = Split-Path (Split-Path $NewCab) -Leaf
        $localFilePath = "$LocalDumpPath\$fileName.zip"
        $ignored = Get-Device $NewCab -Destination $localFilePath
        Write-Host "CAB: $localFilePath" -foregroundcolor Red
    }
}