feat: Improve reliability of hot reload test engine

src/TestApp/Directory.Build.props

@@ -5,4 +5,16 @@
 		<DefineConstants>$(DefineConstants);IS_UNO_RUNTIMETEST_PROJECT</DefineConstants>
 	</PropertyGroup>
 
+	<ItemGroup>
+		<AssemblyAttribute Include="Uno.UI.RuntimeTests.Engine.RuntimeTestsSourceProjectAttribute">
+			<_Parameter1>$(MSBuildProjectFullPath)</_Parameter1>
+		</AssemblyAttribute>
+
+		<AssemblyAttribute Include="Uno.UI.RuntimeTests.Engine.RuntimeTestDevServerAttribute" Condition="'$(PkgUno_UI_DevServer)' != ''">
+			<_Parameter1>$([System.IO.Path]::GetFileName(`$(PkgUno_UI_DevServer)`)) </_Parameter1>
+		</AssemblyAttribute>
+		<AssemblyAttribute Include="Uno.UI.RuntimeTests.Engine.RuntimeTestDevServerAttribute" Condition="'$(PkgUno_WinUI_DevServer)' != ''">
+			<_Parameter1>$([System.IO.Path]::GetFileName(`$(PkgUno_WinUI_DevServer)`)) </_Parameter1>
+		</AssemblyAttribute>
+	</ItemGroup>
 </Project>

src/TestApp/shared/App.xaml.cs

@@ -166,7 +166,11 @@ private static void InitializeLogging()
 				builder.AddFilter("Microsoft", LogLevel.Warning);
 
 				builder.AddFilter("Uno.UI.RuntimeTests.HotReload", LogLevel.Debug);
+
+				// Hot reload testing
 				builder.AddFilter("Uno.UI.RemoteControl", LogLevel.Debug);
+				builder.AddFilter("Uno.UI.RuntimeTests.Internal.Helpers", LogLevel.Debug); // DevServer and SecondaryApp
+				builder.AddFilter("Uno.UI.RuntimeTests.HotReloadHelper", LogLevel.Trace); // Helper used by tests in secondary app instances (non debuggable)
 
 				// Generic Xaml events
 				// builder.AddFilter("Windows.UI.Xaml", LogLevel.Debug );

src/Uno.UI.RuntimeTests.Engine.Library/Engine/ExternalRunner/RuntimeTestDevServerAttribute.cs

@@ -0,0 +1,30 @@
+#if !UNO_RUNTIMETESTS_DISABLE_EMBEDDEDRUNNER
+#nullable enable
+
+#if !IS_UNO_RUNTIMETEST_PROJECT
+#pragma warning disable
+#endif
+
+using System;
+using System.Linq;
+
+namespace Uno.UI.RuntimeTests.Engine;
+
+[AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)]
+public sealed class RuntimeTestDevServerAttribute : Attribute
+{
+	// Note: We prefer to capture it at compilation time instead of using reflection,
+	//		 so if dev-server assembly has been overriden with invalid version (e.g. for debug purposes),
+	//		 we are still able to get the right one.
+
+	/// <summary>
+	/// The version of the DevServer package used to compile the test engine.
+	/// </summary>
+	public string Version { get; }
+
+	public RuntimeTestDevServerAttribute(string version)
+	{
+		Version = version;
+	}
+}
+#endif

src/Uno.UI.RuntimeTests.Engine.Library/Engine/ExternalRunner/RuntimeTestsSourceProjectAttribute.cs

@@ -14,11 +14,15 @@ namespace Uno.UI.RuntimeTests.Engine;
 [AttributeUsage(AttributeTargets.Assembly, AllowMultiple = false)]
 public sealed class RuntimeTestsSourceProjectAttribute : Attribute
 {
+	// Note: This is somehow a duplicate of the Uno.UI.RemoteControl.ProjectConfigurationAttribute but it allows us to have the attribute even in release ==>> TODO: which is needed anyway ???
+	//		 and also to be defined on the assembly that is compiling the runtime test engine (instead on the app head only) which allows us to use it in HotReloadTestHelper without having any dependency on the app type.
+
 	public string ProjectFullPath { get; }
 
 	public RuntimeTestsSourceProjectAttribute(string projectFullPath)
 	{
 		ProjectFullPath = projectFullPath;
 	}
 }
+
 #endif

src/Uno.UI.RuntimeTests.Engine.Library/Engine/ExternalRunner/_Private/DevServer.cs

@@ -10,14 +10,19 @@
 #endif
 
 using System;
+using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Net;
 using System.Net.Sockets;
+using System.Reflection;
+using System.Runtime.CompilerServices;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using HarfBuzzSharp;
+using Microsoft.Extensions.Logging;
 using Uno.Extensions;
 using Uno.Logging;
 using Uno.UI.RuntimeTests.Engine;
@@ -33,6 +38,8 @@ namespace Uno.UI.RuntimeTests.Internal.Helpers;
 /// </remarks>
 internal sealed partial class DevServer : IAsyncDisposable
 {
+	private static readonly ILogger _log = typeof(DevServer).Log();
+	private static int _instance;
 	private static string? _devServerPath;
 
 	/// <summary>
@@ -78,45 +85,72 @@ private static async Task<string> PullDevServer(CancellationToken ct)
 
 		try
 		{
-			var rawVersion = await ProcessHelper.ExecuteAsync(
-				ct,
-				"dotnet",
-				new() { "--version" },
-				dir,
-				"[GET_DOTNET_VERSION]");
-			var dotnetVersion = GetDotnetVersion(rawVersion);
-
-			var csProj = @$"<Project Sdk=""Microsoft.NET.Sdk"">
+			using (var log = _log.Scope("GET_DOTNET_VERSION"))
+			{
+				var rawVersion = await ProcessHelper.ExecuteAsync(
+					ct,
+					"dotnet",
+					new() { "--version" },
+					Environment.CurrentDirectory, // Needed to get the version used by the current app (i.e. including global.json)
+					log);
+				var dotnetVersion = GetDotnetVersion(rawVersion);
+
+				var csProj = @$"<Project Sdk=""Microsoft.NET.Sdk"">
 	<PropertyGroup>
 		<OutputType>Exe</OutputType>
 		<TargetFramework>net{dotnetVersion.Major}.{dotnetVersion.Minor}</TargetFramework>
 	</PropertyGroup>
-</Project>"; 
-			await File.WriteAllTextAsync(Path.Combine(dir, "PullDevServer.csproj"), csProj, ct);
-
-			await ProcessHelper.ExecuteAsync(
-				ct,
-				"dotnet",
-				new() { "add", "package", "Uno.WinUI.DevServer", "--prerelease" },
-				dir,
-				"[PULL_DEV_SERVER]");
-
-			var data = await ProcessHelper.ExecuteAsync(
-				ct,
-				"dotnet",
-				new() { "build", "/t:GetRemoteControlHostPath" },
-				dir,
-				"[GET_DEV_SERVER_PATH]");
-
-			if (GetConfigurationValue(data, "RemoteControlHostPath") is { Length: > 0 } path)
-			{
-				return path;
+</Project>";
+				await File.WriteAllTextAsync(Path.Combine(dir, "PullDevServer.csproj"), csProj, ct);
 			}
-			else
+
+			using (var log = _log.Scope("PULL_DEV_SERVER"))
 			{
-				throw new InvalidOperationException("Failed to get remote control host path");
+				var args = new List<string> { "add", "package" };
+#if HAS_UNO_WINUI || WINDOWS_WINUI
+				args.Add("Uno.WinUI.DevServer");
+#else
+				args.Add("Uno.UI.DevServer");
+#endif
+				// If the assembly is not a debug version it should have a valid version
+				// Note: This is the version of the RemoteControl assembly, not the RemoteControl.Host, but they should be in sync (both are part of the DevServer package)
+				if (Type.GetType("Uno.UI.RemoteControl.RemoteControlClient, Uno.UI.RemoteControl", throwOnError: false)
+					?.Assembly
+					.GetCustomAttribute<AssemblyInformationalVersionAttribute>()
+					?.InformationalVersion is { Length: > 0 } runtimeVersion
+					&& Regex.Match(runtimeVersion, @"^(?<version>\d+\.\d+\.\d+(-\w+\.\d+))+") is {Success: true} match)
+				{
+					args.Add("--version");
+					args.Add(match.Groups["version"].Value);
+				}
+				// Otherwise we use the version used to compile the test engine
+				else if (typeof(DevServer).Assembly.GetCustomAttribute<RuntimeTestDevServerAttribute>()?.Version is { Length: > 0 } version)
+				{
+					args.Add("--version");
+					args.Add(version);
+				}
+				// As a last chance, we just use the latest version
+				else
+				{
+					args.Add("--prerelease"); // latest version
+				}
+
+				await ProcessHelper.ExecuteAsync(ct, "dotnet", args, dir, log);
 			}
 
+			using (var log = _log.Scope("GET_DEV_SERVER_PATH"))
+			{
+				var data = await ProcessHelper.ExecuteAsync(
+					ct,
+					"dotnet",
+					new() { "build", "/t:GetRemoteControlHostPath" },
+					dir,
+					log);
+
+				return GetConfigurationValue(data, "RemoteControlHostPath") is { Length: > 0 } path 
+					? path 
+					: throw new InvalidOperationException("Failed to get remote control host path");
+			}
 		}
 		finally
 		{
@@ -135,7 +169,7 @@ private static DevServer StartCore(string hostBinPath, int port)
 	{
 		if (!File.Exists(hostBinPath))
 		{
-			typeof(DevServer).Log().Error($"DevServer {hostBinPath} does not exist");
+			_log.Error($"DevServer {hostBinPath} does not exist");
 			throw new InvalidOperationException($"Unable to find {hostBinPath}");
 		}
 
@@ -150,7 +184,7 @@ private static DevServer StartCore(string hostBinPath, int port)
 
 		var process = new System.Diagnostics.Process { StartInfo = pi };
 
-		process.StartAndLog("DEV_SERVER");
+		process.StartAndLog(_log.Scope($"DEV_SERVER_{Interlocked.Increment(ref _instance):D2}"));
 
 		return new DevServer(process, port);
 	}
@@ -196,4 +230,5 @@ public async ValueTask DisposeAsync()
 		await _process.WaitForExitAsync(CancellationToken.None);
 	}
 }
+
 #endif

src/Uno.UI.RuntimeTests.Engine.Library/Engine/ExternalRunner/_Private/LogScope.cs

@@ -0,0 +1,45 @@
+#if !UNO_RUNTIMETESTS_DISABLE_UI
+using System;
+using System.Linq;
+using Microsoft.Extensions.Logging;
+
+namespace Uno.UI.RuntimeTests.Internal.Helpers;
+
+internal static class LoggerExtensions
+{
+	public static LogScope Scope(this ILogger log, string scopeName)
+#if false
+		=> new(log, log.BeginScope(scopeName));
+#else
+		=> new (Uno.Extensions.LogExtensionPoint.AmbientLoggerFactory.CreateLogger(typeof(DevServer).FullName + "#" + scopeName));
+#endif
+}
+
+internal readonly struct LogScope : IDisposable, ILogger
+{
+	private readonly ILogger _logger;
+	private readonly IDisposable? _scope;
+
+	public LogScope(ILogger logger, IDisposable? scope = null)
+	{
+		_logger = logger;
+		_scope = scope;
+	}
+
+	/// <inheritdoc />
+	public void Log<TState>(LogLevel logLevel, EventId eventId, TState state, Exception? exception, Func<TState, Exception?, string> formatter)
+		=> _logger.Log(logLevel, eventId, state, exception, formatter);
+
+	/// <inheritdoc />
+	public bool IsEnabled(LogLevel logLevel)
+		=> _logger.IsEnabled(logLevel);
+
+	/// <inheritdoc />
+	public IDisposable BeginScope<TState>(TState state)
+		=> _logger.BeginScope(state);
+
+	/// <inheritdoc />
+	public void Dispose()
+		=> _scope?.Dispose();
+}
+#endif

src/Uno.UI.RuntimeTests.Engine.Library/Engine/ExternalRunner/_Private/ProcessHelper.cs

@@ -7,6 +7,7 @@
 using System.Collections.Generic;
 using System.Threading;
 using System.Threading.Tasks;
+using Microsoft.Extensions.Logging;
 using Uno.Extensions;
 using Uno.Logging;
 
@@ -26,14 +27,14 @@ public static async Task<string> ExecuteAsync(
 		string executable,
 		List<string> parameters,
 		string workingDirectory,
-		string logPrefix,
+		ILogger log,
 		Dictionary<string, string>? environmentVariables = null)
 	{
 		var process = SetupProcess(executable, parameters, workingDirectory, environmentVariables);
 		var output = new StringBuilder();
 		var error = new StringBuilder();
 
-		typeof(ProcessHelper).Log().Debug(logPrefix + " waiting for process exit");
+		log.Debug("Waiting for process exit");
 
 		// hookup the event handlers to capture the data that is received
 		process.OutputDataReceived += (sender, args) => output.Append(args.Data);
@@ -55,46 +56,46 @@ public static async Task<string> ExecuteAsync(
 
 		await process.WaitForExitWithCancellationAsync(ct);
 
-		process.EnsureSuccess(logPrefix, error);
+		process.EnsureSuccess(log, error);
 
 		return output.ToString();
 	}
 
 	public static async Task ExecuteAndLogAsync(
 		this Process process,
-		string logPrefix,
+		ILogger log,
 		CancellationToken ct)
 	{
-		process.StartAndLog(logPrefix);
+		process.StartAndLog(log);
 
-		typeof(ProcessHelper).Log().Debug(logPrefix + " waiting for process exit");
+		typeof(ProcessHelper).Log().Debug(log + " waiting for process exit");
 
 		await process.WaitForExitWithCancellationAsync(ct);
 
-		process.EnsureSuccess(logPrefix);
+		process.EnsureSuccess(log);
 	}
 
 	public static Process StartAndLog(
 		string executable,
 		List<string> parameters,
 		string workingDirectory,
-		string logPrefix,
+		ILogger log,
 		Dictionary<string, string>? environmentVariables = null)
-		=> SetupProcess(executable, parameters, workingDirectory, environmentVariables).StartAndLog(logPrefix);
+		=> SetupProcess(executable, parameters, workingDirectory, environmentVariables).StartAndLog(log);
 
 	public static Process StartAndLog(
 		this Process process,
-		string logPrefix)
+		ILogger log)
 	{
 		process.StartInfo.RedirectStandardOutput = true;
 		process.StartInfo.RedirectStandardError = true;
 
 		// hookup the event handlers to capture the data that is received
-		process.OutputDataReceived += (sender, args) => typeof(ProcessHelper).Log().Debug(logPrefix + ": " + args.Data ?? "<Empty>");
-		process.ErrorDataReceived += (sender, args) => typeof(ProcessHelper).Log().Error(logPrefix + ": " + args.Data ?? "<Empty>");
+		process.OutputDataReceived += (sender, args) => log.Debug(args.Data ?? "<Empty>");
+		process.ErrorDataReceived += (sender, args) => log.Error(args.Data ?? "<Empty>");
 
 		var pi = process.StartInfo;
-		typeof(ProcessHelper).Log().Debug($"Started process (wd:{pi.WorkingDirectory}): {pi.FileName} {string.Join(" ", pi.ArgumentList)})");
+		log.Debug($"Started process (wd:{pi.WorkingDirectory}): {pi.FileName} {string.Join(" ", pi.ArgumentList)})");
 
 		process.Start();
 
@@ -148,22 +149,22 @@ public static async Task WaitForExitWithCancellationAsync(this Process process,
 		await process.WaitForExitAsync(ct);
 	}
 
-	public static void EnsureSuccess(this Process process, string logPrefix, StringBuilder error)
+	public static void EnsureSuccess(this Process process, ILogger log, StringBuilder error)
 	{
 		if (process.ExitCode != 0)
 		{
 			var processError = new InvalidOperationException(error.ToString());
-			typeof(ProcessHelper).Log().Error(logPrefix + $" Process '{process.StartInfo.FileName}' failed with code {process.ExitCode}", processError);
+			log.Error($"Process '{process.StartInfo.FileName}' failed with code {process.ExitCode}", processError);
 
 			throw new InvalidOperationException($"Process '{process.StartInfo.FileName}' failed with code {process.ExitCode}", processError);
 		}
 	}
 
-	public static void EnsureSuccess(this Process process, string logPrefix)
+	public static void EnsureSuccess(this Process process, ILogger log)
 	{
 		if (process.ExitCode != 0)
 		{
-			typeof(ProcessHelper).Log().Error(logPrefix + $" Process '{process.StartInfo.FileName}' failed with code {process.ExitCode}");
+			log.Error($"Process '{process.StartInfo.FileName}' failed with code {process.ExitCode}");
 
 			throw new InvalidOperationException($"Process '{process.StartInfo.FileName}' failed with code {process.ExitCode}");
 		}