From c27485857146c314bfe687282b379c8bc5ac4aaa Mon Sep 17 00:00:00 2001 From: Dan Siegel Date: Sun, 23 Jun 2024 21:46:35 -0500 Subject: [PATCH 1/3] feat: validate against current version of Uno (cherry picked from commit a14abe1d2c6d5bce3093eb0035ec37d815637531) --- tools/Uno.Sdk.Updater/Program.cs | 4 +- tools/Uno.Sdk.Updater/Services/NuGetClient.cs | 330 ++++++++++++------ .../{ => Utils}/LocalFileSystem.cs | 2 +- .../Utils/PackageValidationRecord.cs | 34 ++ 4 files changed, 267 insertions(+), 103 deletions(-) rename tools/Uno.Sdk.Updater/{ => Utils}/LocalFileSystem.cs (96%) create mode 100644 tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs diff --git a/tools/Uno.Sdk.Updater/Program.cs b/tools/Uno.Sdk.Updater/Program.cs index 93127ad7..b165feda 100644 --- a/tools/Uno.Sdk.Updater/Program.cs +++ b/tools/Uno.Sdk.Updater/Program.cs @@ -6,6 +6,7 @@ using Uno.Sdk.Models; using Uno.Sdk.Services; using Uno.Sdk.Updater; +using Uno.Sdk.Updater.Utils; const string UnoSdkPackageId = "Uno.Sdk.Private"; @@ -288,7 +289,8 @@ static async Task UpdateGroup(ManifestGroup group, NuGetVersion u preview = false; } - var packageId = group.Packages.First(); + var packageId = group.Packages.FirstOrDefault(x => x.Contains("WinUI", StringComparison.InvariantCultureIgnoreCase) && x.Contains("Uno", StringComparison.InvariantCultureIgnoreCase)) ?? + group.Packages.First(); var version = await client.GetVersionAsync(packageId, preview); var newGroup = group with { Version = version }; diff --git a/tools/Uno.Sdk.Updater/Services/NuGetClient.cs b/tools/Uno.Sdk.Updater/Services/NuGetClient.cs index 08469c9b..ba5dde6a 100644 --- a/tools/Uno.Sdk.Updater/Services/NuGetClient.cs +++ b/tools/Uno.Sdk.Updater/Services/NuGetClient.cs @@ -1,109 +1,237 @@ +#nullable enable using System.Net.Http.Json; +using System.Xml; +using System.Xml.Linq; +using System.Xml.XPath; using Uno.Sdk.Models; +using Uno.Sdk.Updater.Utils; namespace Uno.Sdk.Services; internal class NuGetApiClient : IDisposable { - private HttpClient PublicNuGetClient { get; } = new HttpClient - { - BaseAddress = new Uri("https://api.nuget.org") - }; - - private HttpClient PrivateNuGetClient { get; } = new HttpClient - { - BaseAddress = new Uri("https://pkgs.dev.azure.com") - }; - - public NuGetVersion? UnoVersion { get; set; } - - public async Task DownloadPackageAsync(string packageId, string version) - { - var downloadUrl = $"/uno-platform/1dd81cbd-cb35-41de-a570-b0df3571a196/_apis/packaging/feeds/e7ce08df-613a-41a3-8449-d42784dd45ce/nuget/packages/{packageId}/versions/{version}/content"; - using var response = await PrivateNuGetClient.GetAsync(downloadUrl); - - if (!response.IsSuccessStatusCode) - return Stream.Null; - - using var tempStream = await response.Content.ReadAsStreamAsync(); - var memoryStream = new MemoryStream(); - await tempStream.CopyToAsync(memoryStream); - - return memoryStream; - } - - internal record VersionsResponse(string[] Versions); - - public async Task> GetPackageVersions(string packageId) - { - var allVersions = new List(); - var publicVersions = await GetPublicPackageVersions(packageId); - allVersions.AddRange(publicVersions); - - if (!UnoVersion.HasValue || !UnoVersion.Value.IsPreview) - { - var privateVersions = await GetPrivatePackageVersions(packageId); - allVersions.AddRange(privateVersions); - } - - var output = new List(); - foreach (var version in allVersions.Distinct()) - { - if (NuGetVersion.TryParse(version, out var nugetVersion)) - { - output.Add(nugetVersion); - } - } - - return output.OrderByDescending(x => x); - } - - private async Task> GetPrivatePackageVersions(string packageId) - { - try - { - var response = await PrivateNuGetClient.GetFromJsonAsync($"/uno-platform/1dd81cbd-cb35-41de-a570-b0df3571a196/_packaging/e7ce08df-613a-41a3-8449-d42784dd45ce/nuget/v3/flat2/{packageId.ToLowerInvariant()}/index.json"); - return response?.Versions ?? []; - } - catch - { - return []; - } - } - - private async Task> GetPublicPackageVersions(string packageId) - { - try - { - var response = await PublicNuGetClient.GetFromJsonAsync($"/v3-flatcontainer/{packageId.ToLowerInvariant()}/index.json"); - return response?.Versions ?? []; - } - catch - { - return []; - } - } - - public async Task GetVersionAsync(string packageId, bool preview, string? minimumVersionString = null) - { - var versions = await GetPackageVersions(packageId); - versions = versions.Where(x => x.IsPreview == preview); - - if (NuGetVersion.TryParse(minimumVersionString, out var minimumVersion)) - { - versions = versions.Where(x => minimumVersion.Version <= x.Version); - } - - if (!versions.Any()) - { - return string.Empty; - } - - return versions.OrderByDescending(x => x).First().OriginalVersion; - } - - public void Dispose() - { - PublicNuGetClient.Dispose(); - } + private const string UnoWinUIPackageId = "Uno.WinUI"; + private static PackageValidationRecord _validation = new (); + + private HttpClient PublicNuGetClient { get; } = new HttpClient + { + BaseAddress = new Uri("https://api.nuget.org") + }; + + private HttpClient PrivateNuGetClient { get; } = new HttpClient + { + BaseAddress = new Uri("https://pkgs.dev.azure.com") + }; + + public NuGetVersion? UnoVersion { get; set; } + + public async Task DownloadPackageAsync(string packageId, string version) + { + var downloadUrl = $"/uno-platform/1dd81cbd-cb35-41de-a570-b0df3571a196/_apis/packaging/feeds/e7ce08df-613a-41a3-8449-d42784dd45ce/nuget/packages/{packageId}/versions/{version}/content"; + using var response = await PrivateNuGetClient.GetAsync(downloadUrl); + + if (!response.IsSuccessStatusCode) + return Stream.Null; + + using var tempStream = await response.Content.ReadAsStreamAsync(); + var memoryStream = new MemoryStream(); + await tempStream.CopyToAsync(memoryStream); + + return memoryStream; + } + + internal record VersionsResponse(string[] Versions); + + public async Task> GetPackageVersions(string packageId) + { + var allVersions = new List(); + var publicVersions = await GetPublicPackageVersions(packageId); + allVersions.AddRange(publicVersions); + + if (!UnoVersion.HasValue || !UnoVersion.Value.IsPreview) + { + var privateVersions = await GetPrivatePackageVersions(packageId); + allVersions.AddRange(privateVersions); + } + + var output = new List(); + foreach (var version in allVersions.Distinct()) + { + if (NuGetVersion.TryParse(version, out var nugetVersion)) + { + output.Add(nugetVersion); + } + } + + var latestVersions = output + .GroupBy(x => GetGroupVersion(packageId, x)) + .Select(g => g.OrderByDescending(x => x).First()) + .OrderByDescending(x => x) + .ToArray(); + + if (!RequiresValidation(packageId)) + { + return latestVersions; + } + + var validatedOutput = new List(); + foreach(var version in latestVersions) + { + if (await ValidatePackage(packageId, version)) + { + validatedOutput.Add(version); + continue; + } + + break; + } + + return validatedOutput; + } + + private static NuGetVersion GetGroupVersion(string packageId, NuGetVersion packageVersion) + { + if (packageId.StartsWith("Uno", StringComparison.InvariantCultureIgnoreCase)) + { + return NuGetVersion.Parse($"{packageVersion.Version.Major}.{packageVersion.Version.Minor}.0"); + } + + return NuGetVersion.Parse($"{packageVersion.Version.Major}.{packageVersion.Version.Minor}.{packageVersion.Version.Build}"); + } + + private bool RequiresValidation(string packageId) => + UnoVersion is not null && + packageId.Contains("Uno", StringComparison.InvariantCultureIgnoreCase) && + !packageId.Contains("Sdk", StringComparison.InvariantCultureIgnoreCase); + + private async Task ValidatePackage(string packageId, NuGetVersion version) + { + if (_validation.HasBeenChecked(packageId, version)) + { + return _validation.IsValid(packageId, version); + } + + var nuspecUrl = $"/v3-flatcontainer/{packageId.ToLowerInvariant()}/{version.OriginalVersion}/{packageId.ToLowerInvariant()}.nuspec"; + using var response = PublicNuGetClient.GetAsync(nuspecUrl).Result; + if (!response.IsSuccessStatusCode) + { + // This could happen if the package we are checking is not publicly available. + return true; + } + + var packageResponse = await response.Content.ReadAsStringAsync(); + var xDocument = XDocument.Parse(packageResponse); + + // Define the namespace manager + var namespaceManager = new XmlNamespaceManager(new NameTable()); + namespaceManager.AddNamespace("ns", "http://schemas.microsoft.com/packaging/2013/05/nuspec.xsd"); + + // Select dependency groups + var dependencyGroups = xDocument.XPathSelectElements("//ns:dependencies/ns:group", namespaceManager); + + foreach (var group in dependencyGroups) + { + if (_validation.HasBeenChecked(packageId, version)) + continue; + + _validation.AddResult(packageId, version, await IsCompatibleWithUnoWinUI(group)); + } + + _validation.AddResult(packageId, version, true); + return _validation.IsValid(packageId, version); + } + + private async Task IsCompatibleWithUnoWinUI(XElement group) + { + var dependencies = group.Elements().Where(e => e.Name.LocalName == "dependency").ToList(); + var unoWinUIDependency = dependencies.FirstOrDefault(d => d.Attribute("id")?.Value == UnoWinUIPackageId); + + // We don't have a dependency on Uno.WinUI + if (unoWinUIDependency is null) + { + var unoDependencies = dependencies.Where(x => x.Attribute("id")?.Value.Contains("Uno", StringComparison.InvariantCultureIgnoreCase) ?? false); + + // Check for Transitive Dependency + if (unoDependencies.Any()) + { + var transitiveDependencies = unoDependencies.ToDictionary(x => x.Attribute("id")!.Value, x => NuGetVersion.Parse(x.Attribute("version")!.Value)); + foreach((var packageId, var version) in transitiveDependencies) + { + if (_validation.HasBeenChecked(packageId, version)) + { + continue; + } + else if (!await ValidatePackage(packageId, version)) + { + return false; + } + } + } + + return true; + } + + if (!NuGetVersion.TryParse(unoWinUIDependency.Attribute("version")?.Value, out var unoWinUIVersion)) + { + // This shouldn't happen + return false; + } + + if (!_validation.HasBeenChecked(UnoWinUIPackageId, unoWinUIVersion)) + { + _validation.AddResult(UnoWinUIPackageId, unoWinUIVersion, UnoVersion >= unoWinUIVersion); + } + + return _validation.IsValid(UnoWinUIPackageId, unoWinUIVersion); + } + + private async Task> GetPrivatePackageVersions(string packageId) + { + try + { + var response = await PrivateNuGetClient.GetFromJsonAsync($"/uno-platform/1dd81cbd-cb35-41de-a570-b0df3571a196/_packaging/e7ce08df-613a-41a3-8449-d42784dd45ce/nuget/v3/flat2/{packageId.ToLowerInvariant()}/index.json"); + return response?.Versions ?? []; + } + catch + { + return []; + } + } + + private async Task> GetPublicPackageVersions(string packageId) + { + try + { + var response = await PublicNuGetClient.GetFromJsonAsync($"/v3-flatcontainer/{packageId.ToLowerInvariant()}/index.json"); + return response?.Versions ?? []; + } + catch + { + return []; + } + } + + public async Task GetVersionAsync(string packageId, bool preview, string? minimumVersionString = null) + { + var versions = await GetPackageVersions(packageId); + versions = versions.Where(x => x.IsPreview == preview); + + // https://api.nuget.org/v3-flatcontainer/uno.extensions.hosting.winui/4.2.0-dev.137/uno.extensions.hosting.winui.nuspec + if (NuGetVersion.TryParse(minimumVersionString, out var minimumVersion)) + { + versions = versions.Where(x => minimumVersion.Version <= x.Version); + } + + if (!versions.Any()) + { + return string.Empty; + } + + return versions.OrderByDescending(x => x).First().OriginalVersion; + } + + public void Dispose() + { + PublicNuGetClient.Dispose(); + } } diff --git a/tools/Uno.Sdk.Updater/LocalFileSystem.cs b/tools/Uno.Sdk.Updater/Utils/LocalFileSystem.cs similarity index 96% rename from tools/Uno.Sdk.Updater/LocalFileSystem.cs rename to tools/Uno.Sdk.Updater/Utils/LocalFileSystem.cs index e3d89de7..4504c936 100644 --- a/tools/Uno.Sdk.Updater/LocalFileSystem.cs +++ b/tools/Uno.Sdk.Updater/Utils/LocalFileSystem.cs @@ -1,6 +1,6 @@ using Uno.Sdk.Models; -namespace Uno.Sdk.Updater; +namespace Uno.Sdk.Updater.Utils; internal static class UpdaterBuildContext { diff --git a/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs b/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs new file mode 100644 index 00000000..a4350aa1 --- /dev/null +++ b/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs @@ -0,0 +1,34 @@ +#nullable enable +using Uno.Sdk.Models; + +namespace Uno.Sdk.Updater.Utils; + +internal sealed class PackageValidationRecord +{ + private readonly Dictionary> _validated = []; + + internal bool HasBeenChecked(string packageId, NuGetVersion version) => + GetResult(packageId, version) is not null; + + internal bool IsValid(string packageId, NuGetVersion version) => + GetResult(packageId, version)?.IsValid ?? throw new InvalidOperationException($"Package {packageId} - {version}, has not been checked"); + + internal void AddResult(string packageId, NuGetVersion version, bool validated) + { + if (HasBeenChecked(packageId, version)) + { + return; + } + else if (!_validated.ContainsKey(packageId)) + { + _validated[packageId] = []; + } + + _validated[packageId].Add(new VersionValidationResult(version, validated)); + } + + private VersionValidationResult? GetResult(string packageId, NuGetVersion version) => + _validated.TryGetValue(packageId, out List? value) ? value.FirstOrDefault(x => x.Version == version) : null; + + private record VersionValidationResult(NuGetVersion Version, bool IsValid); +} From ca1f83d6f48829b182c9557e5d466a2049e6e0d5 Mon Sep 17 00:00:00 2001 From: Dan Siegel Date: Wed, 26 Jun 2024 09:49:27 -0500 Subject: [PATCH 2/3] chore: add caching (cherry picked from commit 52482e406dae006ff7df0ecdc6961d8fa6a8b049) --- tools/Uno.Sdk.Updater/Program.cs | 3 +++ tools/Uno.Sdk.Updater/Services/NuGetClient.cs | 11 ++++++++++- .../Uno.Sdk.Updater/Utils/PackageValidationRecord.cs | 5 +++++ 3 files changed, 18 insertions(+), 1 deletion(-) diff --git a/tools/Uno.Sdk.Updater/Program.cs b/tools/Uno.Sdk.Updater/Program.cs index b165feda..364e7d89 100644 --- a/tools/Uno.Sdk.Updater/Program.cs +++ b/tools/Uno.Sdk.Updater/Program.cs @@ -293,6 +293,7 @@ static async Task UpdateGroup(ManifestGroup group, NuGetVersion u group.Packages.First(); var version = await client.GetVersionAsync(packageId, preview); + version = !string.IsNullOrEmpty(group.Version) && NuGetVersion.Parse(version) < NuGetVersion.Parse(group.Version) ? group.Version : version; var newGroup = group with { Version = version }; if (group.Version != newGroup.Version) @@ -316,6 +317,8 @@ static async Task UpdateGroup(ManifestGroup group, NuGetVersion u { Console.WriteLine($"Updated Version Override for '{group.Group}' - '{key}' to '{version}'."); } + + version = NuGetVersion.Parse(version) < versionOverride ? versionOverrideString : version; updatedOverrides.Add(key, version); } diff --git a/tools/Uno.Sdk.Updater/Services/NuGetClient.cs b/tools/Uno.Sdk.Updater/Services/NuGetClient.cs index ba5dde6a..f7580766 100644 --- a/tools/Uno.Sdk.Updater/Services/NuGetClient.cs +++ b/tools/Uno.Sdk.Updater/Services/NuGetClient.cs @@ -12,6 +12,7 @@ internal class NuGetApiClient : IDisposable { private const string UnoWinUIPackageId = "Uno.WinUI"; private static PackageValidationRecord _validation = new (); + private static Dictionary> _cachedVersions = []; private HttpClient PublicNuGetClient { get; } = new HttpClient { @@ -44,6 +45,11 @@ internal record VersionsResponse(string[] Versions); public async Task> GetPackageVersions(string packageId) { + if (_cachedVersions.TryGetValue(packageId, out var cachedVersions)) + { + return cachedVersions; + } + var allVersions = new List(); var publicVersions = await GetPublicPackageVersions(packageId); allVersions.AddRange(publicVersions); @@ -71,10 +77,12 @@ public async Task> GetPackageVersions(string packageId if (!RequiresValidation(packageId)) { + _cachedVersions[packageId] = latestVersions; return latestVersions; } var validatedOutput = new List(); + Console.WriteLine($"Validating available versions for {packageId}..."); foreach(var version in latestVersions) { if (await ValidatePackage(packageId, version)) @@ -86,6 +94,7 @@ public async Task> GetPackageVersions(string packageId break; } + _cachedVersions[packageId] = validatedOutput; return validatedOutput; } @@ -217,7 +226,7 @@ public async Task GetVersionAsync(string packageId, bool preview, string versions = versions.Where(x => x.IsPreview == preview); // https://api.nuget.org/v3-flatcontainer/uno.extensions.hosting.winui/4.2.0-dev.137/uno.extensions.hosting.winui.nuspec - if (NuGetVersion.TryParse(minimumVersionString, out var minimumVersion)) + if (!string.IsNullOrEmpty(minimumVersionString) && NuGetVersion.TryParse(minimumVersionString, out var minimumVersion)) { versions = versions.Where(x => minimumVersion.Version <= x.Version); } diff --git a/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs b/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs index a4350aa1..18f00a47 100644 --- a/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs +++ b/tools/Uno.Sdk.Updater/Utils/PackageValidationRecord.cs @@ -24,6 +24,11 @@ internal void AddResult(string packageId, NuGetVersion version, bool validated) _validated[packageId] = []; } + if (!validated) + { + Console.WriteLine($"{packageId} {version} is not a valid dependency."); + } + _validated[packageId].Add(new VersionValidationResult(version, validated)); } From 2b3a111821db38087418371f6a88b3dea7c4305d Mon Sep 17 00:00:00 2001 From: Dan Siegel Date: Wed, 10 Jul 2024 07:00:44 -0600 Subject: [PATCH 3/3] chore: filter specifically on Uno.Sdk (cherry picked from commit e60256f1d6cf57ee9a92055ffad87f23a9d63d9f) --- tools/Uno.Sdk.Updater/Services/NuGetClient.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/Uno.Sdk.Updater/Services/NuGetClient.cs b/tools/Uno.Sdk.Updater/Services/NuGetClient.cs index f7580766..f1ee0a19 100644 --- a/tools/Uno.Sdk.Updater/Services/NuGetClient.cs +++ b/tools/Uno.Sdk.Updater/Services/NuGetClient.cs @@ -111,7 +111,7 @@ private static NuGetVersion GetGroupVersion(string packageId, NuGetVersion packa private bool RequiresValidation(string packageId) => UnoVersion is not null && packageId.Contains("Uno", StringComparison.InvariantCultureIgnoreCase) && - !packageId.Contains("Sdk", StringComparison.InvariantCultureIgnoreCase); + !packageId.StartsWith("Uno.Sdk", StringComparison.InvariantCultureIgnoreCase); private async Task ValidatePackage(string packageId, NuGetVersion version) {