From c2f1e7eabd208f23a1be5a281bcd4b906228edc7 Mon Sep 17 00:00:00 2001 From: Elinor Fung Date: Mon, 18 Sep 2023 09:35:56 -0700 Subject: [PATCH 01/12] Check for .deps.json when enumerating framework paths (#92032) This adds a check for the framework's .deps.json instead of just the existence of the directory. To avoid extra file checks in the normal/happy path (where all framework version folder are valid), when resolving, it only does the check after resolving the best version match. And if that version directory isn't valid, it tries resolving again without it. Backport of #90035 --- .../FrameworkResolution/MultipleHives.cs | 16 +++- .../MultilevelSharedFxLookup.cs | 9 ++- .../HostActivation.Tests/NativeHostApis.cs | 20 +++-- .../tests/TestUtils/DotNetBuilder.cs | 40 ++++++--- src/native/corehost/fxr/framework_info.cpp | 81 ++++++++++--------- src/native/corehost/fxr/framework_info.h | 2 +- src/native/corehost/fxr/fx_resolver.cpp | 52 +++++++----- .../corehost/fxr/fx_resolver.messages.cpp | 4 +- src/native/corehost/fxr/hostfxr.cpp | 2 +- 9 files changed, 144 insertions(+), 82 deletions(-) diff --git a/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs b/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs index 0a12cd2640ce9..5a91c88e3a833 100644 --- a/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs +++ b/src/installer/tests/HostActivation.Tests/FrameworkResolution/MultipleHives.cs @@ -4,6 +4,7 @@ using Microsoft.DotNet.Cli.Build; using Microsoft.DotNet.Cli.Build.Framework; using System; +using System.IO; using System.Runtime.InteropServices; using Xunit; @@ -27,7 +28,8 @@ public void FrameworkHiveSelection_GlobalHiveWithBetterMatch() RunTest( runtimeConfig => runtimeConfig .WithFramework(MicrosoftNETCoreApp, "5.0.0")) - .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, "5.1.2"); + .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, "5.1.2") + .And.HaveStdErrContaining($"Ignoring FX version [5.0.0] without .deps.json"); } [Fact] @@ -37,7 +39,8 @@ public void FrameworkHiveSelection_MainHiveWithBetterMatch() RunTest( runtimeConfig => runtimeConfig .WithFramework(MicrosoftNETCoreApp, "6.0.0")) - .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, "6.1.2"); + .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, "6.1.2") + .And.HaveStdErrContaining($"Ignoring FX version [6.0.0] without .deps.json"); } [Fact] @@ -51,7 +54,8 @@ public void FrameworkHiveSelection_CurrentDirectoryIsIgnored() .WithRuntimeConfigCustomizer(runtimeConfig => runtimeConfig .WithFramework(MicrosoftNETCoreApp, "5.0.0")) .WithWorkingDirectory(SharedState.DotNetCurrentHive.BinPath)) - .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, "5.2.0"); + .ShouldHaveResolvedFramework(MicrosoftNETCoreApp, "5.2.0") + .And.HaveStdErrContaining($"Ignoring FX version [5.0.0] without .deps.json"); } private CommandResult RunTest(Func runtimeConfig) @@ -89,6 +93,12 @@ public SharedTestState() .AddMicrosoftNETCoreAppFrameworkMockHostPolicy("6.1.2") .Build(); + // Empty Microsoft.NETCore.App directory - should not be recognized as a valid framework + // Version is the best match for some test cases, but they should be ignored + string netCoreAppDir = Path.Combine(DotNetMainHive.BinPath, "shared", Constants.MicrosoftNETCoreApp); + Directory.CreateDirectory(Path.Combine(netCoreAppDir, "5.0.0")); + Directory.CreateDirectory(Path.Combine(netCoreAppDir, "6.0.0")); + DotNetGlobalHive = DotNet("GlobalHive") .AddMicrosoftNETCoreAppFrameworkMockHostPolicy("5.1.2") .AddMicrosoftNETCoreAppFrameworkMockHostPolicy("6.2.0") diff --git a/src/installer/tests/HostActivation.Tests/MultilevelSharedFxLookup.cs b/src/installer/tests/HostActivation.Tests/MultilevelSharedFxLookup.cs index 788963fa33e49..889ba0c7da383 100644 --- a/src/installer/tests/HostActivation.Tests/MultilevelSharedFxLookup.cs +++ b/src/installer/tests/HostActivation.Tests/MultilevelSharedFxLookup.cs @@ -106,6 +106,9 @@ public MultilevelSharedFxLookup() _builtSharedUberFxDir = Path.Combine(_builtDotnet, "shared", "Microsoft.UberFramework", _sharedFxVersion); SharedFramework.CreateUberFrameworkArtifacts(_builtSharedFxDir, _builtSharedUberFxDir, SystemCollectionsImmutableAssemblyVersion, SystemCollectionsImmutableFileVersion); + // Empty Microsoft.NETCore.App directory - should not be recognized as a valid framework + Directory.CreateDirectory(Path.Combine(_exeSharedFxBaseDir, "9999.9.9")); + // Trace messages used to identify from which folder the framework was picked _hostPolicyDllName = Path.GetFileName(fixture.TestProject.HostPolicyDll); _exeSelectedMessage = $"The expected {_hostPolicyDllName} directory is [{_exeSharedFxBaseDir}"; @@ -237,7 +240,8 @@ public void SharedMultilevelFxLookup_Must_Verify_Folders_in_the_Correct_Order() .CaptureStdErr() .Execute() .Should().Pass() - .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.0"); + .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.0") + .And.HaveStdErrContaining("Ignoring FX version [9999.9.9] without .deps.json"); } [Fact] @@ -341,7 +345,8 @@ public void SharedMultilevelFxLookup_Must_Not_Roll_Forward_If_Framework_Version_ .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.0-dummy2") .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.2") .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.3") - .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.0-dummy3"); + .And.HaveStdOutContaining("Microsoft.NETCore.App 9999.0.0-dummy3") + .And.HaveStdErrContaining("Ignoring FX version [9999.9.9] without .deps.json"); } } } diff --git a/src/installer/tests/HostActivation.Tests/NativeHostApis.cs b/src/installer/tests/HostActivation.Tests/NativeHostApis.cs index d9feb540d70bc..659f8374619d9 100644 --- a/src/installer/tests/HostActivation.Tests/NativeHostApis.cs +++ b/src/installer/tests/HostActivation.Tests/NativeHostApis.cs @@ -97,7 +97,7 @@ public SdkResolutionFixture(SharedTestState state) Directory.CreateDirectory(WorkingDir); - // start with an empty global.json, it will be ignored, but prevent one lying on disk + // start with an empty global.json, it will be ignored, but prevent one lying on disk // on a given machine from impacting the test. File.WriteAllText(GlobalJson, "{}"); @@ -117,12 +117,22 @@ public SdkResolutionFixture(SharedTestState state) foreach ((string fwName, string[] fwVersions) in ProgramFilesGlobalFrameworks) { foreach (string fwVersion in fwVersions) - Directory.CreateDirectory(Path.Combine(ProgramFilesGlobalFrameworksDir, fwName, fwVersion)); + AddFrameworkDirectory(ProgramFilesGlobalFrameworksDir, fwName, fwVersion); } foreach ((string fwName, string[] fwVersions) in LocalFrameworks) { foreach (string fwVersion in fwVersions) - Directory.CreateDirectory(Path.Combine(LocalFrameworksDir, fwName, fwVersion)); + AddFrameworkDirectory(LocalFrameworksDir, fwName, fwVersion); + + // Empty framework directory - this should not be recognized as a valid framework directory + Directory.CreateDirectory(Path.Combine(LocalFrameworksDir, fwName, "9.9.9")); + } + + static void AddFrameworkDirectory(string frameworkDir, string name, string version) + { + string versionDir = Path.Combine(frameworkDir, name, version); + Directory.CreateDirectory(versionDir); + File.WriteAllText(Path.Combine(versionDir, $"{name}.deps.json"), string.Empty); } } } @@ -230,7 +240,7 @@ public void Hostfxr_resolve_sdk2_without_global_json_and_disallowing_previews() [Fact] public void Hostfxr_resolve_sdk2_with_global_json_and_disallowing_previews() { - // With global.json specifying a preview, roll forward to preview + // With global.json specifying a preview, roll forward to preview // since flag has no impact if global.json specifies a preview. // Also check that global.json that impacted resolution is reported. @@ -511,7 +521,7 @@ public void Hostfxr_get_dotnet_environment_info_with_multilevel_lookup_only_self public void Hostfxr_get_dotnet_environment_info_global_install_path() { var f = new SdkResolutionFixture(sharedTestState); - + f.Dotnet.Exec(f.AppDll, new[] { "hostfxr_get_dotnet_environment_info" }) .CaptureStdOut() .CaptureStdErr() diff --git a/src/installer/tests/TestUtils/DotNetBuilder.cs b/src/installer/tests/TestUtils/DotNetBuilder.cs index d4cb58340ca6e..d2d125fa581b8 100644 --- a/src/installer/tests/TestUtils/DotNetBuilder.cs +++ b/src/installer/tests/TestUtils/DotNetBuilder.cs @@ -51,8 +51,7 @@ public DotNetBuilder(string basePath, string builtDotnet, string name) public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockHostPolicy(string version) { // ./shared/Microsoft.NETCore.App/ - create a mock of the root framework - string netCoreAppPath = Path.Combine(_path, "shared", "Microsoft.NETCore.App", version); - Directory.CreateDirectory(netCoreAppPath); + string netCoreAppPath = AddFramework(Constants.MicrosoftNETCoreApp, version); // ./shared/Microsoft.NETCore.App//hostpolicy.dll - this is a mock, will not actually load CoreCLR string mockHostPolicyFileName = RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("mockhostpolicy"); @@ -122,8 +121,7 @@ public DotNetBuilder RemoveHostFxr(Version version = null) public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockCoreClr(string version, Action customizer = null) { // ./shared/Microsoft.NETCore.App/ - create a mock of the root framework - string netCoreAppPath = Path.Combine(_path, "shared", "Microsoft.NETCore.App", version); - Directory.CreateDirectory(netCoreAppPath); + string netCoreAppPath = AddFramework(Constants.MicrosoftNETCoreApp, version); string hostPolicyFileName = RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("hostpolicy"); string coreclrFileName = RuntimeInformationExtensions.GetSharedLibraryFileNameForCurrentPlatform("coreclr"); @@ -131,9 +129,9 @@ public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockCoreClr(string version, string currentRid = _repoDirectories.TargetRID; - NetCoreAppBuilder.ForNETCoreApp("Microsoft.NETCore.App", currentRid) + NetCoreAppBuilder.ForNETCoreApp(Constants.MicrosoftNETCoreApp, currentRid) .WithStandardRuntimeFallbacks() - .WithProject("Microsoft.NETCore.App", version, p => p + .WithProject(Constants.MicrosoftNETCoreApp, version, p => p .WithNativeLibraryGroup(null, g => g // ./shared/Microsoft.NETCore.App//coreclr.dll - this is a mock, will not actually run CoreClr .WithAsset((new NetCoreAppBuilder.RuntimeFileBuilder($"runtimes/{currentRid}/native/{coreclrFileName}")) @@ -158,19 +156,18 @@ public DotNetBuilder AddMicrosoftNETCoreAppFrameworkMockCoreClr(string version, /// Framework version /// Customization function for the runtime config /// - /// The added mock framework will only contain a runtime.config.json file. + /// The added mock framework will only contain a deps.json and a runtime.config.json file. /// public DotNetBuilder AddFramework( string name, string version, Action runtimeConfigCustomizer) { - // ./shared// - create a mock of effectively empty non-root framework - string path = Path.Combine(_path, "shared", name, version); - Directory.CreateDirectory(path); + // ./shared// - create a mock of the framework + string path = AddFramework(name, version); // ./shared///.runtimeconfig.json - runtime config which can be customized - RuntimeConfig runtimeConfig = new RuntimeConfig(Path.Combine(path, name + ".runtimeconfig.json")); + RuntimeConfig runtimeConfig = new RuntimeConfig(Path.Combine(path, $"{name}.runtimeconfig.json")); runtimeConfigCustomizer(runtimeConfig); runtimeConfig.Save(); @@ -193,6 +190,27 @@ public DotNetBuilder AddMockSDK( return this; } + /// + /// Add a minimal mock framework with the specified framework name and version + /// + /// Framework name + /// Framework version + /// Framework directory + /// + /// The added mock framework will only contain a deps.json. + /// + private string AddFramework(string name, string version) + { + // ./shared// - create a mock of effectively the framework + string path = Path.Combine(_path, "shared", name, version); + Directory.CreateDirectory(path); + + // ./shared///.deps.json - empty file + File.WriteAllText(Path.Combine(path, $"{name}.deps.json"), string.Empty); + + return path; + } + public DotNetCli Build() { return new DotNetCli(_path); diff --git a/src/native/corehost/fxr/framework_info.cpp b/src/native/corehost/fxr/framework_info.cpp index 0859f6e57c9d7..fd2e8c73e79f3 100644 --- a/src/native/corehost/fxr/framework_info.cpp +++ b/src/native/corehost/fxr/framework_info.cpp @@ -34,7 +34,7 @@ bool compare_by_name_and_version(const framework_info &a, const framework_info & /*static*/ void framework_info::get_all_framework_infos( const pal::string_t& own_dir, - const pal::string_t& fx_name, + const pal::char_t* fx_name, std::vector* framework_infos) { std::vector hive_dir; @@ -42,49 +42,58 @@ bool compare_by_name_and_version(const framework_info &a, const framework_info & int32_t hive_depth = 0; - for (pal::string_t dir : hive_dir) + for (const pal::string_t& dir : hive_dir) { auto fx_shared_dir = dir; append_path(&fx_shared_dir, _X("shared")); - if (pal::directory_exists(fx_shared_dir)) + if (!pal::directory_exists(fx_shared_dir)) + continue; + + std::vector fx_names; + if (fx_name != nullptr) { - std::vector fx_names; - if (fx_name.length()) - { - // Use the provided framework name - fx_names.push_back(fx_name); - } - else - { - // Read all frameworks, including "Microsoft.NETCore.App" - pal::readdir_onlydirectories(fx_shared_dir, &fx_names); - } + // Use the provided framework name + fx_names.push_back(fx_name); + } + else + { + // Read all frameworks, including "Microsoft.NETCore.App" + pal::readdir_onlydirectories(fx_shared_dir, &fx_names); + } - for (pal::string_t fx_name : fx_names) - { - auto fx_dir = fx_shared_dir; - append_path(&fx_dir, fx_name.c_str()); + for (const pal::string_t& fx_name_local : fx_names) + { + auto fx_dir = fx_shared_dir; + append_path(&fx_dir, fx_name_local.c_str()); + + if (!pal::directory_exists(fx_dir)) + continue; + + trace::verbose(_X("Gathering FX locations in [%s]"), fx_dir.c_str()); - if (pal::directory_exists(fx_dir)) + std::vector versions; + pal::readdir_onlydirectories(fx_dir, &versions); + for (const pal::string_t& ver : versions) + { + // Make sure we filter out any non-version folders. + fx_ver_t parsed; + if (!fx_ver_t::parse(ver, &parsed, false)) + continue; + + // Check that the framework's .deps.json exists. + pal::string_t fx_version_dir = fx_dir; + append_path(&fx_version_dir, ver.c_str()); + if (!library_exists_in_dir(fx_version_dir, fx_name_local + _X(".deps.json"), nullptr)) { - trace::verbose(_X("Gathering FX locations in [%s]"), fx_dir.c_str()); - - std::vector versions; - pal::readdir_onlydirectories(fx_dir, &versions); - for (const auto& ver : versions) - { - // Make sure we filter out any non-version folders. - fx_ver_t parsed; - if (fx_ver_t::parse(ver, &parsed, false)) - { - trace::verbose(_X("Found FX version [%s]"), ver.c_str()); - - framework_info info(fx_name, fx_dir, parsed, hive_depth); - framework_infos->push_back(info); - } - } + trace::verbose(_X("Ignoring FX version [%s] without .deps.json"), ver.c_str()); + continue; } + + trace::verbose(_X("Found FX version [%s]"), ver.c_str()); + + framework_info info(fx_name_local, fx_dir, parsed, hive_depth); + framework_infos->push_back(info); } } @@ -97,7 +106,7 @@ bool compare_by_name_and_version(const framework_info &a, const framework_info & /*static*/ bool framework_info::print_all_frameworks(const pal::string_t& own_dir, const pal::string_t& leading_whitespace) { std::vector framework_infos; - get_all_framework_infos(own_dir, _X(""), &framework_infos); + get_all_framework_infos(own_dir, nullptr, &framework_infos); for (framework_info info : framework_infos) { trace::println(_X("%s%s %s [%s]"), leading_whitespace.c_str(), info.name.c_str(), info.version.as_str().c_str(), info.path.c_str()); diff --git a/src/native/corehost/fxr/framework_info.h b/src/native/corehost/fxr/framework_info.h index e4eea09cb14c2..d9111aef06b20 100644 --- a/src/native/corehost/fxr/framework_info.h +++ b/src/native/corehost/fxr/framework_info.h @@ -17,7 +17,7 @@ struct framework_info static void get_all_framework_infos( const pal::string_t& own_dir, - const pal::string_t& fx_name, + const pal::char_t* fx_name, std::vector* framework_infos); static bool print_all_frameworks(const pal::string_t& own_dir, const pal::string_t& leading_whitespace); diff --git a/src/native/corehost/fxr/fx_resolver.cpp b/src/native/corehost/fxr/fx_resolver.cpp index 6091c9a5d4b55..216318b6047f0 100644 --- a/src/native/corehost/fxr/fx_resolver.cpp +++ b/src/native/corehost/fxr/fx_resolver.cpp @@ -174,9 +174,6 @@ namespace if (best_match == fx_ver_t()) { - // This is not strictly necessary, we just need to return version which doesn't exist. - // But it's cleaner to return the desider reference then invalid -1.-1.-1 version. - best_match = fx_ref.get_fx_version_number(); trace::verbose(_X("Framework reference didn't resolve to any available version.")); } else if (trace::is_enabled()) @@ -211,7 +208,8 @@ namespace pal::string_t selected_fx_version; fx_ver_t selected_ver; - for (pal::string_t dir : hive_dir) + pal::string_t deps_file_name = fx_ref.get_fx_name() + _X(".deps.json"); + for (pal::string_t& dir : hive_dir) { auto fx_dir = dir; trace::verbose(_X("Searching FX directory in [%s]"), fx_dir.c_str()); @@ -235,7 +233,7 @@ namespace fx_ref.get_fx_version().c_str()); append_path(&fx_dir, fx_ref.get_fx_version().c_str()); - if (pal::directory_exists(fx_dir)) + if (library_exists_in_dir(fx_dir, deps_file_name, nullptr)) { selected_fx_dir = fx_dir; selected_fx_version = fx_ref.get_fx_version(); @@ -258,27 +256,39 @@ namespace } fx_ver_t resolved_ver = resolve_framework_reference_from_version_list(version_list, fx_ref); - - pal::string_t resolved_ver_str = resolved_ver.as_str(); - append_path(&fx_dir, resolved_ver_str.c_str()); - - if (pal::directory_exists(fx_dir)) + while (resolved_ver != fx_ver_t()) { - if (selected_ver != fx_ver_t()) + pal::string_t resolved_ver_str = resolved_ver.as_str(); + pal::string_t resolved_fx_dir = fx_dir; + append_path(&resolved_fx_dir, resolved_ver_str.c_str()); + + // Check that the framework's .deps.json exists. To minimize the file checks done in the most common + // scenario (.deps.json exists), only check after resolving the version and if the .deps.json doesn't + // exist, attempt resolving again without that version. + if (!library_exists_in_dir(resolved_fx_dir, deps_file_name, nullptr)) { - // Compare the previous hive_dir selection with the current hive_dir to see which one is the better match - std::vector version_list; - version_list.push_back(resolved_ver); - version_list.push_back(selected_ver); + // Remove the version and try resolving again + trace::verbose(_X("Ignoring FX version [%s] without .deps.json"), resolved_ver_str.c_str()); + version_list.erase(std::find(version_list.cbegin(), version_list.cend(), resolved_ver)); resolved_ver = resolve_framework_reference_from_version_list(version_list, fx_ref); } - - if (resolved_ver != selected_ver) + else { - trace::verbose(_X("Changing Selected FX version from [%s] to [%s]"), selected_fx_dir.c_str(), fx_dir.c_str()); - selected_ver = resolved_ver; - selected_fx_dir = fx_dir; - selected_fx_version = resolved_ver_str; + if (selected_ver != fx_ver_t()) + { + // Compare the previous hive_dir selection with the current hive_dir to see which one is the better match + resolved_ver = resolve_framework_reference_from_version_list({ resolved_ver, selected_ver }, fx_ref); + } + + if (resolved_ver != selected_ver) + { + trace::verbose(_X("Changing Selected FX version from [%s] to [%s]"), selected_fx_dir.c_str(), resolved_fx_dir.c_str()); + selected_ver = resolved_ver; + selected_fx_dir = resolved_fx_dir; + selected_fx_version = resolved_ver_str; + } + + break; } } } diff --git a/src/native/corehost/fxr/fx_resolver.messages.cpp b/src/native/corehost/fxr/fx_resolver.messages.cpp index 960a1c42c5c37..91539bc6903ee 100644 --- a/src/native/corehost/fxr/fx_resolver.messages.cpp +++ b/src/native/corehost/fxr/fx_resolver.messages.cpp @@ -100,14 +100,14 @@ void fx_resolver_t::display_missing_framework_error( if (fx_dir.length()) { fx_ver_dirs = fx_dir; - framework_info::get_all_framework_infos(get_directory(fx_dir), fx_name, &framework_infos); + framework_info::get_all_framework_infos(get_directory(fx_dir), fx_name.c_str(), &framework_infos); } else { fx_ver_dirs = dotnet_root; } - framework_info::get_all_framework_infos(dotnet_root, fx_name, &framework_infos); + framework_info::get_all_framework_infos(dotnet_root, fx_name.c_str(), &framework_infos); // Display the error message about missing FX. if (fx_version.length()) diff --git a/src/native/corehost/fxr/hostfxr.cpp b/src/native/corehost/fxr/hostfxr.cpp index 4011dbc439785..98fbb8e00bbd9 100644 --- a/src/native/corehost/fxr/hostfxr.cpp +++ b/src/native/corehost/fxr/hostfxr.cpp @@ -425,7 +425,7 @@ SHARED_API int32_t HOSTFXR_CALLTYPE hostfxr_get_dotnet_environment_info( } std::vector framework_infos; - framework_info::get_all_framework_infos(dotnet_dir, _X(""), &framework_infos); + framework_info::get_all_framework_infos(dotnet_dir, nullptr, &framework_infos); std::vector environment_framework_infos; std::vector framework_versions; From f13337f23e0c3c9bd48c82574e7deedca80b1112 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Thu, 28 Sep 2023 19:19:00 -0600 Subject: [PATCH 02/12] [release/6.0] Update Ubuntu 16.04 amd64 queues to 22.04 (#92794) * [release/6.0] Update Ubuntu 16.04 queues to 22.04 * add svc prefix --- .../coreclr/templates/helix-queues-setup.yml | 6 +++--- .../libraries/helix-queues-setup.yml | 20 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/eng/pipelines/coreclr/templates/helix-queues-setup.yml b/eng/pipelines/coreclr/templates/helix-queues-setup.yml index 1ee222e97d1ef..3232003e2ef05 100644 --- a/eng/pipelines/coreclr/templates/helix-queues-setup.yml +++ b/eng/pipelines/coreclr/templates/helix-queues-setup.yml @@ -66,9 +66,9 @@ jobs: # Linux musl x64 - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - - (Alpine.314.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 + - (Alpine.314.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - (Alpine.314.Amd64)ubuntu.1604.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 + - (Alpine.314.Amd64)ubuntu.2204.amd64.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 # Linux musl arm32 - ${{ if eq(parameters.platform, 'Linux_musl_arm') }}: @@ -97,7 +97,7 @@ jobs: - (Debian.10.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - (Debian.11.Amd64)Ubuntu.1804.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - Ubuntu.1804.Amd64 - - (Fedora.34.Amd64)Ubuntu.1604.amd64@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix + - (Fedora.34.Amd64)Ubuntu.2204.amd64.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - RedHat.7.Amd64 # OSX arm64 diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index c1b40ec3ad7fe..51989907e0a53 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -41,9 +41,9 @@ jobs: # Linux musl x64 - ${{ if eq(parameters.platform, 'Linux_musl_x64') }}: - - (Alpine.314.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 + - (Alpine.314.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.14-helix-amd64 - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: - - (Alpine.313.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-helix-amd64 + - (Alpine.313.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:alpine-3.13-helix-amd64 # Linux musl arm64 - ${{ if and(eq(parameters.platform, 'Linux_musl_arm64'), eq(parameters.jobParameters.isFullMatrix, true)) }}: @@ -56,23 +56,23 @@ jobs: - ${{ if and(eq(parameters.jobParameters.testScope, 'outerloop'), eq(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - RedHat.7.Amd64.Open - SLES.15.Amd64.Open - - (Fedora.34.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - - (Ubuntu.1910.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64 - - (Debian.10.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 + - (Fedora.34.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix + - (Ubuntu.1910.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64 + - (Debian.10.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - ${{ if or(ne(parameters.jobParameters.testScope, 'outerloop'), ne(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: - - (Centos.7.Amd64.Open)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix + - (Centos.7.Amd64.Open)Ubuntu.2204.Amd64.Open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix - RedHat.7.Amd64.Open - Ubuntu.1804.Amd64.Open - SLES.12.Amd64.Open - SLES.15.Amd64.Open - - (Fedora.34.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - - (Ubuntu.1910.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64 + - (Fedora.34.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix + - (Ubuntu.1910.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64 - (Debian.10.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - - (Mariner.1.0.Amd64.Open)ubuntu.1604.amd64.open@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix + - (Mariner.1.0.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix - ${{ if eq(parameters.jobParameters.isFullMatrix, false) }}: - - (Centos.7.Amd64.Open)Ubuntu.1604.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix + - (Centos.7.Amd64.Open)Ubuntu.2204.Amd64.Open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-helix - RedHat.7.Amd64.Open - (Debian.10.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - Ubuntu.1804.Amd64.Open From f95e1784ee6ebcd10d1f78fda7794df34ca6d388 Mon Sep 17 00:00:00 2001 From: Eirik Tsarpalis Date: Sat, 30 Sep 2023 05:16:44 +0300 Subject: [PATCH 03/12] [release/6.0-staging] Fix JsonDocument thread safety. (#92832) * Fix JsonDocument thread safety. (#76716) Co-authored-by: stoub@microsoft.com * Update ServicingVersion. --- .../src/System.Text.Json.csproj | 4 +- .../System/Text/Json/Document/JsonDocument.cs | 19 +-------- .../JsonDocumentTests.cs | 42 +++++++++++++++++-- 3 files changed, 41 insertions(+), 24 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index f81e9dd4fd84c..0b046e5c6c0cf 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -9,8 +9,8 @@ enable true true - false - 8 + true + 9 Provides high-performance and low-allocating types that serialize objects to JavaScript Object Notation (JSON) text and deserialize JSON text to objects, with UTF-8 support built-in. Also provides types to read and write JSON text encoded as UTF-8, and to create an in-memory document object model (DOM), that is read-only, for random access of the JSON elements within a structured view of the data. Commonly Used Types: diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs index 6b075966bb736..7e7bec82f2c40 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Document/JsonDocument.cs @@ -30,8 +30,6 @@ public sealed partial class JsonDocument : IDisposable private PooledByteBufferWriter? _extraPooledByteBufferWriter; private bool _hasExtraPooledByteBufferWriter; - private (int, string?) _lastIndexAndString = (-1, null); - internal bool IsDisposable { get; } /// @@ -278,14 +276,6 @@ private ReadOnlyMemory GetPropertyRawValue(int valueIndex) { CheckNotDisposed(); - (int lastIdx, string? lastString) = _lastIndexAndString; - - if (lastIdx == index) - { - Debug.Assert(lastString != null); - return lastString; - } - DbRow row = _parsedData.Get(index); JsonTokenType tokenType = row.TokenType; @@ -300,6 +290,7 @@ private ReadOnlyMemory GetPropertyRawValue(int valueIndex) ReadOnlySpan data = _utf8Json.Span; ReadOnlySpan segment = data.Slice(row.Location, row.SizeOrLength); + string lastString; if (row.HasComplexChildren) { int backslash = segment.IndexOf(JsonConstants.BackSlash); @@ -311,7 +302,6 @@ private ReadOnlyMemory GetPropertyRawValue(int valueIndex) } Debug.Assert(lastString != null); - _lastIndexAndString = (index, lastString); return lastString; } @@ -321,13 +311,6 @@ internal bool TextEquals(int index, ReadOnlySpan otherText, bool isPropert int matchIndex = isPropertyName ? index - DbRow.Size : index; - (int lastIdx, string? lastString) = _lastIndexAndString; - - if (lastIdx == matchIndex) - { - return otherText.SequenceEqual(lastString.AsSpan()); - } - byte[]? otherUtf8TextArray = null; int length = checked(otherText.Length * JsonConstants.MaxExpansionFactorWhileTranscoding); diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs index 67a203da03b72..36c2a56f746d1 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/JsonDocumentTests.cs @@ -14,6 +14,7 @@ using System.Linq; using System.Runtime.InteropServices; using System.Threading.Tasks; +using System.Threading; namespace System.Text.Json.Tests { @@ -2719,11 +2720,9 @@ public static void ObjectEnumeratorIndependentWalk() Assert.Equal(test, property.Value.GetInt32()); test++; - // Subsequent read of the same JsonProperty doesn't allocate a new string - // (if another property is inspected from the same document that guarantee - // doesn't hold). + // Subsequent read of the same JsonProperty should return an equal string string propertyName2 = property.Name; - Assert.Same(propertyName, propertyName2); + Assert.Equal(propertyName, propertyName2); } test = 0; @@ -3582,6 +3581,41 @@ public static void NameEquals_Empty_Throws() } } + [ConditionalFact(typeof(PlatformDetection), nameof(PlatformDetection.IsThreadingSupported))] + [OuterLoop] // thread-safety / stress test + public static async Task GetString_ConcurrentUse_ThreadSafe() + { + using (JsonDocument doc = JsonDocument.Parse(SR.SimpleObjectJson)) + { + JsonElement first = doc.RootElement.GetProperty("first"); + JsonElement last = doc.RootElement.GetProperty("last"); + + const int Iters = 10_000; + using (var gate = new Barrier(2)) + { + await Task.WhenAll( + Task.Factory.StartNew(() => + { + gate.SignalAndWait(); + for (int i = 0; i < Iters; i++) + { + Assert.Equal("John", first.GetString()); + Assert.True(first.ValueEquals("John")); + } + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default), + Task.Factory.StartNew(() => + { + gate.SignalAndWait(); + for (int i = 0; i < Iters; i++) + { + Assert.Equal("Smith", last.GetString()); + Assert.True(last.ValueEquals("Smith")); + } + }, CancellationToken.None, TaskCreationOptions.LongRunning, TaskScheduler.Default)); + } + } + } + private static void BuildSegmentedReader( out Utf8JsonReader reader, ReadOnlyMemory data, From 61fb33fa77b506208c025607a3eb269c4adfc76d Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 5 Oct 2023 10:06:39 -0600 Subject: [PATCH 04/12] [release/6.0-staging] Update dependencies from dotnet/runtime-assets (#92884) * Update dependencies from https://github.com/dotnet/runtime-assets build 20231002.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Windows.Extensions.TestData From Version 6.0.0-beta.23410.1 -> To Version 6.0.0-beta.23502.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20231002.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Windows.Extensions.TestData From Version 6.0.0-beta.23410.1 -> To Version 6.0.0-beta.23502.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20231002.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Windows.Extensions.TestData From Version 6.0.0-beta.23410.1 -> To Version 6.0.0-beta.23502.2 * Update dependencies from https://github.com/dotnet/runtime-assets build 20231002.2 Microsoft.DotNet.CilStrip.Sources , System.ComponentModel.TypeConverter.TestData , System.Data.Common.TestData , System.Drawing.Common.TestData , System.IO.Compression.TestData , System.IO.Packaging.TestData , System.Net.TestData , System.Private.Runtime.UnicodeData , System.Runtime.Numerics.TestData , System.Runtime.TimeZoneData , System.Security.Cryptography.X509Certificates.TestData , System.Windows.Extensions.TestData From Version 6.0.0-beta.23410.1 -> To Version 6.0.0-beta.23502.2 --------- Co-authored-by: dotnet-maestro[bot] --- NuGet.config | 1 - eng/Version.Details.xml | 48 ++++++++++++++++++++--------------------- eng/Versions.props | 24 ++++++++++----------- 3 files changed, 36 insertions(+), 37 deletions(-) diff --git a/NuGet.config b/NuGet.config index f03ecf8fdc4d9..d5dbd101f043e 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,7 +9,6 @@ - diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 75e43e3f8b51f..915d22daed6df 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -102,49 +102,49 @@ https://github.com/microsoft/vstest 140434f7109d357d0158ade9e5164a4861513965 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 https://github.com/dotnet/llvm-project @@ -246,9 +246,9 @@ https://github.com/dotnet/hotreload-utils dd752be1a2fa9ea5c55fbbf1e28dc65902b3a476 - + https://github.com/dotnet/runtime-assets - b642300c07f64ba35fdf1e2d2c4476107519f8f4 + cee46f91361cc16cb1b0e95f52c87f1f5600f2c7 https://github.com/dotnet/roslyn-analyzers diff --git a/eng/Versions.props b/eng/Versions.props index ce33daa90d4a1..459f13492a60e 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -108,18 +108,18 @@ 4.5.0 6.0.0-rc.1.21415.6 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 - 6.0.0-beta.23410.1 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 + 6.0.0-beta.23502.2 1.0.0-prerelease.21416.5 1.0.0-prerelease.21416.5 From e3df2ed93f3b320e246d72a8a485e8ba3e052597 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Thu, 5 Oct 2023 13:43:08 -0600 Subject: [PATCH 05/12] [release/6.0-staging] Re-add emsdk key to NuGet.config (#93074) --- NuGet.config | 1 + 1 file changed, 1 insertion(+) diff --git a/NuGet.config b/NuGet.config index d5dbd101f043e..f03ecf8fdc4d9 100644 --- a/NuGet.config +++ b/NuGet.config @@ -9,6 +9,7 @@ + From 4f5047f921a47b002b744d8449915b93937d3e04 Mon Sep 17 00:00:00 2001 From: Jeremy Koritzinsky Date: Mon, 9 Oct 2023 12:55:21 -0700 Subject: [PATCH 06/12] [release/6.0] Fix a memory leak in runtime interop stubs when using an array of structs of types that use old-style managed marshalers (#93147) --- src/coreclr/vm/ilmarshalers.h | 26 -------------------------- 1 file changed, 26 deletions(-) diff --git a/src/coreclr/vm/ilmarshalers.h b/src/coreclr/vm/ilmarshalers.h index 8d24d07500472..1eb3079f9c2f3 100644 --- a/src/coreclr/vm/ilmarshalers.h +++ b/src/coreclr/vm/ilmarshalers.h @@ -3134,39 +3134,13 @@ class ILMngdMarshaler : public ILMarshaler void EmitClearNative(ILCodeStream* pslILEmit) override { WRAPPER_NO_CONTRACT; - ILCodeLabel* pNoManagedValueLabel = nullptr; - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pNoManagedValueLabel = pslILEmit->NewCodeLabel(); - pslILEmit->EmitLDARG(StructMarshalStubs::MANAGED_STRUCT_ARGIDX); - pslILEmit->EmitBRFALSE(pNoManagedValueLabel); - } - EmitCallMngdMarshalerMethod(pslILEmit, GetClearNativeMethod()); - - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pslILEmit->EmitLabel(pNoManagedValueLabel); - } } void EmitClearNativeContents(ILCodeStream* pslILEmit) override { WRAPPER_NO_CONTRACT; - ILCodeLabel* pNoManagedValueLabel = nullptr; - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pNoManagedValueLabel = pslILEmit->NewCodeLabel(); - pslILEmit->EmitLDARG(StructMarshalStubs::MANAGED_STRUCT_ARGIDX); - pslILEmit->EmitBRFALSE(pNoManagedValueLabel); - } - EmitCallMngdMarshalerMethod(pslILEmit, GetClearNativeContentsMethod()); - - if (IsFieldMarshal(m_dwMarshalFlags)) - { - pslILEmit->EmitLabel(pNoManagedValueLabel); - } } bool NeedsClearCLR() override From 7318da8d5440e6ca51d005399a81e5f3e5bf817d Mon Sep 17 00:00:00 2001 From: Koundinya Veluri Date: Wed, 11 Oct 2023 15:32:35 -0700 Subject: [PATCH 07/12] Make thread pool thread timeouts configurable (#92988) - Added two config options, one that configures the worker and wait thread timeouts, and another that enables keeping some number of worker threads alive after they are created - This enables services that take periodic traffic to keep some worker threads around for better latency, while allowing extra threads to time out as appropriate for the service --- src/coreclr/inc/clrconfigvalues.h | 2 + src/coreclr/vm/comthreadpool.cpp | 28 ++++++ src/coreclr/vm/eeconfig.cpp | 16 ++++ src/coreclr/vm/eeconfig.h | 6 ++ src/coreclr/vm/win32threadpool.cpp | 85 +++++++++++++++++-- src/coreclr/vm/win32threadpool.h | 7 +- .../PortableThreadPool.WorkerThread.cs | 46 +++++++++- .../System/Threading/PortableThreadPool.cs | 17 +++- 8 files changed, 197 insertions(+), 10 deletions(-) diff --git a/src/coreclr/inc/clrconfigvalues.h b/src/coreclr/inc/clrconfigvalues.h index 7f6698684f9fb..69b0dd758b751 100644 --- a/src/coreclr/inc/clrconfigvalues.h +++ b/src/coreclr/inc/clrconfigvalues.h @@ -577,6 +577,8 @@ RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("Thread #else // !TARGET_ARM64 RETAIL_CONFIG_DWORD_INFO(INTERNAL_ThreadPool_UnfairSemaphoreSpinLimit, W("ThreadPool_UnfairSemaphoreSpinLimit"), 0x46, "Maximum number of spins a thread pool worker thread performs before waiting for work") #endif // TARGET_ARM64 +RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_ThreadPool_ThreadTimeoutMs, W("ThreadPool_ThreadTimeoutMs"), (DWORD)-2, "The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and exiting. Set to -1 to disable the timeout. Applies to worker threads, completion port threads, and wait threads. Also see the ThreadPool_ThreadsToKeepAlive config value for relevant information.", CLRConfig::LookupOptions::ParseIntegerAsBase10) +RETAIL_CONFIG_DWORD_INFO_EX(EXTERNAL_ThreadPool_ThreadsToKeepAlive, W("ThreadPool_ThreadsToKeepAlive"), 0, "The number of worker or completion port threads to keep alive after they are created. Set to -1 to keep all created worker or completion port threads alive. When the ThreadPool_ThreadTimeoutMs config value is also set, for worker and completion port threads the timeout applies to threads in the respective pool that are in excess of the number configured for ThreadPool_ThreadsToKeepAlive.", CLRConfig::LookupOptions::ParseIntegerAsBase10) RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_Disable, W("HillClimbing_Disable"), 0, "Disables hill climbing for thread adjustments in the thread pool"); RETAIL_CONFIG_DWORD_INFO(INTERNAL_HillClimbing_WavePeriod, W("HillClimbing_WavePeriod"), 4, ""); diff --git a/src/coreclr/vm/comthreadpool.cpp b/src/coreclr/vm/comthreadpool.cpp index 578f30e4f3d14..8432da6361157 100644 --- a/src/coreclr/vm/comthreadpool.cpp +++ b/src/coreclr/vm/comthreadpool.cpp @@ -192,6 +192,34 @@ FCIMPL4(INT32, ThreadPoolNative::GetNextConfigUInt32Value, case 19: if (TryGetConfig(CLRConfig::INTERNAL_HillClimbing_SampleIntervalHigh, false, W("System.Threading.ThreadPool.HillClimbing.SampleIntervalHigh"))) { return 20; } FALLTHROUGH; case 20: if (TryGetConfig(CLRConfig::INTERNAL_HillClimbing_GainExponent, false, W("System.Threading.ThreadPool.HillClimbing.GainExponent"))) { return 21; } FALLTHROUGH; + case 21: + { + int threadPoolThreadTimeoutMs = g_pConfig->ThreadPoolThreadTimeoutMs(); + if (threadPoolThreadTimeoutMs >= -1) + { + *configValueRef = (UINT32)threadPoolThreadTimeoutMs; + *isBooleanRef = false; + *appContextConfigNameRef = W("System.Threading.ThreadPool.ThreadTimeoutMs"); + return 22; + } + + FALLTHROUGH; + } + + case 22: + { + int threadPoolThreadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive(); + if (threadPoolThreadsToKeepAlive >= -1) + { + *configValueRef = (UINT32)threadPoolThreadsToKeepAlive; + *isBooleanRef = false; + *appContextConfigNameRef = W("System.Threading.ThreadPool.ThreadsToKeepAlive"); + return 23; + } + + FALLTHROUGH; + } + default: *configValueRef = 0; *isBooleanRef = false; diff --git a/src/coreclr/vm/eeconfig.cpp b/src/coreclr/vm/eeconfig.cpp index 01ddbc873b940..f111cf0833263 100644 --- a/src/coreclr/vm/eeconfig.cpp +++ b/src/coreclr/vm/eeconfig.cpp @@ -237,6 +237,9 @@ HRESULT EEConfig::Init() bDiagnosticSuspend = false; #endif + threadPoolThreadTimeoutMs = -2; // not configured + threadPoolThreadsToKeepAlive = 0; + fDisableDefaultCodeVersioning = false; #if defined(FEATURE_TIERED_COMPILATION) @@ -738,6 +741,19 @@ HRESULT EEConfig::sync() #endif //_DEBUG + threadPoolThreadTimeoutMs = + (int)Configuration::GetKnobDWORDValue( + W("System.Threading.ThreadPool.ThreadTimeoutMs"), + CLRConfig::EXTERNAL_ThreadPool_ThreadTimeoutMs); + threadPoolThreadsToKeepAlive = + (int)Configuration::GetKnobDWORDValue( + W("System.Threading.ThreadPool.ThreadsToKeepAlive"), + CLRConfig::EXTERNAL_ThreadPool_ThreadsToKeepAlive); + if (threadPoolThreadsToKeepAlive < -1) + { + threadPoolThreadsToKeepAlive = 0; + } + m_fInteropValidatePinnedObjects = (CLRConfig::GetConfigValue(CLRConfig::UNSUPPORTED_InteropValidatePinnedObjects) != 0); m_fInteropLogArguments = (CLRConfig::GetConfigValue(CLRConfig::EXTERNAL_InteropLogArguments) != 0); diff --git a/src/coreclr/vm/eeconfig.h b/src/coreclr/vm/eeconfig.h index 8d387ce87e54f..2335752ff92d8 100644 --- a/src/coreclr/vm/eeconfig.h +++ b/src/coreclr/vm/eeconfig.h @@ -483,6 +483,9 @@ class EEConfig #endif + int ThreadPoolThreadTimeoutMs() const { LIMITED_METHOD_CONTRACT; return threadPoolThreadTimeoutMs; } + int ThreadPoolThreadsToKeepAlive() const { LIMITED_METHOD_CONTRACT; return threadPoolThreadsToKeepAlive; } + private: //---------------------------------------------------------------- bool fInited; // have we synced to the registry at least once? @@ -680,6 +683,9 @@ class EEConfig DWORD testThreadAbort; #endif + int threadPoolThreadTimeoutMs; + int threadPoolThreadsToKeepAlive; + bool fDisableDefaultCodeVersioning; #if defined(FEATURE_TIERED_COMPILATION) diff --git a/src/coreclr/vm/win32threadpool.cpp b/src/coreclr/vm/win32threadpool.cpp index 7ffbc34948a47..1c8f26fa75f14 100644 --- a/src/coreclr/vm/win32threadpool.cpp +++ b/src/coreclr/vm/win32threadpool.cpp @@ -126,6 +126,10 @@ SVAL_IMPL(ThreadpoolMgr::LIST_ENTRY,ThreadpoolMgr,TimerQueue); // queue of time unsigned int ThreadpoolMgr::LastCPThreadCreation=0; // last time a completion port thread was created unsigned int ThreadpoolMgr::NumberOfProcessors; // = NumberOfWorkerThreads - no. of blocked threads +DWORD ThreadpoolMgr::WorkerThreadTimeoutMs = 20 * 1000; +DWORD ThreadpoolMgr::IOCompletionThreadTimeoutMs = 15 * 1000; +int ThreadpoolMgr::NumWorkerThreadsBeingKeptAlive = 0; +int ThreadpoolMgr::NumIOCompletionThreadsBeingKeptAlive = 0; CrstStatic ThreadpoolMgr::WorkerCriticalSection; CLREvent * ThreadpoolMgr::RetiredCPWakeupEvent; // wakeup event for completion port threads @@ -354,6 +358,13 @@ BOOL ThreadpoolMgr::Initialize() NumberOfProcessors = GetCurrentProcessCpuCount(); InitPlatformVariables(); + int threadTimeoutMs = g_pConfig->ThreadPoolThreadTimeoutMs(); + if (threadTimeoutMs >= -1) + { + WorkerThreadTimeoutMs = (DWORD)threadTimeoutMs; + IOCompletionThreadTimeoutMs = (DWORD)threadTimeoutMs; + } + EX_TRY { if (!UsePortableThreadPool()) @@ -1809,6 +1820,9 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) ThreadCounter::Counts counts, oldCounts, newCounts; bool foundWork = true, wasNotRecalled = true; + bool isThreadKeepAliveInitialized = false; + bool keepThreadAlive = false; + counts = WorkerCounter.GetCleanCounts(); if (ETW_EVENT_ENABLED(MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context, ThreadPoolWorkerThreadStart)) FireEtwThreadPoolWorkerThreadStart(counts.NumActive, counts.NumRetired, GetClrInstanceId()); @@ -1865,6 +1879,35 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) GCX_PREEMP_NO_DTOR(); _ASSERTE(pThread == NULL || !pThread->PreemptiveGCDisabled()); + if (!isThreadKeepAliveInitialized && fThreadInit) + { + // Determine whether to keep this thread alive. Some threads may always be kept alive based on config. + isThreadKeepAliveInitialized = true; + int threadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive(); + if (threadsToKeepAlive != 0) + { + if (threadsToKeepAlive < 0) + { + keepThreadAlive = true; + } + else + { + int count = VolatileLoadWithoutBarrier(&NumWorkerThreadsBeingKeptAlive); + while (count < threadsToKeepAlive) + { + int countBeforeUpdate = InterlockedCompareExchangeT(&NumWorkerThreadsBeingKeptAlive, count + 1, count); + if (countBeforeUpdate == count) + { + keepThreadAlive = true; + break; + } + + count = countBeforeUpdate; + } + } + } + } + // make sure there's really work. If not, go back to sleep // counts volatile read paired with CompareExchangeCounts loop set @@ -1951,7 +1994,7 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) while (true) { RetryRetire: - if (RetiredWorkerSemaphore->Wait(WorkerTimeout)) + if (RetiredWorkerSemaphore->Wait(keepThreadAlive ? INFINITE : WorkerThreadTimeoutMs)) { foundWork = true; @@ -2028,7 +2071,7 @@ DWORD WINAPI ThreadpoolMgr::WorkerThreadStart(LPVOID lpArgs) FireEtwThreadPoolWorkerThreadWait(counts.NumActive, counts.NumRetired, GetClrInstanceId()); RetryWaitForWork: - if (WorkerSemaphore->Wait(WorkerTimeout, WorkerThreadSpinLimit, NumberOfProcessors)) + if (WorkerSemaphore->Wait(keepThreadAlive ? INFINITE : WorkerThreadTimeoutMs, WorkerThreadSpinLimit, NumberOfProcessors)) { foundWork = true; goto Work; @@ -3153,13 +3196,13 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) PIOCompletionContext context; BOOL fIsCompletionContext; - const DWORD CP_THREAD_WAIT = 15000; /* milliseconds */ - _ASSERTE(GlobalCompletionPort != NULL); BOOL fThreadInit = FALSE; Thread *pThread = NULL; + bool isThreadKeepAliveInitialized = false; + bool keepThreadAlive = false; DWORD cpThreadWait = 0; if (g_fEEStarted) { @@ -3196,7 +3239,7 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) ThreadCounter::Counts oldCounts; ThreadCounter::Counts newCounts; - cpThreadWait = CP_THREAD_WAIT; + cpThreadWait = IOCompletionThreadTimeoutMs; for (;; ) { Top: @@ -3224,6 +3267,36 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) GCX_PREEMP_NO_DTOR(); + if (!isThreadKeepAliveInitialized && fThreadInit) + { + // Determine whether to keep this thread alive. Some threads may always be kept alive based on config. + isThreadKeepAliveInitialized = true; + int threadsToKeepAlive = g_pConfig->ThreadPoolThreadsToKeepAlive(); + if (threadsToKeepAlive != 0) + { + if (threadsToKeepAlive < 0) + { + keepThreadAlive = true; + } + else + { + int count = VolatileLoadWithoutBarrier(&NumIOCompletionThreadsBeingKeptAlive); + while (count < threadsToKeepAlive) + { + int countBeforeUpdate = + InterlockedCompareExchangeT(&NumIOCompletionThreadsBeingKeptAlive, count + 1, count); + if (countBeforeUpdate == count) + { + keepThreadAlive = true; + break; + } + + count = countBeforeUpdate; + } + } + } + } + // // We're about to wait on the IOCP; mark ourselves as no longer "working." // @@ -3238,7 +3311,7 @@ DWORD WINAPI ThreadpoolMgr::CompletionPortThreadStart(LPVOID lpArgs) // one thread listening for completions. So there's no point in having a timeout; it will // only use power unnecessarily. // - cpThreadWait = (newCounts.NumActive == 1) ? INFINITE : CP_THREAD_WAIT; + cpThreadWait = (newCounts.NumActive == 1 || keepThreadAlive) ? INFINITE : IOCompletionThreadTimeoutMs; if (oldCounts == CPThreadCounter.CompareExchangeCounts(newCounts, oldCounts)) break; diff --git a/src/coreclr/vm/win32threadpool.h b/src/coreclr/vm/win32threadpool.h index 4f3397821ded4..20aeccce7f4e5 100644 --- a/src/coreclr/vm/win32threadpool.h +++ b/src/coreclr/vm/win32threadpool.h @@ -1049,6 +1049,11 @@ class ThreadpoolMgr static unsigned int LastCPThreadCreation; // last time a completion port thread was created static unsigned int NumberOfProcessors; // = NumberOfWorkerThreads - no. of blocked threads + static DWORD WorkerThreadTimeoutMs; + static DWORD IOCompletionThreadTimeoutMs; + static int NumWorkerThreadsBeingKeptAlive; + static int NumIOCompletionThreadsBeingKeptAlive; + static BOOL IsApcPendingOnWaitThread; // Indicates if an APC is pending on the wait thread // This needs to be non-hosted, because worker threads can run prior to EE startup. @@ -1058,8 +1063,6 @@ class ThreadpoolMgr static CrstStatic WorkerCriticalSection; private: - static const DWORD WorkerTimeout = 20 * 1000; - DECLSPEC_ALIGN(MAX_CACHE_LINE_SIZE) SVAL_DECL(ThreadCounter,WorkerCounter); // diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs index 886e774fd22fb..24f1f03359502 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.WorkerThread.cs @@ -7,6 +7,8 @@ namespace System.Threading { internal sealed partial class PortableThreadPool { + private int _numThreadsBeingKeptAlive; + /// /// The worker thread infastructure for the CLR thread pool. /// @@ -17,6 +19,22 @@ private static partial class WorkerThread // preexisting threads from running out of memory when using new stack space in low-memory situations. public const int EstimatedAdditionalStackUsagePerThreadBytes = 64 << 10; // 64 KB + private static readonly short ThreadsToKeepAlive = DetermineThreadsToKeepAlive(); + + private static short DetermineThreadsToKeepAlive() + { + const short DefaultThreadsToKeepAlive = 0; + + // The number of worker threads to keep alive after they are created. Set to -1 to keep all created worker + // threads alive. When the ThreadTimeoutMs config value is also set, for worker threads the timeout applies to + // worker threads that are in excess of the number configured for ThreadsToKeepAlive. + short threadsToKeepAlive = + AppContextConfigHelper.GetInt16Config( + "System.Threading.ThreadPool.ThreadsToKeepAlive", + DefaultThreadsToKeepAlive); + return threadsToKeepAlive >= -1 ? threadsToKeepAlive : DefaultThreadsToKeepAlive; + } + /// /// Semaphore for controlling how many threads are currently working. /// @@ -51,10 +69,36 @@ private static void WorkerThreadStart() LowLevelLock threadAdjustmentLock = threadPoolInstance._threadAdjustmentLock; LowLevelLifoSemaphore semaphore = s_semaphore; + // Determine the idle timeout to use for this thread. Some threads may always be kept alive based on config. + int timeoutMs = ThreadPoolThreadTimeoutMs; + if (ThreadsToKeepAlive != 0) + { + if (ThreadsToKeepAlive < 0) + { + timeoutMs = Timeout.Infinite; + } + else + { + int count = threadPoolInstance._numThreadsBeingKeptAlive; + while (count < ThreadsToKeepAlive) + { + int countBeforeUpdate = + Interlocked.CompareExchange(ref threadPoolInstance._numThreadsBeingKeptAlive, count + 1, count); + if (countBeforeUpdate == count) + { + timeoutMs = Timeout.Infinite; + break; + } + + count = countBeforeUpdate; + } + } + } + while (true) { bool spinWait = true; - while (semaphore.Wait(ThreadPoolThreadTimeoutMs, spinWait)) + while (semaphore.Wait(timeoutMs, spinWait)) { bool alreadyRemovedWorkingWorker = false; while (TakeActiveRequest(threadPoolInstance)) diff --git a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs index a425298cacd8e..5dd945b013448 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Threading/PortableThreadPool.cs @@ -12,7 +12,6 @@ namespace System.Threading /// internal sealed partial class PortableThreadPool { - private const int ThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests. private const int SmallStackSizeBytes = 256 * 1024; private const short MaxPossibleThreadCount = short.MaxValue; @@ -33,6 +32,22 @@ internal sealed partial class PortableThreadPool private static readonly short ForcedMaxWorkerThreads = AppContextConfigHelper.GetInt16Config("System.Threading.ThreadPool.MaxThreads", 0, false); + private static readonly int ThreadPoolThreadTimeoutMs = DetermineThreadPoolThreadTimeoutMs(); + + private static int DetermineThreadPoolThreadTimeoutMs() + { + const int DefaultThreadPoolThreadTimeoutMs = 20 * 1000; // If you change this make sure to change the timeout times in the tests. + + // The amount of time in milliseconds a thread pool thread waits without having done any work before timing out and + // exiting. Set to -1 to disable the timeout. Applies to worker threads and wait threads. Also see the + // ThreadsToKeepAlive config value for relevant information. + int timeoutMs = + AppContextConfigHelper.GetInt32Config( + "System.Threading.ThreadPool.ThreadTimeoutMs", + DefaultThreadPoolThreadTimeoutMs); + return timeoutMs >= -1 ? timeoutMs : DefaultThreadPoolThreadTimeoutMs; + } + [ThreadStatic] private static object? t_completionCountObject; From 8b530d463bce645ca8f17259a9a4b4274e805917 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Fri, 13 Oct 2023 16:18:54 -0600 Subject: [PATCH 08/12] [release/6.0-staging] Update dependencies from dotnet/xharness (#93396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Update dependencies from https://github.com/dotnet/xharness build 20230905.2 Microsoft.DotNet.XHarness.CLI , Microsoft.DotNet.XHarness.TestRunners.Xunit From Version 6.0.0-prerelease.23401.2 -> To Version 6.0.0-prerelease.23455.2 * Re-add removed feed --------- Co-authored-by: dotnet-maestro[bot] Co-authored-by: Carlos Sánchez López <1175054+carlossanlop@users.noreply.github.com> --- .config/dotnet-tools.json | 2 +- eng/Version.Details.xml | 8 ++++---- eng/Versions.props | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index b059ebd61620f..b4a4cb6770cd5 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -15,7 +15,7 @@ ] }, "microsoft.dotnet.xharness.cli": { - "version": "6.0.0-prerelease.23401.2", + "version": "6.0.0-prerelease.23455.2", "commands": [ "xharness" ] diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 915d22daed6df..de85f30804098 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -214,13 +214,13 @@ https://github.com/mono/linker c8499798a2a09639174e2f5c694d6652794cc73d - + https://github.com/dotnet/xharness - 5aa438c8aa5ca20646b5d91149fd9be8646dbe65 + dcd239f92887f600f75093d5ffff27b2dfeb034b - + https://github.com/dotnet/xharness - 5aa438c8aa5ca20646b5d91149fd9be8646dbe65 + dcd239f92887f600f75093d5ffff27b2dfeb034b https://github.com/dotnet/arcade diff --git a/eng/Versions.props b/eng/Versions.props index 459f13492a60e..1c978de492611 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -143,8 +143,8 @@ 1.0.1-prerelease-00006 17.4.0-preview-20220707-01 - 6.0.0-prerelease.23401.2 - 6.0.0-prerelease.23401.2 + 6.0.0-prerelease.23455.2 + 6.0.0-prerelease.23455.2 6.0.0-alpha.0.23411.2 6.0.0-alpha.0.23367.3 2.4.2-pre.9 From bd3c9794e7aef0afc016b847bf0baa2038bba786 Mon Sep 17 00:00:00 2001 From: Jan Kotas Date: Mon, 16 Oct 2023 11:29:28 -0700 Subject: [PATCH 09/12] Workaround for https://github.com/dotnet/runtime/issues/93442 (#93530) --- src/coreclr/inc/safemath.h | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/coreclr/inc/safemath.h b/src/coreclr/inc/safemath.h index 3c020de688478..f1b0300d58195 100644 --- a/src/coreclr/inc/safemath.h +++ b/src/coreclr/inc/safemath.h @@ -481,6 +481,9 @@ template class ClrSafeInt Which ought to inline nicely */ // Returns true if safe, false for overflow. +#if defined(_MSC_VER) && defined(HOST_ARM64) // Workaround for https://github.com/dotnet/runtime/issues/93442 +#pragma optimize("", off) +#endif static bool multiply(T lhs, T rhs, T &result) { if(Is64Bit()) @@ -675,6 +678,9 @@ template class ClrSafeInt } } } +#if defined(_MSC_VER) && defined(HOST_ARM64) // Workaround for https://github.com/dotnet/runtime/issues/93442 +#pragma optimize("", on) +#endif // Returns true if safe, false on overflow static inline bool addition(T lhs, T rhs, T &result) From a753662e3fb950521bfec3af79f3783cc08f460f Mon Sep 17 00:00:00 2001 From: Anton Firszov Date: Mon, 16 Oct 2023 22:09:12 +0200 Subject: [PATCH 10/12] TcpReceiveSendGetsCanceledByDispose: update test for change in Linux kernel. (#93198) (#93554) --- .../tests/FunctionalTests/SendReceive/SendReceive.cs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs index 4d43977fa2b3c..9e6d4cf7fc6f7 100644 --- a/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs +++ b/src/libraries/System.Net.Sockets/tests/FunctionalTests/SendReceive/SendReceive.cs @@ -1002,11 +1002,12 @@ await Task.WhenAny(disposeTask, receiveTask) [ActiveIssue("https://github.com/dotnet/runtime/issues/50568", TestPlatforms.Android)] public async Task TcpReceiveSendGetsCanceledByDispose(bool receiveOrSend, bool ipv6Server, bool dualModeClient) { - // RHEL7 kernel has a bug preventing close(AF_UNKNOWN) to succeed with IPv6 sockets. - // In this case Dispose will trigger a graceful shutdown, which means that receive will succeed on socket2. - // This bug is fixed in kernel 3.10.0-1160.25+. - // TODO: Remove this, once CI machines are updated to a newer kernel. - bool mayShutdownGraceful = UsesSync && PlatformDetection.IsRedHatFamily7 && receiveOrSend && (ipv6Server || dualModeClient); + // .NET uses connect(AF_UNSPEC) to abort on-going operations on Linux. + // Linux 6.4+ introduced a change (4faeee0cf8a5d88d63cdbc3bab124fb0e6aed08c) which disallows + // this operation while operations are on-going. + // When the connect fails, .NET falls back to use shutdown(SHUT_RDWR). + // This causes the receive on socket2 to succeed instead of failing with ConnectionReset. + bool mayShutdownGraceful = UsesSync && PlatformDetection.IsLinux && receiveOrSend; // We try this a couple of times to deal with a timing race: if the Dispose happens // before the operation is started, the peer won't see a ConnectionReset SocketException and we won't From 142b0fd293ca4a0445dc5f6b4d32bff144bf6cb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Mon, 16 Oct 2023 18:02:55 -0600 Subject: [PATCH 11/12] ProcessTests: allow WorkingSet to be zero just after launching the process. (#85649) (#93574) * ProcessTests: allow WorkingSet to be zero just after launching the process. The WorkingSet tests fail on Fedora 38+ because a zero working set value is observed just after the process start. Co-authored-by: Tom Deseyn --- .../tests/ProcessTests.cs | 82 ++++++++++++++++++- 1 file changed, 78 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs index 27d05f0d7075d..e49dd958395d1 100644 --- a/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs +++ b/src/libraries/System.Diagnostics.Process/tests/ProcessTests.cs @@ -753,7 +753,25 @@ public void TestPeakWorkingSet64() { CreateDefaultProcess(); - AssertNonZeroAllZeroDarwin(_process.PeakWorkingSet64); + if (OperatingSystem.IsMacOS()) + { + Assert.Equal(0, _process.PeakWorkingSet64); + return; + } + + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { + Assert.NotEqual(0, _process.PeakWorkingSet64); + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -806,7 +824,19 @@ public void TestWorkingSet64() return; } - Assert.InRange(_process.WorkingSet64, 1, long.MaxValue); + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { + Assert.InRange(_process.WorkingSet64, 1, long.MaxValue); + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -1968,9 +1998,29 @@ public void TestPeakWorkingSet() { CreateDefaultProcess(); + if (OperatingSystem.IsMacOS()) + { #pragma warning disable 0618 - AssertNonZeroAllZeroDarwin(_process.PeakWorkingSet); + Assert.Equal(0, _process.PeakWorkingSet); #pragma warning restore 0618 + return; + } + + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { +#pragma warning disable 0618 + Assert.NotEqual(0, _process.PeakWorkingSet); +#pragma warning restore 0618 + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -2034,9 +2084,21 @@ public void TestWorkingSet() return; } + // On recent Linux kernels (6.2+) working set can be zero just after the process started. + ExecuteWithRetryOnLinux(() => + { + try + { #pragma warning disable 0618 - Assert.InRange(_process.WorkingSet, 1, int.MaxValue); + Assert.InRange(_process.WorkingSet, 1, int.MaxValue); #pragma warning restore 0618 + } + catch + { + _process.Refresh(); + throw; + } + }); } [Fact] @@ -2587,5 +2649,17 @@ private SecureString AsSecureString(string str) return secureString; } + + private static void ExecuteWithRetryOnLinux(Action test) + { + if (OperatingSystem.IsLinux()) + { + RetryHelper.Execute(test, retryWhen: ex => ex is XunitException); + } + else + { + test(); + } + } } } From c76ae01da7c977682df1685b4fe4bbe6f226108b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Carlos=20S=C3=A1nchez=20L=C3=B3pez?= <1175054+carlossanlop@users.noreply.github.com> Date: Mon, 16 Oct 2023 20:02:19 -0600 Subject: [PATCH 12/12] [6.0] Bump Ubuntu 19.10 queues to 22.04 (#93578) --- eng/pipelines/libraries/helix-queues-setup.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/pipelines/libraries/helix-queues-setup.yml b/eng/pipelines/libraries/helix-queues-setup.yml index 51989907e0a53..2feb27db013f6 100644 --- a/eng/pipelines/libraries/helix-queues-setup.yml +++ b/eng/pipelines/libraries/helix-queues-setup.yml @@ -57,7 +57,7 @@ jobs: - RedHat.7.Amd64.Open - SLES.15.Amd64.Open - (Fedora.34.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - - (Ubuntu.1910.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64 + - ubuntu.2204.amd64.open.svc - (Debian.10.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - ${{ if or(ne(parameters.jobParameters.testScope, 'outerloop'), ne(parameters.jobParameters.runtimeFlavor, 'mono')) }}: - ${{ if eq(parameters.jobParameters.isFullMatrix, true) }}: @@ -67,7 +67,7 @@ jobs: - SLES.12.Amd64.Open - SLES.15.Amd64.Open - (Fedora.34.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-34-helix - - (Ubuntu.1910.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-19.10-helix-amd64 + - ubuntu.2204.amd64.open.svc - (Debian.10.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-10-helix-amd64 - (Debian.11.Amd64.Open)Ubuntu.1804.Amd64.Open@mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-helix-amd64 - (Mariner.1.0.Amd64.Open)ubuntu.2204.amd64.open.svc@mcr.microsoft.com/dotnet-buildtools/prereqs:cbl-mariner-1.0-helix