From d37d1ee963963b3546826f729268df862d9d565a Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 27 May 2021 13:00:13 +0200 Subject: [PATCH 01/71] move logic related to opening and initializing handle from FileStreamHelpers to SafeFileHandle itself --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 183 ++++++++++++++++-- .../SafeHandles/SafeFileHandle.Windows.cs | 129 ++++++++++++ .../IO/Strategies/FileStreamHelpers.Unix.cs | 82 -------- .../Strategies/FileStreamHelpers.Windows.cs | 128 ------------ .../Net5CompatFileStreamStrategy.Unix.cs | 73 +------ .../Net5CompatFileStreamStrategy.cs | 2 +- .../Strategies/WindowsFileStreamStrategy.cs | 2 +- 7 files changed, 303 insertions(+), 296 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 27efa1fd31979..81a870510be57 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -4,6 +4,10 @@ using System; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices; +using System.Runtime.Versioning; +using System.Threading; +using System.Threading.Tasks; namespace Microsoft.Win32.SafeHandles { @@ -26,15 +30,29 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : this(ownsHand internal bool? IsAsync { get; set; } - /// Opens the specified file with the requested flags and mode. - /// The path to the file. - /// The flags with which to open the file. - /// The mode for opening the file. - /// A SafeFileHandle for the opened file. - internal static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, int mode) + internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) { - Debug.Assert(path != null); - SafeFileHandle handle = Interop.Sys.Open(path, flags, mode); + // Translate the arguments into arguments for an open call. + Interop.Sys.OpenFlags openFlags = PreOpenConfigurationFromOptions(mode, access, share, options); + + // If the file gets created a new, we'll select the permissions for it. Most Unix utilities by default use 666 (read and + // write for all), so we do the same (even though this doesn't match Windows, where by default it's possible to write out + // a file and then execute it). No matter what we choose, it'll be subject to the umask applied by the system, such that the + // actual permissions will typically be less than what we select here. + const Interop.Sys.Permissions OpenPermissions = + Interop.Sys.Permissions.S_IRUSR | Interop.Sys.Permissions.S_IWUSR | + Interop.Sys.Permissions.S_IRGRP | Interop.Sys.Permissions.S_IWGRP | + Interop.Sys.Permissions.S_IROTH | Interop.Sys.Permissions.S_IWOTH; + + SafeFileHandle safeFileHandle = Open(fullPath, openFlags, (int)OpenPermissions); + Init(safeFileHandle, fullPath, mode, access, share, options, preallocationSize); + return safeFileHandle; + } + + private static SafeFileHandle Open(string fullPath, Interop.Sys.OpenFlags flags, int mode) + { + Debug.Assert(fullPath != null); + SafeFileHandle handle = Interop.Sys.Open(fullPath, flags, mode); if (handle.IsInvalid) { @@ -50,11 +68,11 @@ internal static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, in bool isDirectory = (error.Error == Interop.Error.ENOENT) && ((flags & Interop.Sys.OpenFlags.O_CREAT) != 0 - || !DirectoryExists(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(path!))!)); + || !DirectoryExists(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(fullPath!))!)); Interop.CheckIo( error.Error, - path, + fullPath, isDirectory, errorRewriter: e => (e.Error == Interop.Error.EISDIR) ? Interop.Error.EACCES.Info() : e); } @@ -65,12 +83,12 @@ internal static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, in if (Interop.Sys.FStat(handle, out status) != 0) { handle.Dispose(); - throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path); + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), fullPath); } if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR) { handle.Dispose(); - throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), path, isDirectory: true); + throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), fullPath, isDirectory: true); } return handle; @@ -125,5 +143,146 @@ public override bool IsInvalid return h < 0 || h > int.MaxValue; } } + + /// Translates the FileMode, FileAccess, and FileOptions values into flags to be passed when opening the file. + /// The FileMode provided to the stream's constructor. + /// The FileAccess provided to the stream's constructor + /// The FileShare provided to the stream's constructor + /// The FileOptions provided to the stream's constructor + /// The flags value to be passed to the open system call. + private static Interop.Sys.OpenFlags PreOpenConfigurationFromOptions(FileMode mode, FileAccess access, FileShare share, FileOptions options) + { + // Translate FileMode. Most of the values map cleanly to one or more options for open. + Interop.Sys.OpenFlags flags = default; + switch (mode) + { + default: + case FileMode.Open: // Open maps to the default behavior for open(...). No flags needed. + case FileMode.Truncate: // We truncate the file after getting the lock + break; + + case FileMode.Append: // Append is the same as OpenOrCreate, except that we'll also separately jump to the end later + case FileMode.OpenOrCreate: + case FileMode.Create: // We truncate the file after getting the lock + flags |= Interop.Sys.OpenFlags.O_CREAT; + break; + + case FileMode.CreateNew: + flags |= (Interop.Sys.OpenFlags.O_CREAT | Interop.Sys.OpenFlags.O_EXCL); + break; + } + + // Translate FileAccess. All possible values map cleanly to corresponding values for open. + switch (access) + { + case FileAccess.Read: + flags |= Interop.Sys.OpenFlags.O_RDONLY; + break; + + case FileAccess.ReadWrite: + flags |= Interop.Sys.OpenFlags.O_RDWR; + break; + + case FileAccess.Write: + flags |= Interop.Sys.OpenFlags.O_WRONLY; + break; + } + + // Handle Inheritable, other FileShare flags are handled by Init + if ((share & FileShare.Inheritable) == 0) + { + flags |= Interop.Sys.OpenFlags.O_CLOEXEC; + } + + // Translate some FileOptions; some just aren't supported, and others will be handled after calling open. + // - Asynchronous: Handled in ctor, setting _useAsync and SafeFileHandle.IsAsync to true + // - DeleteOnClose: Doesn't have a Unix equivalent, but we approximate it in Dispose + // - Encrypted: No equivalent on Unix and is ignored + // - RandomAccess: Implemented after open if posix_fadvise is available + // - SequentialScan: Implemented after open if posix_fadvise is available + // - WriteThrough: Handled here + if ((options & FileOptions.WriteThrough) != 0) + { + flags |= Interop.Sys.OpenFlags.O_SYNC; + } + + return flags; + } + + private static void Init(SafeFileHandle handle, string path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) + { + handle.IsAsync = (options & FileOptions.Asynchronous) != 0; + + // Lock the file if requested via FileShare. This is only advisory locking. FileShare.None implies an exclusive + // lock on the file and all other modes use a shared lock. While this is not as granular as Windows, not mandatory, + // and not atomic with file opening, it's better than nothing. + Interop.Sys.LockOperations lockOperation = (share == FileShare.None) ? Interop.Sys.LockOperations.LOCK_EX : Interop.Sys.LockOperations.LOCK_SH; + if (Interop.Sys.FLock(_fileHandle, lockOperation | Interop.Sys.LockOperations.LOCK_NB) < 0) + { + // The only error we care about is EWOULDBLOCK, which indicates that the file is currently locked by someone + // else and we would block trying to access it. Other errors, such as ENOTSUP (locking isn't supported) or + // EACCES (the file system doesn't allow us to lock), will only hamper FileStream's usage without providing value, + // given again that this is only advisory / best-effort. + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (errorInfo.Error == Interop.Error.EWOULDBLOCK) + { + throw Interop.GetExceptionForIoErrno(errorInfo, path, isDirectory: false); + } + } + + // These provide hints around how the file will be accessed. Specifying both RandomAccess + // and Sequential together doesn't make sense as they are two competing options on the same spectrum, + // so if both are specified, we prefer RandomAccess (behavior on Windows is unspecified if both are provided). + Interop.Sys.FileAdvice fadv = + (options & FileOptions.RandomAccess) != 0 ? Interop.Sys.FileAdvice.POSIX_FADV_RANDOM : + (options & FileOptions.SequentialScan) != 0 ? Interop.Sys.FileAdvice.POSIX_FADV_SEQUENTIAL : + 0; + if (fadv != 0) + { + CheckFileCall(Interop.Sys.PosixFAdvise(_fileHandle, 0, 0, fadv), + ignoreNotSupported: true); // just a hint. + } + + if (mode == FileMode.Append) + { + // Jump to the end of the file if opened as Append. + _appendStart = SeekCore(_fileHandle, 0, SeekOrigin.End); + } + else if (mode == FileMode.Create || mode == FileMode.Truncate) + { + // Truncate the file now if the file mode requires it. This ensures that the file only will be truncated + // if opened successfully. + if (Interop.Sys.FTruncate(_fileHandle, 0) < 0) + { + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (errorInfo.Error != Interop.Error.EBADF && errorInfo.Error != Interop.Error.EINVAL) + { + // We know the file descriptor is valid and we know the size argument to FTruncate is correct, + // so if EBADF or EINVAL is returned, it means we're dealing with a special file that can't be + // truncated. Ignore the error in such cases; in all others, throw. + throw Interop.GetExceptionForIoErrno(errorInfo, path, isDirectory: false); + } + } + } + + // If preallocationSize has been provided for a creatable and writeable file + if (FileStreamHelpers.ShouldPreallocate(preallocationSize, access, mode)) + { + int fallocateResult = Interop.Sys.PosixFAllocate(_fileHandle, 0, preallocationSize); + if (fallocateResult != 0) + { + _fileHandle.Dispose(); + Interop.Sys.Unlink(path!); // remove the file to mimic Windows behaviour (atomic operation) + + if (fallocateResult == -1) + { + throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, path, preallocationSize)); + } + + Debug.Assert(fallocateResult == -2); + throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, path, preallocationSize)); + } + } + } } } diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index dfcce3cfc65ac..4eaf14bf2e7ab 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -2,6 +2,11 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; +using System.IO; +using System.IO.Strategies; +using System.Runtime.InteropServices; +using System.Text; using System.Threading; namespace Microsoft.Win32.SafeHandles @@ -32,5 +37,129 @@ internal bool? IsAsync protected override bool ReleaseHandle() => Interop.Kernel32.CloseHandle(handle); + + internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) + { + using (DisableMediaInsertionPrompt.Create()) + { + Debug.Assert(fullPath != null); + + if (FileStreamHelpers.ShouldPreallocate(preallocationSize, access, mode)) + { + IntPtr fileHandle = NtCreateFile(fullPath, mode, access, share, options, preallocationSize); + + return ValidateFileHandle(new SafeFileHandle(fileHandle, ownsHandle: true), fullPath, (options & FileOptions.Asynchronous) != 0); + } + + Interop.Kernel32.SECURITY_ATTRIBUTES secAttrs = GetSecAttrs(share); + + int fAccess = + ((access & FileAccess.Read) == FileAccess.Read ? Interop.Kernel32.GenericOperations.GENERIC_READ : 0) | + ((access & FileAccess.Write) == FileAccess.Write ? Interop.Kernel32.GenericOperations.GENERIC_WRITE : 0); + + // Our Inheritable bit was stolen from Windows, but should be set in + // the security attributes class. Don't leave this bit set. + share &= ~FileShare.Inheritable; + + // Must use a valid Win32 constant here... + if (mode == FileMode.Append) + mode = FileMode.OpenOrCreate; + + int flagsAndAttributes = (int)options; + + // For mitigating local elevation of privilege attack through named pipes + // make sure we always call CreateFile with SECURITY_ANONYMOUS so that the + // named pipe server can't impersonate a high privileged client security context + // (note that this is the effective default on CreateFile2) + flagsAndAttributes |= (Interop.Kernel32.SecurityOptions.SECURITY_SQOS_PRESENT | Interop.Kernel32.SecurityOptions.SECURITY_ANONYMOUS); + + SafeFileHandle safeFileHandle = ValidateFileHandle( + Interop.Kernel32.CreateFile(fullPath, fAccess, share, &secAttrs, mode, flagsAndAttributes, IntPtr.Zero), + fullPath, + (options & FileOptions.Asynchronous) != 0); + + return safeFileHandle; + } + } + + private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) + { + uint ntStatus; + IntPtr fileHandle; + + const string mandatoryNtPrefix = @"\??\"; + if (fullPath.StartsWith(mandatoryNtPrefix, StringComparison.Ordinal)) + { + (ntStatus, fileHandle) = Interop.NtDll.CreateFile(fullPath, mode, access, share, options, preallocationSize); + } + else + { + var vsb = new ValueStringBuilder(stackalloc char[1024]); + vsb.Append(mandatoryNtPrefix); + + if (fullPath.StartsWith(@"\\?\", StringComparison.Ordinal)) // NtCreateFile does not support "\\?\" prefix, only "\??\" + { + vsb.Append(fullPath.AsSpan(4)); + } + else + { + vsb.Append(fullPath); + } + + (ntStatus, fileHandle) = Interop.NtDll.CreateFile(vsb.AsSpan(), mode, access, share, options, preallocationSize); + vsb.Dispose(); + } + + switch (ntStatus) + { + case 0: + return fileHandle; + case Interop.NtDll.NT_ERROR_STATUS_DISK_FULL: + throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, fullPath, preallocationSize)); + // NtCreateFile has a bug and it reports STATUS_INVALID_PARAMETER for files + // that are too big for the current file system. Example: creating a 4GB+1 file on a FAT32 drive. + case Interop.NtDll.NT_STATUS_INVALID_PARAMETER: + case Interop.NtDll.NT_ERROR_STATUS_FILE_TOO_LARGE: + throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, fullPath, preallocationSize)); + default: + int error = (int)Interop.NtDll.RtlNtStatusToDosError((int)ntStatus); + throw Win32Marshal.GetExceptionForWin32Error(error, fullPath); + } + } + + private static unsafe Interop.Kernel32.SECURITY_ATTRIBUTES GetSecAttrs(FileShare share) + { + Interop.Kernel32.SECURITY_ATTRIBUTES secAttrs = default; + if ((share & FileShare.Inheritable) != 0) + { + secAttrs = new Interop.Kernel32.SECURITY_ATTRIBUTES + { + nLength = (uint)sizeof(Interop.Kernel32.SECURITY_ATTRIBUTES), + bInheritHandle = Interop.BOOL.TRUE + }; + } + return secAttrs; + } + + private static SafeFileHandle ValidateFileHandle(SafeFileHandle fileHandle, string path, bool useAsyncIO) + { + if (fileHandle.IsInvalid) + { + // Return a meaningful exception with the full path. + + // NT5 oddity - when trying to open "C:\" as a Win32FileStream, + // we usually get ERROR_PATH_NOT_FOUND from the OS. We should + // probably be consistent w/ every other directory. + int errorCode = Marshal.GetLastPInvokeError(); + + if (errorCode == Interop.Errors.ERROR_PATH_NOT_FOUND && path!.Length == PathInternal.GetRootLength(path)) + errorCode = Interop.Errors.ERROR_ACCESS_DENIED; + + throw Win32Marshal.GetExceptionForWin32Error(errorCode, path); + } + + fileHandle.IsAsync = useAsyncIO; + return fileHandle; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs index 1fd6456f3eeb5..6395787823452 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs @@ -20,88 +20,6 @@ private static FileStreamStrategy ChooseStrategyCore(SafeFileHandle handle, File private static FileStreamStrategy ChooseStrategyCore(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize) => new Net5CompatFileStreamStrategy(path, mode, access, share, bufferSize == 0 ? 1 : bufferSize, options, preallocationSize); - internal static SafeFileHandle OpenHandle(string path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) - { - // Translate the arguments into arguments for an open call. - Interop.Sys.OpenFlags openFlags = PreOpenConfigurationFromOptions(mode, access, share, options); - - // If the file gets created a new, we'll select the permissions for it. Most Unix utilities by default use 666 (read and - // write for all), so we do the same (even though this doesn't match Windows, where by default it's possible to write out - // a file and then execute it). No matter what we choose, it'll be subject to the umask applied by the system, such that the - // actual permissions will typically be less than what we select here. - const Interop.Sys.Permissions OpenPermissions = - Interop.Sys.Permissions.S_IRUSR | Interop.Sys.Permissions.S_IWUSR | - Interop.Sys.Permissions.S_IRGRP | Interop.Sys.Permissions.S_IWGRP | - Interop.Sys.Permissions.S_IROTH | Interop.Sys.Permissions.S_IWOTH; - - return SafeFileHandle.Open(path!, openFlags, (int)OpenPermissions); - } - internal static bool GetDefaultIsAsync(SafeFileHandle handle, bool defaultIsAsync) => handle.IsAsync ?? defaultIsAsync; - - /// Translates the FileMode, FileAccess, and FileOptions values into flags to be passed when opening the file. - /// The FileMode provided to the stream's constructor. - /// The FileAccess provided to the stream's constructor - /// The FileShare provided to the stream's constructor - /// The FileOptions provided to the stream's constructor - /// The flags value to be passed to the open system call. - private static Interop.Sys.OpenFlags PreOpenConfigurationFromOptions(FileMode mode, FileAccess access, FileShare share, FileOptions options) - { - // Translate FileMode. Most of the values map cleanly to one or more options for open. - Interop.Sys.OpenFlags flags = default; - switch (mode) - { - default: - case FileMode.Open: // Open maps to the default behavior for open(...). No flags needed. - case FileMode.Truncate: // We truncate the file after getting the lock - break; - - case FileMode.Append: // Append is the same as OpenOrCreate, except that we'll also separately jump to the end later - case FileMode.OpenOrCreate: - case FileMode.Create: // We truncate the file after getting the lock - flags |= Interop.Sys.OpenFlags.O_CREAT; - break; - - case FileMode.CreateNew: - flags |= (Interop.Sys.OpenFlags.O_CREAT | Interop.Sys.OpenFlags.O_EXCL); - break; - } - - // Translate FileAccess. All possible values map cleanly to corresponding values for open. - switch (access) - { - case FileAccess.Read: - flags |= Interop.Sys.OpenFlags.O_RDONLY; - break; - - case FileAccess.ReadWrite: - flags |= Interop.Sys.OpenFlags.O_RDWR; - break; - - case FileAccess.Write: - flags |= Interop.Sys.OpenFlags.O_WRONLY; - break; - } - - // Handle Inheritable, other FileShare flags are handled by Init - if ((share & FileShare.Inheritable) == 0) - { - flags |= Interop.Sys.OpenFlags.O_CLOEXEC; - } - - // Translate some FileOptions; some just aren't supported, and others will be handled after calling open. - // - Asynchronous: Handled in ctor, setting _useAsync and SafeFileHandle.IsAsync to true - // - DeleteOnClose: Doesn't have a Unix equivalent, but we approximate it in Dispose - // - Encrypted: No equivalent on Unix and is ignored - // - RandomAccess: Implemented after open if posix_fadvise is available - // - SequentialScan: Implemented after open if posix_fadvise is available - // - WriteThrough: Handled here - if ((options & FileOptions.WriteThrough) != 0) - { - flags |= Interop.Sys.OpenFlags.O_SYNC; - } - - return flags; - } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs index f9f22b4d698f0..d1e14c3d49756 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs @@ -5,7 +5,6 @@ using System.Diagnostics; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; -using System.Text; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -61,98 +60,6 @@ private static FileStreamStrategy ChooseStrategyCore(string path, FileMode mode, internal static FileStreamStrategy EnableBufferingIfNeeded(WindowsFileStreamStrategy strategy, int bufferSize) => bufferSize > 1 ? new BufferedFileStreamStrategy(strategy, bufferSize) : strategy; - internal static SafeFileHandle OpenHandle(string path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) - => CreateFileOpenHandle(path, mode, access, share, options, preallocationSize); - - private static unsafe SafeFileHandle CreateFileOpenHandle(string path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) - { - using (DisableMediaInsertionPrompt.Create()) - { - Debug.Assert(path != null); - - if (ShouldPreallocate(preallocationSize, access, mode)) - { - IntPtr fileHandle = NtCreateFile(path, mode, access, share, options, preallocationSize); - - return ValidateFileHandle(new SafeFileHandle(fileHandle, ownsHandle: true), path, (options & FileOptions.Asynchronous) != 0); - } - - Interop.Kernel32.SECURITY_ATTRIBUTES secAttrs = GetSecAttrs(share); - - int fAccess = - ((access & FileAccess.Read) == FileAccess.Read ? Interop.Kernel32.GenericOperations.GENERIC_READ : 0) | - ((access & FileAccess.Write) == FileAccess.Write ? Interop.Kernel32.GenericOperations.GENERIC_WRITE : 0); - - // Our Inheritable bit was stolen from Windows, but should be set in - // the security attributes class. Don't leave this bit set. - share &= ~FileShare.Inheritable; - - // Must use a valid Win32 constant here... - if (mode == FileMode.Append) - mode = FileMode.OpenOrCreate; - - int flagsAndAttributes = (int)options; - - // For mitigating local elevation of privilege attack through named pipes - // make sure we always call CreateFile with SECURITY_ANONYMOUS so that the - // named pipe server can't impersonate a high privileged client security context - // (note that this is the effective default on CreateFile2) - flagsAndAttributes |= (Interop.Kernel32.SecurityOptions.SECURITY_SQOS_PRESENT | Interop.Kernel32.SecurityOptions.SECURITY_ANONYMOUS); - - SafeFileHandle safeFileHandle = ValidateFileHandle( - Interop.Kernel32.CreateFile(path, fAccess, share, &secAttrs, mode, flagsAndAttributes, IntPtr.Zero), - path, - (options & FileOptions.Asynchronous) != 0); - - return safeFileHandle; - } - } - - private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) - { - uint ntStatus; - IntPtr fileHandle; - - const string mandatoryNtPrefix = @"\??\"; - if (fullPath.StartsWith(mandatoryNtPrefix, StringComparison.Ordinal)) - { - (ntStatus, fileHandle) = Interop.NtDll.CreateFile(fullPath, mode, access, share, options, preallocationSize); - } - else - { - var vsb = new ValueStringBuilder(stackalloc char[1024]); - vsb.Append(mandatoryNtPrefix); - - if (fullPath.StartsWith(@"\\?\", StringComparison.Ordinal)) // NtCreateFile does not support "\\?\" prefix, only "\??\" - { - vsb.Append(fullPath.AsSpan(4)); - } - else - { - vsb.Append(fullPath); - } - - (ntStatus, fileHandle) = Interop.NtDll.CreateFile(vsb.AsSpan(), mode, access, share, options, preallocationSize); - vsb.Dispose(); - } - - switch (ntStatus) - { - case 0: - return fileHandle; - case Interop.NtDll.NT_ERROR_STATUS_DISK_FULL: - throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, fullPath, preallocationSize)); - // NtCreateFile has a bug and it reports STATUS_INVALID_PARAMETER for files - // that are too big for the current file system. Example: creating a 4GB+1 file on a FAT32 drive. - case Interop.NtDll.NT_STATUS_INVALID_PARAMETER: - case Interop.NtDll.NT_ERROR_STATUS_FILE_TOO_LARGE: - throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, fullPath, preallocationSize)); - default: - int error = (int)Interop.NtDll.RtlNtStatusToDosError((int)ntStatus); - throw Win32Marshal.GetExceptionForWin32Error(error, fullPath); - } - } - internal static bool GetDefaultIsAsync(SafeFileHandle handle, bool defaultIsAsync) { return handle.IsAsync ?? !IsHandleSynchronous(handle, ignoreInvalid: true) ?? defaultIsAsync; @@ -210,41 +117,6 @@ internal static void VerifyHandleIsSync(SafeFileHandle handle) ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); } - private static unsafe Interop.Kernel32.SECURITY_ATTRIBUTES GetSecAttrs(FileShare share) - { - Interop.Kernel32.SECURITY_ATTRIBUTES secAttrs = default; - if ((share & FileShare.Inheritable) != 0) - { - secAttrs = new Interop.Kernel32.SECURITY_ATTRIBUTES - { - nLength = (uint)sizeof(Interop.Kernel32.SECURITY_ATTRIBUTES), - bInheritHandle = Interop.BOOL.TRUE - }; - } - return secAttrs; - } - - private static SafeFileHandle ValidateFileHandle(SafeFileHandle fileHandle, string path, bool useAsyncIO) - { - if (fileHandle.IsInvalid) - { - // Return a meaningful exception with the full path. - - // NT5 oddity - when trying to open "C:\" as a Win32FileStream, - // we usually get ERROR_PATH_NOT_FOUND from the OS. We should - // probably be consistent w/ every other directory. - int errorCode = Marshal.GetLastPInvokeError(); - - if (errorCode == Interop.Errors.ERROR_PATH_NOT_FOUND && path!.Length == PathInternal.GetRootLength(path)) - errorCode = Interop.Errors.ERROR_ACCESS_DENIED; - - throw Win32Marshal.GetExceptionForWin32Error(errorCode, path); - } - - fileHandle.IsAsync = useAsyncIO; - return fileHandle; - } - internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) { Interop.Kernel32.FILE_STANDARD_INFO info; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs index f039c02043381..848b240c7ae05 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs @@ -51,78 +51,7 @@ private void Init(FileMode mode, FileShare share, string originalPath, FileOptio if (_useAsyncIO) _asyncState = new AsyncState(); - _fileHandle.IsAsync = _useAsyncIO; - - // Lock the file if requested via FileShare. This is only advisory locking. FileShare.None implies an exclusive - // lock on the file and all other modes use a shared lock. While this is not as granular as Windows, not mandatory, - // and not atomic with file opening, it's better than nothing. - Interop.Sys.LockOperations lockOperation = (share == FileShare.None) ? Interop.Sys.LockOperations.LOCK_EX : Interop.Sys.LockOperations.LOCK_SH; - if (Interop.Sys.FLock(_fileHandle, lockOperation | Interop.Sys.LockOperations.LOCK_NB) < 0) - { - // The only error we care about is EWOULDBLOCK, which indicates that the file is currently locked by someone - // else and we would block trying to access it. Other errors, such as ENOTSUP (locking isn't supported) or - // EACCES (the file system doesn't allow us to lock), will only hamper FileStream's usage without providing value, - // given again that this is only advisory / best-effort. - Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); - if (errorInfo.Error == Interop.Error.EWOULDBLOCK) - { - throw Interop.GetExceptionForIoErrno(errorInfo, _path, isDirectory: false); - } - } - - // These provide hints around how the file will be accessed. Specifying both RandomAccess - // and Sequential together doesn't make sense as they are two competing options on the same spectrum, - // so if both are specified, we prefer RandomAccess (behavior on Windows is unspecified if both are provided). - Interop.Sys.FileAdvice fadv = - (options & FileOptions.RandomAccess) != 0 ? Interop.Sys.FileAdvice.POSIX_FADV_RANDOM : - (options & FileOptions.SequentialScan) != 0 ? Interop.Sys.FileAdvice.POSIX_FADV_SEQUENTIAL : - 0; - if (fadv != 0) - { - CheckFileCall(Interop.Sys.PosixFAdvise(_fileHandle, 0, 0, fadv), - ignoreNotSupported: true); // just a hint. - } - - if (mode == FileMode.Append) - { - // Jump to the end of the file if opened as Append. - _appendStart = SeekCore(_fileHandle, 0, SeekOrigin.End); - } - else if (mode == FileMode.Create || mode == FileMode.Truncate) - { - // Truncate the file now if the file mode requires it. This ensures that the file only will be truncated - // if opened successfully. - if (Interop.Sys.FTruncate(_fileHandle, 0) < 0) - { - Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); - if (errorInfo.Error != Interop.Error.EBADF && errorInfo.Error != Interop.Error.EINVAL) - { - // We know the file descriptor is valid and we know the size argument to FTruncate is correct, - // so if EBADF or EINVAL is returned, it means we're dealing with a special file that can't be - // truncated. Ignore the error in such cases; in all others, throw. - throw Interop.GetExceptionForIoErrno(errorInfo, _path, isDirectory: false); - } - } - } - - // If preallocationSize has been provided for a creatable and writeable file - if (FileStreamHelpers.ShouldPreallocate(preallocationSize, _access, mode)) - { - int fallocateResult = Interop.Sys.PosixFAllocate(_fileHandle, 0, preallocationSize); - if (fallocateResult != 0) - { - _fileHandle.Dispose(); - Interop.Sys.Unlink(_path!); // remove the file to mimic Windows behaviour (atomic operation) - - if (fallocateResult == -1) - { - throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, _path, preallocationSize)); - } - - Debug.Assert(fallocateResult == -2); - throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, _path, preallocationSize)); - } - } + Debug.Assert(_fileHandle.IsAsync == _useAsyncIO); } /// Initializes a stream from an already open file handle (file descriptor). diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs index d257df97722da..1ff5b1c81c9a9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs @@ -89,7 +89,7 @@ internal Net5CompatFileStreamStrategy(string path, FileMode mode, FileAccess acc if ((options & FileOptions.Asynchronous) != 0) _useAsyncIO = true; - _fileHandle = FileStreamHelpers.OpenHandle(fullPath, mode, access, share, options, preallocationSize); + _fileHandle = SafeFileHandle.Open(fullPath, mode, access, share, options, preallocationSize); try { diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs index a5ebdd6f5da59..7fba6bb32a8d6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs @@ -45,7 +45,7 @@ internal WindowsFileStreamStrategy(string path, FileMode mode, FileAccess access _access = access; _share = share; - _fileHandle = FileStreamHelpers.OpenHandle(fullPath, mode, access, share, options, preallocationSize); + _fileHandle = SafeFileHandle.Open(fullPath, mode, access, share, options, preallocationSize); try { From 539393dc92ae6da0e1b82ace33f4e63a0634d66b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 27 May 2021 13:15:00 +0200 Subject: [PATCH 02/71] move the arguments validation logic common for FileStream ctor and File.OpenHandle to FileStreamHelpers --- .../src/System/IO/FileStream.cs | 67 +---------------- .../System/IO/Strategies/FileStreamHelpers.cs | 71 +++++++++++++++++++ 2 files changed, 72 insertions(+), 66 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs index 7cbe9e32bcbd4..de1dceec67d31 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs @@ -17,9 +17,6 @@ public class FileStream : Stream internal const FileShare DefaultShare = FileShare.Read; private const bool DefaultIsAsync = false; - /// Caches whether Serialization Guard has been disabled for file writes - private static int s_cachedSerializationSwitch; - private readonly FileStreamStrategy _strategy; [EditorBrowsable(EditorBrowsableState.Never)] @@ -175,69 +172,7 @@ public FileStream(string path, FileStreamOptions options) private FileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize) { - if (path == null) - { - throw new ArgumentNullException(nameof(path), SR.ArgumentNull_Path); - } - else if (path.Length == 0) - { - throw new ArgumentException(SR.Argument_EmptyPath, nameof(path)); - } - - // don't include inheritable in our bounds check for share - FileShare tempshare = share & ~FileShare.Inheritable; - string? badArg = null; - - if (mode < FileMode.CreateNew || mode > FileMode.Append) - { - badArg = nameof(mode); - } - else if (access < FileAccess.Read || access > FileAccess.ReadWrite) - { - badArg = nameof(access); - } - else if (tempshare < FileShare.None || tempshare > (FileShare.ReadWrite | FileShare.Delete)) - { - badArg = nameof(share); - } - - if (badArg != null) - { - throw new ArgumentOutOfRangeException(badArg, SR.ArgumentOutOfRange_Enum); - } - - // NOTE: any change to FileOptions enum needs to be matched here in the error validation - if (options != FileOptions.None && (options & ~(FileOptions.WriteThrough | FileOptions.Asynchronous | FileOptions.RandomAccess | FileOptions.DeleteOnClose | FileOptions.SequentialScan | FileOptions.Encrypted | (FileOptions)0x20000000 /* NoBuffering */)) != 0) - { - throw new ArgumentOutOfRangeException(nameof(options), SR.ArgumentOutOfRange_Enum); - } - else if (bufferSize < 0) - { - throw new ArgumentOutOfRangeException(nameof(bufferSize), SR.ArgumentOutOfRange_NeedPosNum); - } - else if (preallocationSize < 0) - { - throw new ArgumentOutOfRangeException(nameof(preallocationSize), SR.ArgumentOutOfRange_NeedNonNegNum); - } - - // Write access validation - if ((access & FileAccess.Write) == 0) - { - if (mode == FileMode.Truncate || mode == FileMode.CreateNew || mode == FileMode.Create || mode == FileMode.Append) - { - // No write access, mode and access disagree but flag access since mode comes first - throw new ArgumentException(SR.Format(SR.Argument_InvalidFileModeAndAccessCombo, mode, access), nameof(access)); - } - } - - if ((access & FileAccess.Read) != 0 && mode == FileMode.Append) - { - throw new ArgumentException(SR.Argument_InvalidAppendMode, nameof(access)); - } - else if ((access & FileAccess.Write) == FileAccess.Write) - { - SerializationInfo.ThrowIfDeserializationInProgress("AllowFileWrites", ref s_cachedSerializationSwitch); - } + FileStreamHelpers.ValidateArguments(path, mode, access, share, bufferSize, options, preallocationSize); _strategy = FileStreamHelpers.ChooseStrategy(this, path, mode, access, share, bufferSize, options, preallocationSize); } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs index 7dfaf7fe52dc7..50a6bb262c9ba 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs @@ -1,12 +1,16 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Runtime.Serialization; using Microsoft.Win32.SafeHandles; namespace System.IO.Strategies { internal static partial class FileStreamHelpers { + /// Caches whether Serialization Guard has been disabled for file writes + private static int s_cachedSerializationSwitch; + internal static bool UseNet5CompatStrategy { get; } = AppContextConfigHelper.GetBooleanConfig("System.IO.UseNet5CompatFileStream", "DOTNET_SYSTEM_IO_USENET5COMPATFILESTREAM"); internal static FileStreamStrategy ChooseStrategy(FileStream fileStream, SafeFileHandle handle, FileAccess access, FileShare share, int bufferSize, bool isAsync) @@ -41,5 +45,72 @@ internal static bool ShouldPreallocate(long preallocationSize, FileAccess access && (access & FileAccess.Write) != 0 && mode != FileMode.Open && mode != FileMode.Append && !OperatingSystem.IsBrowser(); // WASM limitation + + internal static void ValidateArguments(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize) + { + if (path == null) + { + throw new ArgumentNullException(nameof(path), SR.ArgumentNull_Path); + } + else if (path.Length == 0) + { + throw new ArgumentException(SR.Argument_EmptyPath, nameof(path)); + } + + // don't include inheritable in our bounds check for share + FileShare tempshare = share & ~FileShare.Inheritable; + string? badArg = null; + + if (mode < FileMode.CreateNew || mode > FileMode.Append) + { + badArg = nameof(mode); + } + else if (access < FileAccess.Read || access > FileAccess.ReadWrite) + { + badArg = nameof(access); + } + else if (tempshare < FileShare.None || tempshare > (FileShare.ReadWrite | FileShare.Delete)) + { + badArg = nameof(share); + } + + if (badArg != null) + { + throw new ArgumentOutOfRangeException(badArg, SR.ArgumentOutOfRange_Enum); + } + + // NOTE: any change to FileOptions enum needs to be matched here in the error validation + if (options != FileOptions.None && (options & ~(FileOptions.WriteThrough | FileOptions.Asynchronous | FileOptions.RandomAccess | FileOptions.DeleteOnClose | FileOptions.SequentialScan | FileOptions.Encrypted | (FileOptions)0x20000000 /* NoBuffering */)) != 0) + { + throw new ArgumentOutOfRangeException(nameof(options), SR.ArgumentOutOfRange_Enum); + } + else if (bufferSize < 0) + { + throw new ArgumentOutOfRangeException(nameof(bufferSize), SR.ArgumentOutOfRange_NeedPosNum); + } + else if (preallocationSize < 0) + { + throw new ArgumentOutOfRangeException(nameof(preallocationSize), SR.ArgumentOutOfRange_NeedNonNegNum); + } + + // Write access validation + if ((access & FileAccess.Write) == 0) + { + if (mode == FileMode.Truncate || mode == FileMode.CreateNew || mode == FileMode.Create || mode == FileMode.Append) + { + // No write access, mode and access disagree but flag access since mode comes first + throw new ArgumentException(SR.Format(SR.Argument_InvalidFileModeAndAccessCombo, mode, access), nameof(access)); + } + } + + if ((access & FileAccess.Read) != 0 && mode == FileMode.Append) + { + throw new ArgumentException(SR.Argument_InvalidAppendMode, nameof(access)); + } + else if ((access & FileAccess.Write) == FileAccess.Write) + { + SerializationInfo.ThrowIfDeserializationInProgress("AllowFileWrites", ref s_cachedSerializationSwitch); + } + } } } From b60e876a8f27701443967b1dbdbf8f1e5729862d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 27 May 2021 13:16:06 +0200 Subject: [PATCH 03/71] introduce File.OpenHandle --- .../tests/File/OpenHandle.cs | 27 +++++++++++ .../tests/FileStream/ctor_options_as.cs | 2 +- .../src/System/IO/File.cs | 46 ++++++++++++++++++- .../System.Runtime/ref/System.Runtime.cs | 1 + 4 files changed, 73 insertions(+), 3 deletions(-) create mode 100644 src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs new file mode 100644 index 0000000000000..6ad877bc160fa --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; + +namespace System.IO.Tests +{ + // to avoid a lot of code duplication, we reuse FileStream tests + public class File_OpenHandle : FileStream_ctor_options_as + { + protected override FileStream CreateFileStream(string path, FileMode mode) + => new FileStream(File.OpenHandle(path, mode), mode == FileMode.Append ? FileAccess.Write : FileAccess.ReadWrite); + + protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access) + => new FileStream(File.OpenHandle(path, mode, access), access); + + protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options) + => new FileStream(File.OpenHandle(path, mode, access, share, options), access, bufferSize); + + [Fact] + public override void NegativePreallocationSizeThrows() + { + ArgumentOutOfRangeException ex = Assert.Throws( + () => File.OpenHandle("validPath", FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, -1)); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_options_as.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_options_as.cs index a202814d7f256..e319d321b4fb5 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_options_as.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_options_as.cs @@ -67,7 +67,7 @@ public partial class NoParallelTests { } public partial class FileStream_ctor_options_as : FileStream_ctor_options_as_base { [Fact] - public void NegativePreallocationSizeThrows() + public virtual void NegativePreallocationSizeThrows() { string filePath = GetPathToNonExistingFile(); ArgumentOutOfRangeException ex = Assert.Throws( diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs index c8f63b8b32d89..23eed7ff433f6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs @@ -1,19 +1,21 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.Diagnostics.CodeAnalysis; -using System.IO; using System.Runtime.ExceptionServices; using System.Runtime.Versioning; using System.Text; using System.Threading; using System.Threading.Tasks; +using Microsoft.Win32.SafeHandles; #if MS_IO_REDIST +using System; +using System.IO; + namespace Microsoft.IO #else namespace System.IO @@ -1052,5 +1054,45 @@ private static async Task InternalWriteAllTextAsync(StreamWriter sw, string cont ? Task.FromCanceled(cancellationToken) : InternalWriteAllLinesAsync(AsyncStreamWriter(path, encoding, append: true), contents, cancellationToken); } + +#if !MS_IO_REDIST + /// + /// Initializes a new instance of the class with the specified path, creation mode, read/write and sharing permission, the access other SafeFileHandles can have to the same file, additional file options and the allocation size. + /// + /// A relative or absolute path for the file that the current instance will encapsulate. + /// One of the enumeration values that determines how to open or create the file. The default value is + /// A bitwise combination of the enumeration values that determines how the file can be accessed. The default value is + /// A bitwise combination of the enumeration values that determines how the file will be shared by processes. The default value is . + /// The initial allocation size in bytes for the file. A positive value is effective only when a regular file is being created, overwritten, or replaced. + /// Negative values are not allowed. In other cases (including the default 0 value), it's ignored. + /// An object that describes optional parameters to use. + /// is . + /// is an empty string (""), contains only white space, or contains one or more invalid characters. + /// -or- + /// refers to a non-file device, such as CON:, COM1:, LPT1:, etc. in an NTFS environment. + /// refers to a non-file device, such as CON:, COM1:, LPT1:, etc. in a non-NTFS environment. + /// is negative. + /// -or- + /// , , or contain an invalid value. + /// The file cannot be found, such as when is or , and the file specified by does not exist. The file must already exist in these modes. + /// An I/O error, such as specifying when the file specified by already exists, occurred. + /// -or- + /// The disk was full (when was provided and was pointing to a regular file). + /// -or- + /// The file was too large (when was provided and was pointing to a regular file). + /// The caller does not have the required permission. + /// The specified path is invalid, such as being on an unmapped drive. + /// The requested is not permitted by the operating system for the specified , such as when is or and the file or directory is set for read-only access. + /// -or- + /// is specified for , but file encryption is not supported on the current platform. + /// The specified path, file name, or both exceed the system-defined maximum length. + public static SafeFileHandle OpenHandle(string path, FileMode mode = FileMode.Open, FileAccess access = FileAccess.Read, + FileShare share = FileShare.Read, FileOptions options = FileOptions.None, long preallocationSize = 0) + { + Strategies.FileStreamHelpers.ValidateArguments(path, mode, access, share, 0 /* bufferSize */, options, preallocationSize); + + return SafeFileHandle.Open(path, mode, access, share, options, preallocationSize); + } +#endif } } diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 224a13e6abffd..77298430b0884 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -7563,6 +7563,7 @@ public static void Move(string sourceFileName, string destFileName, bool overwri public static System.IO.FileStream Open(string path, System.IO.FileMode mode) { throw null; } public static System.IO.FileStream Open(string path, System.IO.FileMode mode, System.IO.FileAccess access) { throw null; } public static System.IO.FileStream Open(string path, System.IO.FileMode mode, System.IO.FileAccess access, System.IO.FileShare share) { throw null; } + public static Microsoft.Win32.SafeHandles.SafeFileHandle OpenHandle(string path, System.IO.FileMode mode = System.IO.FileMode.Open, System.IO.FileAccess access = System.IO.FileAccess.Read, System.IO.FileShare share = System.IO.FileShare.Read, System.IO.FileOptions options = System.IO.FileOptions.None, long preallocationSize = 0) { throw null; } public static System.IO.FileStream OpenRead(string path) { throw null; } public static System.IO.StreamReader OpenText(string path) { throw null; } public static System.IO.FileStream OpenWrite(string path) { throw null; } From c61d63934e47663b36d631c33315496402cecc2e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 09:24:46 +0200 Subject: [PATCH 04/71] fix the new tests --- .../System.IO.FileSystem/tests/File/OpenHandle.cs | 11 +++++++---- .../tests/System.IO.FileSystem.Tests.csproj | 1 + 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index 6ad877bc160fa..90bd432f1ae2d 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -9,19 +9,22 @@ namespace System.IO.Tests public class File_OpenHandle : FileStream_ctor_options_as { protected override FileStream CreateFileStream(string path, FileMode mode) - => new FileStream(File.OpenHandle(path, mode), mode == FileMode.Append ? FileAccess.Write : FileAccess.ReadWrite); + { + FileAccess access = mode == FileMode.Append ? FileAccess.Write : FileAccess.ReadWrite; + return new FileStream(File.OpenHandle(path, mode, access, preallocationSize: PreallocationSize), access); + } protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access) - => new FileStream(File.OpenHandle(path, mode, access), access); + => new FileStream(File.OpenHandle(path, mode, access, preallocationSize: PreallocationSize), access); protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options) - => new FileStream(File.OpenHandle(path, mode, access, share, options), access, bufferSize); + => new FileStream(File.OpenHandle(path, mode, access, share, options, PreallocationSize), access, bufferSize, (options & FileOptions.Asynchronous) != 0); [Fact] public override void NegativePreallocationSizeThrows() { ArgumentOutOfRangeException ex = Assert.Throws( - () => File.OpenHandle("validPath", FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, -1)); + () => File.OpenHandle("validPath", FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, preallocationSize: -1)); } } } diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index dba6172218ca0..6f6597b8101ea 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -157,6 +157,7 @@ + From ad252deae6d2f5c7e7fb1dfafffc06eb37b9084c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 09:26:09 +0200 Subject: [PATCH 05/71] FileStream created from a handle should allow for bufferSize == 0 --- .../tests/FileStream/ctor_sfh_fa_buffer.cs | 8 +++----- .../System.Private.CoreLib/src/System/IO/FileStream.cs | 2 +- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs index 745ff57850888..ff9db7f4a63d3 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs @@ -18,14 +18,12 @@ protected virtual FileStream CreateFileStream(SafeFileHandle handle, FileAccess return new FileStream(handle, access, bufferSize); } - [Theory, - InlineData(0), - InlineData(-1)] - public void InvalidBufferSize_Throws(int size) + [Fact] + public void NegativeBufferSize_Throws() { using (var handle = new SafeFileHandle(new IntPtr(1), ownsHandle: false)) { - AssertExtensions.Throws("bufferSize", () => CreateFileStream(handle, FileAccess.Read, size)); + AssertExtensions.Throws("bufferSize", () => CreateFileStream(handle, FileAccess.Read, -1)); } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs index de1dceec67d31..af4d973438739 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs @@ -75,7 +75,7 @@ private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int { throw new ArgumentOutOfRangeException(nameof(access), SR.ArgumentOutOfRange_Enum); } - else if (bufferSize <= 0) + else if (bufferSize < 0) { throw new ArgumentOutOfRangeException(nameof(bufferSize), SR.ArgumentOutOfRange_NeedPosNum); } From 1540dbab49d84f9f1dbe41308562a701436d5d8b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 09:26:44 +0200 Subject: [PATCH 06/71] File.OpenHandle should always use full path --- src/libraries/System.Private.CoreLib/src/System/IO/File.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs index 23eed7ff433f6..579abdb0dcac4 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs @@ -1091,7 +1091,7 @@ public static SafeFileHandle OpenHandle(string path, FileMode mode = FileMode.Op { Strategies.FileStreamHelpers.ValidateArguments(path, mode, access, share, 0 /* bufferSize */, options, preallocationSize); - return SafeFileHandle.Open(path, mode, access, share, options, preallocationSize); + return SafeFileHandle.Open(Path.GetFullPath(path), mode, access, share, options, preallocationSize); } #endif } From d8a3113de97d6b75f378c617f279b919ab5c38b1 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 11:10:10 +0200 Subject: [PATCH 07/71] fix a bug in NtCreateFile mapping logic --- .../Windows/NtDll/Interop.NtCreateFile.cs | 4 +-- .../System.IO.FileSystem/tests/File/Create.cs | 32 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs index 9ce82c7f4ad45..04765e4fbd43d 100644 --- a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs +++ b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs @@ -138,9 +138,9 @@ private static DesiredAccess GetDesiredAccess(FileAccess access, FileMode fileMo { result |= DesiredAccess.SYNCHRONIZE; // required by FILE_SYNCHRONOUS_IO_NONALERT } - if ((options & FileOptions.DeleteOnClose) != 0 || fileMode == FileMode.Create) + if ((options & FileOptions.DeleteOnClose) != 0) { - result |= DesiredAccess.DELETE; // required by FILE_DELETE_ON_CLOSE and FILE_SUPERSEDE (which deletes a file if it exists) + result |= DesiredAccess.DELETE; // required by FILE_DELETE_ON_CLOSE } return result; diff --git a/src/libraries/System.IO.FileSystem/tests/File/Create.cs b/src/libraries/System.IO.FileSystem/tests/File/Create.cs index a4a8de349f3bb..c5ce80d742684 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/Create.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/Create.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Linq; using Xunit; namespace System.IO.Tests @@ -51,6 +52,37 @@ public void ValidCreation() } } + [Fact] + public void CreateAndOverwrite() + { + const byte initialContent = 1; + + DirectoryInfo testDir = Directory.CreateDirectory(GetTestFilePath()); + string testFile = Path.Combine(testDir.FullName, GetTestFileName()); + + // Create + using (FileStream stream = Create(testFile)) + { + Assert.True(File.Exists(testFile)); + + stream.WriteByte(initialContent); + + Assert.Equal(1, stream.Length); + Assert.Equal(1, stream.Position); + } + + Assert.Equal(initialContent, File.ReadAllBytes(testFile).Single()); + + // Overwrite + using (FileStream stream = Create(testFile)) + { + Assert.Equal(0, stream.Length); + Assert.Equal(0, stream.Position); + } + + Assert.Empty(File.ReadAllBytes(testFile)); + } + [ConditionalFact(nameof(UsingNewNormalization))] [PlatformSpecific(TestPlatforms.Windows)] // Valid Windows path extended prefix public void ValidCreation_ExtendedSyntax() From 02cd564885e437b08b060d9e1076ee3a41736393 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 11:11:45 +0200 Subject: [PATCH 08/71] always use NtCreateFile for SafeFileHandle.Open --- .../SafeHandles/SafeFileHandle.Windows.cs | 87 +++++-------------- .../System.Private.CoreLib.Shared.projitems | 3 - 2 files changed, 21 insertions(+), 69 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 4eaf14bf2e7ab..dea5744ea2cb0 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -2,9 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; -using System.Diagnostics; using System.IO; -using System.IO.Strategies; using System.Runtime.InteropServices; using System.Text; using System.Threading; @@ -13,72 +11,42 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private bool? _isAsync; - public SafeFileHandle() : base(true) { - _isAsync = null; } public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : base(ownsHandle) { SetHandle(preexistingHandle); - - _isAsync = null; } - internal bool? IsAsync + private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, bool isAsync) : base(ownsHandle) { - get => _isAsync; - set => _isAsync = value; + SetHandle(preexistingHandle); + + IsAsync = isAsync; } + internal bool? IsAsync { get; set; } + internal ThreadPoolBoundHandle? ThreadPoolBinding { get; set; } - protected override bool ReleaseHandle() => - Interop.Kernel32.CloseHandle(handle); + protected override bool ReleaseHandle() => Interop.Kernel32.CloseHandle(handle); internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) { using (DisableMediaInsertionPrompt.Create()) { - Debug.Assert(fullPath != null); - - if (FileStreamHelpers.ShouldPreallocate(preallocationSize, access, mode)) - { - IntPtr fileHandle = NtCreateFile(fullPath, mode, access, share, options, preallocationSize); - - return ValidateFileHandle(new SafeFileHandle(fileHandle, ownsHandle: true), fullPath, (options & FileOptions.Asynchronous) != 0); - } - - Interop.Kernel32.SECURITY_ATTRIBUTES secAttrs = GetSecAttrs(share); - - int fAccess = - ((access & FileAccess.Read) == FileAccess.Read ? Interop.Kernel32.GenericOperations.GENERIC_READ : 0) | - ((access & FileAccess.Write) == FileAccess.Write ? Interop.Kernel32.GenericOperations.GENERIC_WRITE : 0); - - // Our Inheritable bit was stolen from Windows, but should be set in - // the security attributes class. Don't leave this bit set. - share &= ~FileShare.Inheritable; + SafeFileHandle fileHandle = new SafeFileHandle( + NtCreateFile(fullPath, mode, access, share, options, preallocationSize), + ownsHandle: true, + isAsync: (options & FileOptions.Asynchronous) != 0); - // Must use a valid Win32 constant here... - if (mode == FileMode.Append) - mode = FileMode.OpenOrCreate; + fileHandle.Validate(fullPath); - int flagsAndAttributes = (int)options; + fileHandle.Init(); - // For mitigating local elevation of privilege attack through named pipes - // make sure we always call CreateFile with SECURITY_ANONYMOUS so that the - // named pipe server can't impersonate a high privileged client security context - // (note that this is the effective default on CreateFile2) - flagsAndAttributes |= (Interop.Kernel32.SecurityOptions.SECURITY_SQOS_PRESENT | Interop.Kernel32.SecurityOptions.SECURITY_ANONYMOUS); - - SafeFileHandle safeFileHandle = ValidateFileHandle( - Interop.Kernel32.CreateFile(fullPath, fAccess, share, &secAttrs, mode, flagsAndAttributes, IntPtr.Zero), - fullPath, - (options & FileOptions.Asynchronous) != 0); - - return safeFileHandle; + return fileHandle; } } @@ -118,7 +86,7 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, fullPath, preallocationSize)); // NtCreateFile has a bug and it reports STATUS_INVALID_PARAMETER for files // that are too big for the current file system. Example: creating a 4GB+1 file on a FAT32 drive. - case Interop.NtDll.NT_STATUS_INVALID_PARAMETER: + case Interop.NtDll.NT_STATUS_INVALID_PARAMETER when preallocationSize > 0: case Interop.NtDll.NT_ERROR_STATUS_FILE_TOO_LARGE: throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, fullPath, preallocationSize)); default: @@ -127,23 +95,9 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac } } - private static unsafe Interop.Kernel32.SECURITY_ATTRIBUTES GetSecAttrs(FileShare share) + private void Validate(string path) { - Interop.Kernel32.SECURITY_ATTRIBUTES secAttrs = default; - if ((share & FileShare.Inheritable) != 0) - { - secAttrs = new Interop.Kernel32.SECURITY_ATTRIBUTES - { - nLength = (uint)sizeof(Interop.Kernel32.SECURITY_ATTRIBUTES), - bInheritHandle = Interop.BOOL.TRUE - }; - } - return secAttrs; - } - - private static SafeFileHandle ValidateFileHandle(SafeFileHandle fileHandle, string path, bool useAsyncIO) - { - if (fileHandle.IsInvalid) + if (IsInvalid) { // Return a meaningful exception with the full path. @@ -152,14 +106,15 @@ private static SafeFileHandle ValidateFileHandle(SafeFileHandle fileHandle, stri // probably be consistent w/ every other directory. int errorCode = Marshal.GetLastPInvokeError(); - if (errorCode == Interop.Errors.ERROR_PATH_NOT_FOUND && path!.Length == PathInternal.GetRootLength(path)) + if (errorCode == Interop.Errors.ERROR_PATH_NOT_FOUND && path.Length == PathInternal.GetRootLength(path)) + { errorCode = Interop.Errors.ERROR_ACCESS_DENIED; + } throw Win32Marshal.GetExceptionForWin32Error(errorCode, path); } + } - fileHandle.IsAsync = useAsyncIO; - return fileHandle; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index bbbd804c83e6f..69a6cbd913d20 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1610,9 +1610,6 @@ Common\Interop\Windows\Interop.OBJECT_ATTRIBUTES.cs - - Common\Interop\Windows\Kernel32\Interop.SecurityOptions.cs - Common\Interop\Windows\Kernel32\Interop.SetCurrentDirectory.cs From 1430dd69f7e630c6fa8cb02deb9f131c84852370 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 11:13:24 +0200 Subject: [PATCH 09/71] move the ThreadPoolBinding initialization logic to one place --- .../SafeHandles/SafeFileHandle.Windows.cs | 31 ++++++++- .../AsyncWindowsFileStreamStrategy.cs | 57 ----------------- .../Net5CompatFileStreamStrategy.Unix.cs | 8 +-- .../Net5CompatFileStreamStrategy.Windows.cs | 63 +------------------ .../Net5CompatFileStreamStrategy.cs | 2 +- .../Strategies/WindowsFileStreamStrategy.cs | 8 +-- 6 files changed, 36 insertions(+), 133 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index dea5744ea2cb0..1fcac77d9bfe6 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -44,7 +44,7 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA fileHandle.Validate(fullPath); - fileHandle.Init(); + fileHandle.InitThreadPoolBindingIfNeeded(); return fileHandle; } @@ -115,6 +115,35 @@ private void Validate(string path) } } + internal void InitThreadPoolBindingIfNeeded() + { + if (IsAsync == true && ThreadPoolBinding == null) + { + // This is necessary for async IO using IO Completion ports via our + // managed Threadpool API's. This (theoretically) calls the OS's + // BindIoCompletionCallback method, and passes in a stub for the + // LPOVERLAPPED_COMPLETION_ROUTINE. This stub looks at the Overlapped + // struct for this request and gets a delegate to a managed callback + // from there, which it then calls on a threadpool thread. (We allocate + // our native OVERLAPPED structs 2 pointers too large and store EE state + // & GC handles there, one to an IAsyncResult, the other to a delegate.) + try + { + ThreadPoolBinding = ThreadPoolBoundHandle.BindHandle(this); + } + catch (ArgumentException ex) + { + throw new IOException(SR.IO_BindHandleFailed, ex); + } + finally + { + if (ThreadPoolBinding == null) + { + // We should close the handle so that the handle is not open until SafeFileHandle GC + Dispose(); + } + } + } } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs index 6719dd9389566..47e4025c11b8e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs @@ -45,63 +45,6 @@ protected override void Dispose(bool disposing) Interlocked.Exchange(ref _reusableValueTaskSource, null)?.Dispose(); } - protected override void OnInitFromHandle(SafeFileHandle handle) - { - // This is necessary for async IO using IO Completion ports via our - // managed Threadpool API's. This calls the OS's - // BindIoCompletionCallback method, and passes in a stub for the - // LPOVERLAPPED_COMPLETION_ROUTINE. This stub looks at the Overlapped - // struct for this request and gets a delegate to a managed callback - // from there, which it then calls on a threadpool thread. (We allocate - // our native OVERLAPPED structs 2 pointers too large and store EE - // state & a handle to a delegate there.) - // - // If, however, we've already bound this file handle to our completion port, - // don't try to bind it again because it will fail. A handle can only be - // bound to a single completion port at a time. - if (handle.IsAsync != true) - { - try - { - handle.ThreadPoolBinding = ThreadPoolBoundHandle.BindHandle(handle); - } - catch (Exception ex) - { - // If you passed in a synchronous handle and told us to use - // it asynchronously, throw here. - throw new ArgumentException(SR.Arg_HandleNotAsync, nameof(handle), ex); - } - } - } - - protected override void OnInit() - { - // This is necessary for async IO using IO Completion ports via our - // managed Threadpool API's. This (theoretically) calls the OS's - // BindIoCompletionCallback method, and passes in a stub for the - // LPOVERLAPPED_COMPLETION_ROUTINE. This stub looks at the Overlapped - // struct for this request and gets a delegate to a managed callback - // from there, which it then calls on a threadpool thread. (We allocate - // our native OVERLAPPED structs 2 pointers too large and store EE state - // & GC handles there, one to an IAsyncResult, the other to a delegate.) - try - { - _fileHandle.ThreadPoolBinding = ThreadPoolBoundHandle.BindHandle(_fileHandle); - } - catch (ArgumentException ex) - { - throw new IOException(SR.IO_BindHandleFailed, ex); - } - finally - { - if (_fileHandle.ThreadPoolBinding == null) - { - // We should close the handle so that the handle is not open until SafeFileHandle GC - _fileHandle.Dispose(); - } - } - } - private void TryToReuse(ValueTaskSource source) { source._source.Reset(); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs index 848b240c7ae05..cc62ccc1306cf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs @@ -34,13 +34,7 @@ internal sealed partial class Net5CompatFileStreamStrategy : FileStreamStrategy /// Lazily-initialized value for whether the file supports seeking. private bool? _canSeek; - /// Initializes a stream for reading or writing a Unix file. - /// How the file should be opened. - /// What other access to the file should be allowed. This is currently ignored. - /// The original path specified for the FileStream. - /// Options, passed via arguments as we have no guarantee that _options field was already set. - /// passed to posix_fallocate - private void Init(FileMode mode, FileShare share, string originalPath, FileOptions options, long preallocationSize) + private void Init(FileMode mode, string originalPath, FileOptions options) { // FileStream performs most of the general argument validation. We can assume here that the arguments // are all checked and consistent (e.g. non-null-or-empty path; valid enums in mode, access, share, and options; etc.) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs index ceb9bcc6e4052..577f50832e871 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs @@ -46,40 +46,11 @@ internal sealed partial class Net5CompatFileStreamStrategy : FileStreamStrategy private PreAllocatedOverlapped? _preallocatedOverlapped; // optimization for async ops to avoid per-op allocations private CompletionSource? _currentOverlappedOwner; // async op currently using the preallocated overlapped - private void Init(FileMode mode, FileShare share, string originalPath, FileOptions options, long preallocationSize) + private void Init(FileMode mode, string originalPath, FileOptions options) { FileStreamHelpers.ValidateFileTypeForNonExtendedPaths(_fileHandle, originalPath); - // This is necessary for async IO using IO Completion ports via our - // managed Threadpool API's. This (theoretically) calls the OS's - // BindIoCompletionCallback method, and passes in a stub for the - // LPOVERLAPPED_COMPLETION_ROUTINE. This stub looks at the Overlapped - // struct for this request and gets a delegate to a managed callback - // from there, which it then calls on a threadpool thread. (We allocate - // our native OVERLAPPED structs 2 pointers too large and store EE state - // & GC handles there, one to an IAsyncResult, the other to a delegate.) - if (_useAsyncIO) - { - try - { - _fileHandle.ThreadPoolBinding = ThreadPoolBoundHandle.BindHandle(_fileHandle); - } - catch (ArgumentException ex) - { - throw new IOException(SR.IO_BindHandleFailed, ex); - } - finally - { - if (_fileHandle.ThreadPoolBinding == null) - { - // We should close the handle so that the handle is not open until SafeFileHandle GC - Debug.Assert(!_exposedHandle, "Are we closing handle that we exposed/not own, how?"); - _fileHandle.Dispose(); - } - } - } - - _canSeek = true; + Debug.Assert(!_useAsyncIO || _fileHandle.ThreadPoolBinding != null); // For Append mode... if (mode == FileMode.Append) @@ -115,35 +86,7 @@ private void InitFromHandleImpl(SafeFileHandle handle, bool useAsyncIO) { FileStreamHelpers.GetFileTypeSpecificInformation(handle, out _canSeek, out _isPipe); - // This is necessary for async IO using IO Completion ports via our - // managed Threadpool API's. This calls the OS's - // BindIoCompletionCallback method, and passes in a stub for the - // LPOVERLAPPED_COMPLETION_ROUTINE. This stub looks at the Overlapped - // struct for this request and gets a delegate to a managed callback - // from there, which it then calls on a threadpool thread. (We allocate - // our native OVERLAPPED structs 2 pointers too large and store EE - // state & a handle to a delegate there.) - // - // If, however, we've already bound this file handle to our completion port, - // don't try to bind it again because it will fail. A handle can only be - // bound to a single completion port at a time. - if (useAsyncIO && !(handle.IsAsync ?? false)) - { - try - { - handle.ThreadPoolBinding = ThreadPoolBoundHandle.BindHandle(handle); - } - catch (Exception ex) - { - // If you passed in a synchronous handle and told us to use - // it asynchronously, throw here. - throw new ArgumentException(SR.Arg_HandleNotAsync, nameof(handle), ex); - } - } - else if (!useAsyncIO) - { - FileStreamHelpers.VerifyHandleIsSync(handle); - } + handle.InitThreadPoolBindingIfNeeded(); if (_canSeek) SeekCore(handle, 0, SeekOrigin.Current); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs index 1ff5b1c81c9a9..9a88f5a65d489 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.cs @@ -93,7 +93,7 @@ internal Net5CompatFileStreamStrategy(string path, FileMode mode, FileAccess acc try { - Init(mode, share, path, options, preallocationSize); + Init(mode, path, options); } catch { diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs index 7fba6bb32a8d6..be5b0b6a514ca 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs @@ -226,16 +226,10 @@ public sealed override long Seek(long offset, SeekOrigin origin) internal sealed override void Unlock(long position, long length) => FileStreamHelpers.Unlock(_fileHandle, _path, position, length); - protected abstract void OnInitFromHandle(SafeFileHandle handle); - - protected virtual void OnInit() { } - private void Init(FileMode mode, string originalPath) { FileStreamHelpers.ValidateFileTypeForNonExtendedPaths(_fileHandle, originalPath); - OnInit(); - // For Append mode... if (mode == FileMode.Append) { @@ -270,7 +264,7 @@ private void InitFromHandleImpl(SafeFileHandle handle, out bool canSeek, out boo { FileStreamHelpers.GetFileTypeSpecificInformation(handle, out canSeek, out isPipe); - OnInitFromHandle(handle); + handle.InitThreadPoolBindingIfNeeded(); if (_canSeek) { From ebaa92a95ed7c039ccc315fc3cf88c0fa7b90c19 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 15:25:33 +0200 Subject: [PATCH 10/71] IsAsync: use syscall when not provided and validate it early --- .../tests/FileStream/ctor_sfh_fa.cs | 4 +- .../tests/FileStream/ctor_sfh_fa_buffer.cs | 4 +- .../SafeHandles/SafeFileHandle.Windows.cs | 44 +++++++++++++- .../src/System/IO/FileStream.cs | 25 +++++--- .../Strategies/FileStreamHelpers.Windows.cs | 57 ------------------- .../SyncWindowsFileStreamStrategy.cs | 14 ----- 6 files changed, 64 insertions(+), 84 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa.cs index 49881c43dbab1..5ad232a6d3a94 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa.cs @@ -25,7 +25,7 @@ public void InvalidHandle_Throws() [Fact] public void InvalidAccess_Throws() { - using (var handle = new SafeFileHandle(new IntPtr(1), ownsHandle: false)) + using (var handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write)) { AssertExtensions.Throws("access", () => CreateFileStream(handle, ~FileAccess.Read)); } @@ -34,7 +34,7 @@ public void InvalidAccess_Throws() [Fact] public void InvalidAccess_DoesNotCloseHandle() { - using (var handle = new SafeFileHandle(new IntPtr(1), ownsHandle: false)) + using (var handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write)) { Assert.Throws(() => CreateFileStream(handle, ~FileAccess.Read)); GC.Collect(); diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs index ff9db7f4a63d3..0c480d4a4fcb6 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer.cs @@ -21,7 +21,7 @@ protected virtual FileStream CreateFileStream(SafeFileHandle handle, FileAccess [Fact] public void NegativeBufferSize_Throws() { - using (var handle = new SafeFileHandle(new IntPtr(1), ownsHandle: false)) + using (var handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write)) { AssertExtensions.Throws("bufferSize", () => CreateFileStream(handle, FileAccess.Read, -1)); } @@ -30,7 +30,7 @@ public void NegativeBufferSize_Throws() [Fact] public void InvalidBufferSize_DoesNotCloseHandle() { - using (var handle = new SafeFileHandle(new IntPtr(1), ownsHandle: false)) + using (var handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write)) { Assert.Throws(() => CreateFileStream(handle, FileAccess.Read, -1)); GC.Collect(); diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 1fcac77d9bfe6..7887a912ed872 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -11,23 +11,44 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { + private bool? _isAsync; + public SafeFileHandle() : base(true) { + _isAsync = null; } public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : base(ownsHandle) { SetHandle(preexistingHandle); + + _isAsync = null; } private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, bool isAsync) : base(ownsHandle) { SetHandle(preexistingHandle); - IsAsync = isAsync; + _isAsync = isAsync; } - internal bool? IsAsync { get; set; } + public bool IsAsync + { + get + { + if (IsInvalid) + { + throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); + } + + if (_isAsync == null) + { + _isAsync = !IsHandleSynchronous(); + } + + return _isAsync.Value; + } + } internal ThreadPoolBoundHandle? ThreadPoolBinding { get; set; } @@ -145,5 +166,24 @@ internal void InitThreadPoolBindingIfNeeded() } } } + + private unsafe bool IsHandleSynchronous() + { + uint fileMode; + int status = Interop.NtDll.NtQueryInformationFile( + FileHandle: this, + IoStatusBlock: out _, + FileInformation: &fileMode, + Length: sizeof(uint), + FileInformationClass: Interop.NtDll.FileModeInformation); + + if (status != 0) + { + throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); + } + + // If either of these two flags are set, the file handle is synchronous (not overlapped) + return (fileMode & (uint)(Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_ALERT | Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_NONALERT)) > 0; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs index af4d973438739..1a149d6f295ad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs @@ -3,7 +3,6 @@ using System.ComponentModel; using System.IO.Strategies; -using System.Runtime.Serialization; using System.Runtime.Versioning; using System.Threading; using System.Threading.Tasks; @@ -22,21 +21,21 @@ public class FileStream : Stream [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This constructor has been deprecated. Please use new FileStream(SafeFileHandle handle, FileAccess access) instead. https://go.microsoft.com/fwlink/?linkid=14202")] public FileStream(IntPtr handle, FileAccess access) - : this(handle, access, true, DefaultBufferSize, false) + : this(handle, access, true, DefaultBufferSize, DefaultIsAsync) { } [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This constructor has been deprecated. Please use new FileStream(SafeFileHandle handle, FileAccess access) instead, and optionally make a new SafeFileHandle with ownsHandle=false if needed. https://go.microsoft.com/fwlink/?linkid=14202")] public FileStream(IntPtr handle, FileAccess access, bool ownsHandle) - : this(handle, access, ownsHandle, DefaultBufferSize, false) + : this(handle, access, ownsHandle, DefaultBufferSize, DefaultIsAsync) { } [EditorBrowsable(EditorBrowsableState.Never)] [Obsolete("This constructor has been deprecated. Please use new FileStream(SafeFileHandle handle, FileAccess access, int bufferSize) instead, and optionally make a new SafeFileHandle with ownsHandle=false if needed. https://go.microsoft.com/fwlink/?linkid=14202")] public FileStream(IntPtr handle, FileAccess access, bool ownsHandle, int bufferSize) - : this(handle, access, ownsHandle, bufferSize, false) + : this(handle, access, ownsHandle, bufferSize, DefaultIsAsync) { } @@ -65,7 +64,7 @@ public FileStream(IntPtr handle, FileAccess access, bool ownsHandle, int bufferS } } - private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int bufferSize, bool isAsync) + private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int bufferSize) { if (handle.IsInvalid) { @@ -83,10 +82,20 @@ private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int { ThrowHelper.ThrowObjectDisposedException_FileClosed(); } - else if (handle.IsAsync.HasValue && isAsync != handle.IsAsync.GetValueOrDefault()) + } + + private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int bufferSize, bool isAsync) + { + ValidateHandle(handle, access, bufferSize); + + if (isAsync && !handle.IsAsync) { throw new ArgumentException(SR.Arg_HandleNotAsync, nameof(handle)); } + else if (!isAsync && handle.IsAsync) + { + ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); + } } public FileStream(SafeFileHandle handle, FileAccess access) @@ -95,8 +104,10 @@ public FileStream(SafeFileHandle handle, FileAccess access) } public FileStream(SafeFileHandle handle, FileAccess access, int bufferSize) - : this(handle, access, bufferSize, FileStreamHelpers.GetDefaultIsAsync(handle, DefaultIsAsync)) { + ValidateHandle(handle, access, bufferSize); + + _strategy = FileStreamHelpers.ChooseStrategy(this, handle, access, DefaultShare, bufferSize, handle.IsAsync); } public FileStream(SafeFileHandle handle, FileAccess access, int bufferSize, bool isAsync) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs index d1e14c3d49756..47a5ddf0baa1d 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs @@ -60,63 +60,6 @@ private static FileStreamStrategy ChooseStrategyCore(string path, FileMode mode, internal static FileStreamStrategy EnableBufferingIfNeeded(WindowsFileStreamStrategy strategy, int bufferSize) => bufferSize > 1 ? new BufferedFileStreamStrategy(strategy, bufferSize) : strategy; - internal static bool GetDefaultIsAsync(SafeFileHandle handle, bool defaultIsAsync) - { - return handle.IsAsync ?? !IsHandleSynchronous(handle, ignoreInvalid: true) ?? defaultIsAsync; - } - - internal static unsafe bool? IsHandleSynchronous(SafeFileHandle fileHandle, bool ignoreInvalid) - { - if (fileHandle.IsInvalid) - return null; - - uint fileMode; - - int status = Interop.NtDll.NtQueryInformationFile( - FileHandle: fileHandle, - IoStatusBlock: out _, - FileInformation: &fileMode, - Length: sizeof(uint), - FileInformationClass: Interop.NtDll.FileModeInformation); - - switch (status) - { - case 0: - // We were successful - break; - case Interop.NtDll.STATUS_INVALID_HANDLE: - if (!ignoreInvalid) - { - throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); - } - else - { - return null; - } - default: - // Something else is preventing access - Debug.Fail("Unable to get the file mode information, status was" + status.ToString()); - return null; - } - - // If either of these two flags are set, the file handle is synchronous (not overlapped) - return (fileMode & (uint)(Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_ALERT | Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_NONALERT)) > 0; - } - - internal static void VerifyHandleIsSync(SafeFileHandle handle) - { - // As we can accurately check the handle type when we have access to NtQueryInformationFile we don't need to skip for - // any particular file handle type. - - // If the handle was passed in without an explicit async setting, we already looked it up in GetDefaultIsAsync - if (!handle.IsAsync.HasValue) - return; - - // If we can't check the handle, just assume it is ok. - if (!(IsHandleSynchronous(handle, ignoreInvalid: false) ?? true)) - ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); - } - internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) { Interop.Kernel32.FILE_STANDARD_INFO info; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs index 30499b660ddf0..37ac4ae622321 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs @@ -22,20 +22,6 @@ internal SyncWindowsFileStreamStrategy(string path, FileMode mode, FileAccess ac internal override bool IsAsync => false; - protected override void OnInitFromHandle(SafeFileHandle handle) - { - // As we can accurately check the handle type when we have access to NtQueryInformationFile we don't need to skip for - // any particular file handle type. - - // If the handle was passed in without an explicit async setting, we already looked it up in GetDefaultIsAsync - if (!handle.IsAsync.HasValue) - return; - - // If we can't check the handle, just assume it is ok. - if (!(FileStreamHelpers.IsHandleSynchronous(handle, ignoreInvalid: false) ?? true)) - ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); - } - public override int Read(byte[] buffer, int offset, int count) => ReadSpan(new Span(buffer, offset, count)); public override int Read(Span buffer) => ReadSpan(buffer); From fbbcd4d3c9c753bcf77a81ec9165f54cfbc608ac Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 15:47:22 +0200 Subject: [PATCH 11/71] read all FileOptions, not just FileOptions.Asynchronous --- .../NtDll/Interop.NtQueryInformationFile.cs | 1 - .../SafeHandles/SafeFileHandle.Windows.cs | 55 ++++++++++++++----- 2 files changed, 40 insertions(+), 16 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtQueryInformationFile.cs b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtQueryInformationFile.cs index 644315f2bc2cc..402443b05e944 100644 --- a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtQueryInformationFile.cs +++ b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtQueryInformationFile.cs @@ -2,7 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.Win32.SafeHandles; -using System; using System.Runtime.InteropServices; internal static partial class Interop diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 7887a912ed872..08415a146620e 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -11,25 +11,24 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private bool? _isAsync; + private const FileOptions Unknown = (FileOptions)(-1); + private const FileOptions NoBuffering = (FileOptions)(0x20000000); + private FileOptions _fileOptions = Unknown; public SafeFileHandle() : base(true) { - _isAsync = null; } public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : base(ownsHandle) { SetHandle(preexistingHandle); - - _isAsync = null; } - private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, bool isAsync) : base(ownsHandle) + private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fileOptions) : base(ownsHandle) { SetHandle(preexistingHandle); - _isAsync = isAsync; + _fileOptions = fileOptions; } public bool IsAsync @@ -41,12 +40,12 @@ public bool IsAsync throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); } - if (_isAsync == null) + if (_fileOptions == Unknown) { - _isAsync = !IsHandleSynchronous(); + _fileOptions = GetFileOptions(); } - return _isAsync.Value; + return (_fileOptions & FileOptions.Asynchronous) != 0; } } @@ -61,7 +60,7 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA SafeFileHandle fileHandle = new SafeFileHandle( NtCreateFile(fullPath, mode, access, share, options, preallocationSize), ownsHandle: true, - isAsync: (options & FileOptions.Asynchronous) != 0); + options); fileHandle.Validate(fullPath); @@ -167,13 +166,13 @@ internal void InitThreadPoolBindingIfNeeded() } } - private unsafe bool IsHandleSynchronous() + private unsafe FileOptions GetFileOptions() { - uint fileMode; + Interop.NtDll.CreateOptions options; int status = Interop.NtDll.NtQueryInformationFile( FileHandle: this, IoStatusBlock: out _, - FileInformation: &fileMode, + FileInformation: &options, Length: sizeof(uint), FileInformationClass: Interop.NtDll.FileModeInformation); @@ -182,8 +181,34 @@ private unsafe bool IsHandleSynchronous() throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); } - // If either of these two flags are set, the file handle is synchronous (not overlapped) - return (fileMode & (uint)(Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_ALERT | Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_NONALERT)) > 0; + FileOptions result = FileOptions.None; + + if ((options & (Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_ALERT | Interop.NtDll.CreateOptions.FILE_SYNCHRONOUS_IO_NONALERT)) == 0) + { + result |= FileOptions.Asynchronous; + } + if ((options & Interop.NtDll.CreateOptions.FILE_WRITE_THROUGH) != 0) + { + result |= FileOptions.WriteThrough; + } + if ((options & Interop.NtDll.CreateOptions.FILE_RANDOM_ACCESS) != 0) + { + result |= FileOptions.RandomAccess; + } + if ((options & Interop.NtDll.CreateOptions.FILE_SEQUENTIAL_ONLY) != 0) + { + result |= FileOptions.SequentialScan; + } + if ((options & Interop.NtDll.CreateOptions.FILE_DELETE_ON_CLOSE) != 0) + { + result |= FileOptions.DeleteOnClose; + } + if ((options & Interop.NtDll.CreateOptions.FILE_NO_INTERMEDIATE_BUFFERING) != 0) + { + result |= NoBuffering; + } + + return result; } } } From 7f94f8548425d021a99c983634f2a25246bacfb3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 18:18:07 +0200 Subject: [PATCH 12/71] move IsPipe CanSeek to SafeFileHandle as well (the new APIs like RandomFile.ReadAtOffset are going to need it) --- .../SafeHandles/SafeFileHandle.Windows.cs | 48 +++++++++------ .../Strategies/FileStreamHelpers.Windows.cs | 12 ---- .../Net5CompatFileStreamStrategy.Windows.cs | 5 +- .../Strategies/WindowsFileStreamStrategy.cs | 59 +++++-------------- 4 files changed, 50 insertions(+), 74 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 08415a146620e..7f29553fd9e82 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System; +using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; using System.Text; @@ -14,6 +15,7 @@ public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid private const FileOptions Unknown = (FileOptions)(-1); private const FileOptions NoBuffering = (FileOptions)(0x20000000); private FileOptions _fileOptions = Unknown; + private int _fileType = -1; public SafeFileHandle() : base(true) { @@ -24,30 +26,19 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : base(ownsHand SetHandle(preexistingHandle); } - private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fileOptions) : base(ownsHandle) + private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fileOptions, int fileType) : base(ownsHandle) { SetHandle(preexistingHandle); _fileOptions = fileOptions; + _fileType = fileType; } - public bool IsAsync - { - get - { - if (IsInvalid) - { - throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); - } + public bool IsAsync => (GetFileOptions() & FileOptions.Asynchronous) != 0; - if (_fileOptions == Unknown) - { - _fileOptions = GetFileOptions(); - } + internal bool CanSeek => GetFileType() == Interop.Kernel32.FileTypes.FILE_TYPE_DISK; - return (_fileOptions & FileOptions.Asynchronous) != 0; - } - } + internal bool IsPipe => GetFileType() == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE; internal ThreadPoolBoundHandle? ThreadPoolBinding { get; set; } @@ -60,7 +51,8 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA SafeFileHandle fileHandle = new SafeFileHandle( NtCreateFile(fullPath, mode, access, share, options, preallocationSize), ownsHandle: true, - options); + options, + Interop.Kernel32.FileTypes.FILE_TYPE_DISK); // similarly to FileStream, we assume that only disk files can be referenced by path on Windows fileHandle.Validate(fullPath); @@ -168,6 +160,11 @@ internal void InitThreadPoolBindingIfNeeded() private unsafe FileOptions GetFileOptions() { + if (_fileOptions != (FileOptions)(-1)) + { + return _fileOptions; + } + Interop.NtDll.CreateOptions options; int status = Interop.NtDll.NtQueryInformationFile( FileHandle: this, @@ -208,7 +205,22 @@ private unsafe FileOptions GetFileOptions() result |= NoBuffering; } - return result; + return _fileOptions = result; + } + + private int GetFileType() + { + if (_fileType != -1) + { + _fileType = Interop.Kernel32.GetFileType(this); + + Debug.Assert(_fileType == Interop.Kernel32.FileTypes.FILE_TYPE_DISK + || _fileType == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE + || _fileType == Interop.Kernel32.FileTypes.FILE_TYPE_CHAR, + "Unknown file type!"); + } + + return _fileType; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs index 47a5ddf0baa1d..2d7822a04b1ca 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs @@ -180,18 +180,6 @@ internal static void ValidateFileTypeForNonExtendedPaths(SafeFileHandle handle, } } - internal static void GetFileTypeSpecificInformation(SafeFileHandle handle, out bool canSeek, out bool isPipe) - { - int handleType = Interop.Kernel32.GetFileType(handle); - Debug.Assert(handleType == Interop.Kernel32.FileTypes.FILE_TYPE_DISK - || handleType == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE - || handleType == Interop.Kernel32.FileTypes.FILE_TYPE_CHAR, - "FileStream was passed an unknown file type!"); - - canSeek = handleType == Interop.Kernel32.FileTypes.FILE_TYPE_DISK; - isPipe = handleType == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE; - } - internal static unsafe void SetFileLength(SafeFileHandle handle, string? path, long length) { var eofInfo = new Interop.Kernel32.FILE_END_OF_FILE_INFO diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs index 577f50832e871..b0263e15e2694 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs @@ -61,6 +61,8 @@ private void Init(FileMode mode, string originalPath, FileOptions options) { _appendStart = -1; } + + _canSeek = true; } private void InitFromHandle(SafeFileHandle handle, FileAccess access, bool useAsyncIO) @@ -84,7 +86,8 @@ private void InitFromHandle(SafeFileHandle handle, FileAccess access, bool useAs private void InitFromHandleImpl(SafeFileHandle handle, bool useAsyncIO) { - FileStreamHelpers.GetFileTypeSpecificInformation(handle, out _canSeek, out _isPipe); + _canSeek = handle.CanSeek; + _isPipe = handle.IsPipe; handle.InitThreadPoolBindingIfNeeded(); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs index be5b0b6a514ca..c0f74e671c71b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs @@ -24,16 +24,26 @@ internal abstract class WindowsFileStreamStrategy : FileStreamStrategy internal WindowsFileStreamStrategy(SafeFileHandle handle, FileAccess access, FileShare share) { - InitFromHandle(handle, access, out _canSeek, out _isPipe); - - // Note: Cleaner to set the following fields in ValidateAndInitFromHandle, - // but we can't as they're readonly. _access = access; _share = share; _exposedHandle = true; - // As the handle was passed in, we must set the handle field at the very end to - // avoid the finalizer closing the handle when we throw errors. + _canSeek = handle.CanSeek; + _isPipe = handle.IsPipe; + + handle.InitThreadPoolBindingIfNeeded(); + + if (_canSeek) + { + // given strategy was created out of existing handle, so we have to perform + // a syscall to get the current handle offset + _filePosition = FileStreamHelpers.Seek(handle, _path, 0, SeekOrigin.Current); + } + else + { + _filePosition = 0; + } + _fileHandle = handle; } @@ -241,43 +251,6 @@ private void Init(FileMode mode, string originalPath) } } - private void InitFromHandle(SafeFileHandle handle, FileAccess access, out bool canSeek, out bool isPipe) - { -#if DEBUG - bool hadBinding = handle.ThreadPoolBinding != null; - - try - { -#endif - InitFromHandleImpl(handle, out canSeek, out isPipe); -#if DEBUG - } - catch - { - Debug.Assert(hadBinding || handle.ThreadPoolBinding == null, "We should never error out with a ThreadPoolBinding we've added"); - throw; - } -#endif - } - - private void InitFromHandleImpl(SafeFileHandle handle, out bool canSeek, out bool isPipe) - { - FileStreamHelpers.GetFileTypeSpecificInformation(handle, out canSeek, out isPipe); - - handle.InitThreadPoolBindingIfNeeded(); - - if (_canSeek) - { - // given strategy was created out of existing handle, so we have to perform - // a syscall to get the current handle offset - _filePosition = FileStreamHelpers.Seek(handle, _path, 0, SeekOrigin.Current); - } - else - { - _filePosition = 0; - } - } - public sealed override void SetLength(long value) { if (_appendStart != -1 && value < _appendStart) From 33ecb8d95daddce4fe7ec55fd3ae1170dd38000d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 18:18:28 +0200 Subject: [PATCH 13/71] disable the Append test as it's currently not supported by design --- .../System.IO.FileSystem/tests/File/OpenHandle.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index 90bd432f1ae2d..3ff99f0194959 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -26,5 +26,13 @@ public override void NegativePreallocationSizeThrows() ArgumentOutOfRangeException ex = Assert.Throws( () => File.OpenHandle("validPath", FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, preallocationSize: -1)); } + + [Theory, MemberData(nameof(StreamSpecifiers))] + public override void FileModeAppendExisting(string streamSpecifier) + { + // currently not enabled due to https://github.com/dotnet/runtime/issues/53432 + GC.KeepAlive(streamSpecifier); // to keep the xUnit analyser happy + } + } } } From f1af8f6730f83bf220b725329094f9974bd7dab0 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 28 May 2021 18:38:13 +0200 Subject: [PATCH 14/71] expose SafeFileHandle.IsAsync and write a test for it --- .../tests/File/OpenHandle.cs | 19 +++++++++++++++++++ .../System.Runtime/ref/System.Runtime.cs | 1 + 2 files changed, 20 insertions(+) diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index 3ff99f0194959..859e6d0df5310 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.Win32.SafeHandles; using Xunit; namespace System.IO.Tests @@ -33,6 +34,24 @@ public override void FileModeAppendExisting(string streamSpecifier) // currently not enabled due to https://github.com/dotnet/runtime/issues/53432 GC.KeepAlive(streamSpecifier); // to keep the xUnit analyser happy } + + [Theory] + [InlineData(FileOptions.None)] + [InlineData(FileOptions.Asynchronous)] + public void SafeFileHandle_IsAsync_ReturnsCorrectInformation(FileOptions options) + { + using (var handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write, options: options)) + { + Assert.Equal((options & FileOptions.Asynchronous) != 0, handle.IsAsync); + + // the following code exercises the code path where we don't know FileOptions used for opening the handle + // and instead we ask the OS about it + if (OperatingSystem.IsWindows()) // async file handles are a Windows concept + { + SafeFileHandle createdFromIntPtr = new SafeFileHandle(handle.DangerousGetHandle(), ownsHandle: false); + Assert.Equal((options & FileOptions.Asynchronous) != 0, createdFromIntPtr.IsAsync); + } + } } } } diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 77298430b0884..0faf10fbc23ab 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -21,6 +21,7 @@ public sealed partial class SafeFileHandle : Microsoft.Win32.SafeHandles.SafeHan public SafeFileHandle() : base (default(bool)) { } public SafeFileHandle(System.IntPtr preexistingHandle, bool ownsHandle) : base (default(bool)) { } public override bool IsInvalid { get { throw null; } } + public bool IsAsync { get { throw null; } } protected override bool ReleaseHandle() { throw null; } } public abstract partial class SafeHandleMinusOneIsInvalid : System.Runtime.InteropServices.SafeHandle From a46da914e9e49aeaa5b014387573fb548cf97d73 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 10:10:32 +0200 Subject: [PATCH 15/71] fix compilation errors on Linux --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 116 +++++++++--------- .../SafeHandles/SafeFileHandle.Windows.cs | 3 +- .../IO/Strategies/FileStreamHelpers.Unix.cs | 19 ++- .../Net5CompatFileStreamStrategy.Unix.cs | 61 ++------- 4 files changed, 88 insertions(+), 111 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 81a870510be57..e6bc5111bad41 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -4,15 +4,14 @@ using System; using System.Diagnostics; using System.IO; -using System.Runtime.InteropServices; -using System.Runtime.Versioning; -using System.Threading; -using System.Threading.Tasks; +using System.IO.Strategies; namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { + private bool? _canSeek; + public SafeFileHandle() : this(ownsHandle: true) { } @@ -28,9 +27,45 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : this(ownsHand SetHandle(preexistingHandle); } - internal bool? IsAsync { get; set; } + public bool IsAsync { get; private set; } + + internal bool CanSeek => !IsClosed && (_canSeek ??= Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + + // Each thread will have its own copy. This prevents race conditions if the handle had the last error. + [ThreadStatic] + internal static Interop.ErrorInfo? t_lastCloseErrorInfo; - internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) + protected override bool ReleaseHandle() + { + // When the SafeFileHandle was opened, we likely issued an flock on the created descriptor in order to add + // an advisory lock. This lock should be removed via closing the file descriptor, but close can be + // interrupted, and we don't retry closes. As such, we could end up leaving the file locked, + // which could prevent subsequent usage of the file until this process dies. To avoid that, we proactively + // try to release the lock before we close the handle. (If it's not locked, there's no behavioral + // problem trying to unlock it.) + Interop.Sys.FLock(handle, Interop.Sys.LockOperations.LOCK_UN); // ignore any errors + + // Close the descriptor. Although close is documented to potentially fail with EINTR, we never want + // to retry, as the descriptor could actually have been closed, been subsequently reassigned, and + // be in use elsewhere in the process. Instead, we simply check whether the call was successful. + int result = Interop.Sys.Close(handle); + if (result != 0) + { + t_lastCloseErrorInfo = Interop.Sys.GetLastErrorInfo(); + } + return result == 0; + } + + public override bool IsInvalid + { + get + { + long h = (long)handle; + return h < 0 || h > int.MaxValue; + } + } + + internal static SafeFileHandle Open(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) { // Translate the arguments into arguments for an open call. Interop.Sys.OpenFlags openFlags = PreOpenConfigurationFromOptions(mode, access, share, options); @@ -45,8 +80,18 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA Interop.Sys.Permissions.S_IROTH | Interop.Sys.Permissions.S_IWOTH; SafeFileHandle safeFileHandle = Open(fullPath, openFlags, (int)OpenPermissions); - Init(safeFileHandle, fullPath, mode, access, share, options, preallocationSize); - return safeFileHandle; + try + { + safeFileHandle.Init(fullPath, mode, access, share, options, preallocationSize); + + return safeFileHandle; + } + catch (Exception) + { + safeFileHandle.Dispose(); + + throw; + } } private static SafeFileHandle Open(string fullPath, Interop.Sys.OpenFlags flags, int mode) @@ -110,40 +155,6 @@ private static bool DirectoryExists(string fullPath) return ((fileinfo.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR); } - // Each thread will have its own copy. This prevents race conditions if the handle had the last error. - [ThreadStatic] - internal static Interop.ErrorInfo? t_lastCloseErrorInfo; - - protected override bool ReleaseHandle() - { - // When the SafeFileHandle was opened, we likely issued an flock on the created descriptor in order to add - // an advisory lock. This lock should be removed via closing the file descriptor, but close can be - // interrupted, and we don't retry closes. As such, we could end up leaving the file locked, - // which could prevent subsequent usage of the file until this process dies. To avoid that, we proactively - // try to release the lock before we close the handle. (If it's not locked, there's no behavioral - // problem trying to unlock it.) - Interop.Sys.FLock(handle, Interop.Sys.LockOperations.LOCK_UN); // ignore any errors - - // Close the descriptor. Although close is documented to potentially fail with EINTR, we never want - // to retry, as the descriptor could actually have been closed, been subsequently reassigned, and - // be in use elsewhere in the process. Instead, we simply check whether the call was successful. - int result = Interop.Sys.Close(handle); - if (result != 0) - { - t_lastCloseErrorInfo = Interop.Sys.GetLastErrorInfo(); - } - return result == 0; - } - - public override bool IsInvalid - { - get - { - long h = (long)handle; - return h < 0 || h > int.MaxValue; - } - } - /// Translates the FileMode, FileAccess, and FileOptions values into flags to be passed when opening the file. /// The FileMode provided to the stream's constructor. /// The FileAccess provided to the stream's constructor @@ -209,15 +220,15 @@ private static Interop.Sys.OpenFlags PreOpenConfigurationFromOptions(FileMode mo return flags; } - private static void Init(SafeFileHandle handle, string path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) + private void Init(string path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) { - handle.IsAsync = (options & FileOptions.Asynchronous) != 0; + IsAsync = (options & FileOptions.Asynchronous) != 0; // Lock the file if requested via FileShare. This is only advisory locking. FileShare.None implies an exclusive // lock on the file and all other modes use a shared lock. While this is not as granular as Windows, not mandatory, // and not atomic with file opening, it's better than nothing. Interop.Sys.LockOperations lockOperation = (share == FileShare.None) ? Interop.Sys.LockOperations.LOCK_EX : Interop.Sys.LockOperations.LOCK_SH; - if (Interop.Sys.FLock(_fileHandle, lockOperation | Interop.Sys.LockOperations.LOCK_NB) < 0) + if (Interop.Sys.FLock(this, lockOperation | Interop.Sys.LockOperations.LOCK_NB) < 0) { // The only error we care about is EWOULDBLOCK, which indicates that the file is currently locked by someone // else and we would block trying to access it. Other errors, such as ENOTSUP (locking isn't supported) or @@ -239,20 +250,15 @@ private static void Init(SafeFileHandle handle, string path, FileMode mode, File 0; if (fadv != 0) { - CheckFileCall(Interop.Sys.PosixFAdvise(_fileHandle, 0, 0, fadv), + FileStreamHelpers.CheckFileCall(Interop.Sys.PosixFAdvise(this, 0, 0, fadv), path, ignoreNotSupported: true); // just a hint. } - if (mode == FileMode.Append) - { - // Jump to the end of the file if opened as Append. - _appendStart = SeekCore(_fileHandle, 0, SeekOrigin.End); - } - else if (mode == FileMode.Create || mode == FileMode.Truncate) + if (mode == FileMode.Create || mode == FileMode.Truncate) { // Truncate the file now if the file mode requires it. This ensures that the file only will be truncated // if opened successfully. - if (Interop.Sys.FTruncate(_fileHandle, 0) < 0) + if (Interop.Sys.FTruncate(this, 0) < 0) { Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); if (errorInfo.Error != Interop.Error.EBADF && errorInfo.Error != Interop.Error.EINVAL) @@ -268,10 +274,10 @@ private static void Init(SafeFileHandle handle, string path, FileMode mode, File // If preallocationSize has been provided for a creatable and writeable file if (FileStreamHelpers.ShouldPreallocate(preallocationSize, access, mode)) { - int fallocateResult = Interop.Sys.PosixFAllocate(_fileHandle, 0, preallocationSize); + int fallocateResult = Interop.Sys.PosixFAllocate(this, 0, preallocationSize); if (fallocateResult != 0) { - _fileHandle.Dispose(); + Dispose(); Interop.Sys.Unlink(path!); // remove the file to mimic Windows behaviour (atomic operation) if (fallocateResult == -1) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 7f29553fd9e82..969a4638c7815 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -12,9 +12,8 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private const FileOptions Unknown = (FileOptions)(-1); private const FileOptions NoBuffering = (FileOptions)(0x20000000); - private FileOptions _fileOptions = Unknown; + private FileOptions _fileOptions = (FileOptions)(-1); private int _fileType = -1; public SafeFileHandle() : base(true) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs index 6395787823452..868ccb84fd7f2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Unix.cs @@ -2,11 +2,6 @@ // The .NET Foundation licenses this file to you under the MIT license. using Microsoft.Win32.SafeHandles; -using System.Diagnostics; -using System.Runtime.InteropServices; -using System.Runtime.Versioning; -using System.Threading; -using System.Threading.Tasks; namespace System.IO.Strategies { @@ -20,6 +15,18 @@ private static FileStreamStrategy ChooseStrategyCore(SafeFileHandle handle, File private static FileStreamStrategy ChooseStrategyCore(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize) => new Net5CompatFileStreamStrategy(path, mode, access, share, bufferSize == 0 ? 1 : bufferSize, options, preallocationSize); - internal static bool GetDefaultIsAsync(SafeFileHandle handle, bool defaultIsAsync) => handle.IsAsync ?? defaultIsAsync; + internal static long CheckFileCall(long result, string? path, bool ignoreNotSupported = false) + { + if (result < 0) + { + Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); + if (!(ignoreNotSupported && errorInfo.Error == Interop.Error.ENOTSUP)) + { + throw Interop.GetExceptionForIoErrno(errorInfo, path, isDirectory: false); + } + } + + return result; + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs index cc62ccc1306cf..1100ff4329fc9 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs @@ -13,9 +13,6 @@ namespace System.IO.Strategies /// Provides an implementation of a file stream for Unix files. internal sealed partial class Net5CompatFileStreamStrategy : FileStreamStrategy { - /// File mode. - private FileMode _mode; - /// Advanced options requested when opening the file. private FileOptions _options; @@ -31,19 +28,23 @@ internal sealed partial class Net5CompatFileStreamStrategy : FileStreamStrategy /// private AsyncState? _asyncState; - /// Lazily-initialized value for whether the file supports seeking. - private bool? _canSeek; - private void Init(FileMode mode, string originalPath, FileOptions options) { // FileStream performs most of the general argument validation. We can assume here that the arguments // are all checked and consistent (e.g. non-null-or-empty path; valid enums in mode, access, share, and options; etc.) // Store the arguments - _mode = mode; _options = options; if (_useAsyncIO) + { _asyncState = new AsyncState(); + } + + if (mode == FileMode.Append) + { + // Jump to the end of the file if opened as Append. + _appendStart = SeekCore(_fileHandle, 0, SeekOrigin.End); + } Debug.Assert(_fileHandle.IsAsync == _useAsyncIO); } @@ -54,33 +55,11 @@ private void InitFromHandle(SafeFileHandle handle, FileAccess access, bool useAs if (useAsyncIO) _asyncState = new AsyncState(); - if (CanSeekCore(handle)) // use non-virtual CanSeekCore rather than CanSeek to avoid making virtual call during ctor + if (handle.CanSeek) SeekCore(handle, 0, SeekOrigin.Current); } - /// Gets a value indicating whether the current stream supports seeking. - public override bool CanSeek => CanSeekCore(_fileHandle); - - /// Gets a value indicating whether the current stream supports seeking. - /// - /// Separated out of CanSeek to enable making non-virtual call to this logic. - /// We also pass in the file handle to allow the constructor to use this before it stashes the handle. - /// - private bool CanSeekCore(SafeFileHandle fileHandle) - { - if (fileHandle.IsClosed) - { - return false; - } - - if (!_canSeek.HasValue) - { - // Lazily-initialize whether we're able to seek, tested by seeking to our current location. - _canSeek = Interop.Sys.LSeek(fileHandle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0; - } - - return _canSeek.GetValueOrDefault(); - } + public override bool CanSeek => _fileHandle.CanSeek; public override long Length { @@ -633,31 +612,17 @@ public override long Seek(long offset, SeekOrigin origin) /// The new position in the stream. private long SeekCore(SafeFileHandle fileHandle, long offset, SeekOrigin origin, bool closeInvalidHandle = false) { - Debug.Assert(!fileHandle.IsClosed && CanSeekCore(fileHandle)); + Debug.Assert(!fileHandle.IsClosed && fileHandle.CanSeek); Debug.Assert(origin >= SeekOrigin.Begin && origin <= SeekOrigin.End); - long pos = CheckFileCall(Interop.Sys.LSeek(fileHandle, offset, (Interop.Sys.SeekWhence)(int)origin)); // SeekOrigin values are the same as Interop.libc.SeekWhence values + long pos = FileStreamHelpers.CheckFileCall(Interop.Sys.LSeek(fileHandle, offset, (Interop.Sys.SeekWhence)(int)origin), _path); // SeekOrigin values are the same as Interop.libc.SeekWhence values _filePosition = pos; return pos; } - private long CheckFileCall(long result, bool ignoreNotSupported = false) - { - if (result < 0) - { - Interop.ErrorInfo errorInfo = Interop.Sys.GetLastErrorInfo(); - if (!(ignoreNotSupported && errorInfo.Error == Interop.Error.ENOTSUP)) - { - throw Interop.GetExceptionForIoErrno(errorInfo, _path, isDirectory: false); - } - } - - return result; - } - private int CheckFileCall(int result, bool ignoreNotSupported = false) { - CheckFileCall((long)result, ignoreNotSupported); + FileStreamHelpers.CheckFileCall(result, _path, ignoreNotSupported); return result; } From 9ef23eb21660efd5eab55097ce3ef2c7adce7932 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 10:19:54 +0200 Subject: [PATCH 16/71] don't try to test DeleteOnClose for FileStreams created from handle as it's a Windows-specific concept --- .../tests/File/OpenHandle.cs | 16 ++++++++++++++++ .../FileStream/ctor_str_fm_fa_fs_buffer_fo.cs | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index 859e6d0df5310..5abede66001b3 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -53,5 +53,21 @@ public void SafeFileHandle_IsAsync_ReturnsCorrectInformation(FileOptions options } } } + + [Theory] + [InlineData(FileOptions.DeleteOnClose)] + [InlineData(FileOptions.DeleteOnClose | FileOptions.Asynchronous)] + public override void DeleteOnClose_FileDeletedAfterClose(FileOptions options) + { + // Unix doesn't directly support DeleteOnClose + // For FileStream created out of path, we mimic it by closing the handle first + // and then unlinking the path + // Since SafeFileHandle does not always have the path and we can't find path for given file handle on Unix + // this test runs only on Windows + if (OperatingSystem.IsWindows()) // async file handles are a Windows concept + { + base.DeleteOnClose_FileDeletedAfterClose(options); + } + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs index 26f3a2fc55d2b..a68e709ba04dd 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs @@ -77,7 +77,7 @@ public void ValidFileOptions_Encrypted(FileOptions option) [Theory] [InlineData(FileOptions.DeleteOnClose)] [InlineData(FileOptions.DeleteOnClose | FileOptions.Asynchronous)] - public void DeleteOnClose_FileDeletedAfterClose(FileOptions options) + public virtual void DeleteOnClose_FileDeletedAfterClose(FileOptions options) { string path = GetTestFilePath(); Assert.False(File.Exists(path)); From 485eb933b31f43de065a74a28cc8396ad04caa6b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 10:29:25 +0200 Subject: [PATCH 17/71] take advantage of knowing the file type and avoid one syscall for CanSeek for regular files --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index e6bc5111bad41..4184fe2d10c4e 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -124,8 +124,7 @@ private static SafeFileHandle Open(string fullPath, Interop.Sys.OpenFlags flags, // Make sure it's not a directory; we do this after opening it once we have a file descriptor // to avoid race conditions. - Interop.Sys.FileStatus status; - if (Interop.Sys.FStat(handle, out status) != 0) + if (Interop.Sys.FStat(handle, out Interop.Sys.FileStatus status) != 0) { handle.Dispose(); throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), fullPath); @@ -136,6 +135,15 @@ private static SafeFileHandle Open(string fullPath, Interop.Sys.OpenFlags flags, throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), fullPath, isDirectory: true); } + if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFREG) + { + // we take advantage of the information provided by the fstat syscall + // and for regular files (most common case) + // avoid one extra sys call for determining whether file can be seeked + handle._canSeek = true; + Debug.Assert(Interop.Sys.LSeek(handle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + } + return handle; } From c8818aa24f66e06799618ae7b9b852f63760d74b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 10:44:37 +0200 Subject: [PATCH 18/71] revert some ordering changes to make the code easier to review --- .../tests/File/OpenHandle.cs | 2 +- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 144 +++++++++--------- 2 files changed, 76 insertions(+), 70 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index 5abede66001b3..b3a394549536e 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -62,7 +62,7 @@ public override void DeleteOnClose_FileDeletedAfterClose(FileOptions options) // Unix doesn't directly support DeleteOnClose // For FileStream created out of path, we mimic it by closing the handle first // and then unlinking the path - // Since SafeFileHandle does not always have the path and we can't find path for given file handle on Unix + // Since SafeFileHandle does not always have the path and we can't find path for given file descriptor on Unix // this test runs only on Windows if (OperatingSystem.IsWindows()) // async file handles are a Windows concept { diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 4184fe2d10c4e..4165aa5502e8f 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -31,6 +31,81 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : this(ownsHand internal bool CanSeek => !IsClosed && (_canSeek ??= Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + /// Opens the specified file with the requested flags and mode. + /// The path to the file. + /// The flags with which to open the file. + /// The mode for opening the file. + /// A SafeFileHandle for the opened file. + private static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, int mode) + { + Debug.Assert(path != null); + SafeFileHandle handle = Interop.Sys.Open(path, flags, mode); + + if (handle.IsInvalid) + { + handle.Dispose(); + Interop.ErrorInfo error = Interop.Sys.GetLastErrorInfo(); + + // If we fail to open the file due to a path not existing, we need to know whether to blame + // the file itself or its directory. If we're creating the file, then we blame the directory, + // otherwise we blame the file. + // + // When opening, we need to align with Windows, which considers a missing path to be + // FileNotFound only if the containing directory exists. + + bool isDirectory = (error.Error == Interop.Error.ENOENT) && + ((flags & Interop.Sys.OpenFlags.O_CREAT) != 0 + || !DirectoryExists(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(path!))!)); + + Interop.CheckIo( + error.Error, + path, + isDirectory, + errorRewriter: e => (e.Error == Interop.Error.EISDIR) ? Interop.Error.EACCES.Info() : e); + } + + // Make sure it's not a directory; we do this after opening it once we have a file descriptor + // to avoid race conditions. + Interop.Sys.FileStatus status; + if (Interop.Sys.FStat(handle, out status) != 0) + { + handle.Dispose(); + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path); + } + if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR) + { + handle.Dispose(); + throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), path, isDirectory: true); + } + + if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFREG) + { + // we take advantage of the information provided by the fstat syscall + // and for regular files (most common case) + // avoid one extra sys call for determining whether file can be seeked + handle._canSeek = true; + Debug.Assert(Interop.Sys.LSeek(handle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + } + + return handle; + } + + private static bool DirectoryExists(string fullPath) + { + Interop.Sys.FileStatus fileinfo; + + // First use stat, as we want to follow symlinks. If that fails, it could be because the symlink + // is broken, we don't have permissions, etc., in which case fall back to using LStat to evaluate + // based on the symlink itself. + if (Interop.Sys.Stat(fullPath, out fileinfo) < 0 && + Interop.Sys.LStat(fullPath, out fileinfo) < 0) + { + return false; + } + + return ((fileinfo.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR); + } + // Each thread will have its own copy. This prevents race conditions if the handle had the last error. [ThreadStatic] internal static Interop.ErrorInfo? t_lastCloseErrorInfo; @@ -94,75 +169,6 @@ internal static SafeFileHandle Open(string fullPath, FileMode mode, FileAccess a } } - private static SafeFileHandle Open(string fullPath, Interop.Sys.OpenFlags flags, int mode) - { - Debug.Assert(fullPath != null); - SafeFileHandle handle = Interop.Sys.Open(fullPath, flags, mode); - - if (handle.IsInvalid) - { - handle.Dispose(); - Interop.ErrorInfo error = Interop.Sys.GetLastErrorInfo(); - - // If we fail to open the file due to a path not existing, we need to know whether to blame - // the file itself or its directory. If we're creating the file, then we blame the directory, - // otherwise we blame the file. - // - // When opening, we need to align with Windows, which considers a missing path to be - // FileNotFound only if the containing directory exists. - - bool isDirectory = (error.Error == Interop.Error.ENOENT) && - ((flags & Interop.Sys.OpenFlags.O_CREAT) != 0 - || !DirectoryExists(Path.GetDirectoryName(Path.TrimEndingDirectorySeparator(fullPath!))!)); - - Interop.CheckIo( - error.Error, - fullPath, - isDirectory, - errorRewriter: e => (e.Error == Interop.Error.EISDIR) ? Interop.Error.EACCES.Info() : e); - } - - // Make sure it's not a directory; we do this after opening it once we have a file descriptor - // to avoid race conditions. - if (Interop.Sys.FStat(handle, out Interop.Sys.FileStatus status) != 0) - { - handle.Dispose(); - throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), fullPath); - } - if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR) - { - handle.Dispose(); - throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), fullPath, isDirectory: true); - } - - if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFREG) - { - // we take advantage of the information provided by the fstat syscall - // and for regular files (most common case) - // avoid one extra sys call for determining whether file can be seeked - handle._canSeek = true; - Debug.Assert(Interop.Sys.LSeek(handle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); - } - - return handle; - } - - private static bool DirectoryExists(string fullPath) - { - Interop.Sys.FileStatus fileinfo; - - // First use stat, as we want to follow symlinks. If that fails, it could be because the symlink - // is broken, we don't have permissions, etc., in which case fall back to using LStat to evaluate - // based on the symlink itself. - if (Interop.Sys.Stat(fullPath, out fileinfo) < 0 && - Interop.Sys.LStat(fullPath, out fileinfo) < 0) - { - return false; - } - - return ((fileinfo.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR); - } - /// Translates the FileMode, FileAccess, and FileOptions values into flags to be passed when opening the file. /// The FileMode provided to the stream's constructor. /// The FileAccess provided to the stream's constructor From dc13de067994195efed2fd507b65109938bdb2f2 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 10:50:06 +0200 Subject: [PATCH 19/71] Apply suggestions from code review Co-authored-by: Stephen Toub --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 12 +++++------- .../System.Private.CoreLib/src/System/IO/File.cs | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 4165aa5502e8f..4cbd9b07a4ff4 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -294,13 +294,11 @@ private void Init(string path, FileMode mode, FileAccess access, FileShare share Dispose(); Interop.Sys.Unlink(path!); // remove the file to mimic Windows behaviour (atomic operation) - if (fallocateResult == -1) - { - throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, path, preallocationSize)); - } - - Debug.Assert(fallocateResult == -2); - throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, path, preallocationSize)); + Debug.Assert(fallocateResult == -1 || fallocateResult == -2) + throw new IOException(SR.Format( + fallocateResult == -1 ? SR.IO_DiskFull_Path_AllocationSize : SR.IO_FileTooLarge_Path_AllocationSize, + path, + preallocationSize)); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs index 579abdb0dcac4..de036ba2859d0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs @@ -1089,7 +1089,7 @@ private static async Task InternalWriteAllTextAsync(StreamWriter sw, string cont public static SafeFileHandle OpenHandle(string path, FileMode mode = FileMode.Open, FileAccess access = FileAccess.Read, FileShare share = FileShare.Read, FileOptions options = FileOptions.None, long preallocationSize = 0) { - Strategies.FileStreamHelpers.ValidateArguments(path, mode, access, share, 0 /* bufferSize */, options, preallocationSize); + Strategies.FileStreamHelpers.ValidateArguments(path, mode, access, share, bufferSize: 0, options, preallocationSize); return SafeFileHandle.Open(Path.GetFullPath(path), mode, access, share, options, preallocationSize); } From 6c5825280b5334d7bc0a59ce9bf43250099f1f1d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 11:01:31 +0200 Subject: [PATCH 20/71] add missing ; --- .../src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 4cbd9b07a4ff4..689dc4eac4cb8 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -294,7 +294,7 @@ private void Init(string path, FileMode mode, FileAccess access, FileShare share Dispose(); Interop.Sys.Unlink(path!); // remove the file to mimic Windows behaviour (atomic operation) - Debug.Assert(fallocateResult == -1 || fallocateResult == -2) + Debug.Assert(fallocateResult == -1 || fallocateResult == -2); throw new IOException(SR.Format( fallocateResult == -1 ? SR.IO_DiskFull_Path_AllocationSize : SR.IO_FileTooLarge_Path_AllocationSize, path, From fb24f5c5f70b3965881ec7306bb68d95a4623c07 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 11:36:15 +0200 Subject: [PATCH 21/71] simplify Windows implementation: * with NtCreateFile there is no need for Validation (NtCreateFile returns error and we never create a safe file handle) * unify status codes * complete error mapping --- .../Windows/NtDll/Interop.NtCreateFile.cs | 4 -- .../Interop/Windows/NtDll/Interop.NtStatus.cs | 2 + .../SafeHandles/SafeFileHandle.Windows.cs | 40 +++++-------------- .../Strategies/WindowsFileStreamStrategy.cs | 14 ++----- 4 files changed, 15 insertions(+), 45 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs index 04765e4fbd43d..84263c71e4016 100644 --- a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs +++ b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs @@ -10,10 +10,6 @@ internal static partial class Interop { internal static partial class NtDll { - internal const uint NT_ERROR_STATUS_DISK_FULL = 0xC000007F; - internal const uint NT_ERROR_STATUS_FILE_TOO_LARGE = 0xC0000904; - internal const uint NT_STATUS_INVALID_PARAMETER = 0xC000000D; - // https://msdn.microsoft.com/en-us/library/bb432380.aspx // https://msdn.microsoft.com/en-us/library/windows/hardware/ff566424.aspx [DllImport(Libraries.NtDll, CharSet = CharSet.Unicode, ExactSpelling = true)] diff --git a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtStatus.cs b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtStatus.cs index d79653a64105e..76715c83be916 100644 --- a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtStatus.cs +++ b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtStatus.cs @@ -17,5 +17,7 @@ internal static class StatusOptions internal const uint STATUS_ACCOUNT_RESTRICTION = 0xC000006E; internal const uint STATUS_NONE_MAPPED = 0xC0000073; internal const uint STATUS_INSUFFICIENT_RESOURCES = 0xC000009A; + internal const uint STATUS_DISK_FULL = 0xC000007F; + internal const uint STATUS_FILE_TOO_LARGE = 0xC0000904; } } diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 969a4638c7815..c475bf8f5c04c 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -4,7 +4,6 @@ using System; using System.Diagnostics; using System.IO; -using System.Runtime.InteropServices; using System.Text; using System.Threading; @@ -35,7 +34,7 @@ private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fi public bool IsAsync => (GetFileOptions() & FileOptions.Asynchronous) != 0; - internal bool CanSeek => GetFileType() == Interop.Kernel32.FileTypes.FILE_TYPE_DISK; + internal bool CanSeek => !IsClosed && GetFileType() == Interop.Kernel32.FileTypes.FILE_TYPE_DISK; internal bool IsPipe => GetFileType() == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE; @@ -53,8 +52,6 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA options, Interop.Kernel32.FileTypes.FILE_TYPE_DISK); // similarly to FileStream, we assume that only disk files can be referenced by path on Windows - fileHandle.Validate(fullPath); - fileHandle.InitThreadPoolBindingIfNeeded(); return fileHandle; @@ -91,14 +88,14 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac switch (ntStatus) { - case 0: + case Interop.StatusOptions.STATUS_SUCCESS: return fileHandle; - case Interop.NtDll.NT_ERROR_STATUS_DISK_FULL: + case Interop.StatusOptions.STATUS_DISK_FULL: throw new IOException(SR.Format(SR.IO_DiskFull_Path_AllocationSize, fullPath, preallocationSize)); // NtCreateFile has a bug and it reports STATUS_INVALID_PARAMETER for files // that are too big for the current file system. Example: creating a 4GB+1 file on a FAT32 drive. - case Interop.NtDll.NT_STATUS_INVALID_PARAMETER when preallocationSize > 0: - case Interop.NtDll.NT_ERROR_STATUS_FILE_TOO_LARGE: + case Interop.StatusOptions.STATUS_INVALID_PARAMETER when preallocationSize > 0: + case Interop.StatusOptions.STATUS_FILE_TOO_LARGE: throw new IOException(SR.Format(SR.IO_FileTooLarge_Path_AllocationSize, fullPath, preallocationSize)); default: int error = (int)Interop.NtDll.RtlNtStatusToDosError((int)ntStatus); @@ -106,26 +103,6 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac } } - private void Validate(string path) - { - if (IsInvalid) - { - // Return a meaningful exception with the full path. - - // NT5 oddity - when trying to open "C:\" as a Win32FileStream, - // we usually get ERROR_PATH_NOT_FOUND from the OS. We should - // probably be consistent w/ every other directory. - int errorCode = Marshal.GetLastPInvokeError(); - - if (errorCode == Interop.Errors.ERROR_PATH_NOT_FOUND && path.Length == PathInternal.GetRootLength(path)) - { - errorCode = Interop.Errors.ERROR_ACCESS_DENIED; - } - - throw Win32Marshal.GetExceptionForWin32Error(errorCode, path); - } - } - internal void InitThreadPoolBindingIfNeeded() { if (IsAsync == true && ThreadPoolBinding == null) @@ -165,16 +142,17 @@ private unsafe FileOptions GetFileOptions() } Interop.NtDll.CreateOptions options; - int status = Interop.NtDll.NtQueryInformationFile( + int ntStatus = Interop.NtDll.NtQueryInformationFile( FileHandle: this, IoStatusBlock: out _, FileInformation: &options, Length: sizeof(uint), FileInformationClass: Interop.NtDll.FileModeInformation); - if (status != 0) + if (ntStatus != Interop.StatusOptions.STATUS_SUCCESS) { - throw Win32Marshal.GetExceptionForWin32Error(Interop.Errors.ERROR_INVALID_HANDLE); + int error = (int)Interop.NtDll.RtlNtStatusToDosError(ntStatus); + throw Win32Marshal.GetExceptionForWin32Error(error); } FileOptions result = FileOptions.None; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs index c0f74e671c71b..bd6e8a6013d98 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs @@ -14,8 +14,6 @@ internal abstract class WindowsFileStreamStrategy : FileStreamStrategy protected readonly string? _path; // The path to the opened file. private readonly FileAccess _access; // What file was opened for. private readonly FileShare _share; - private readonly bool _canSeek; // Whether can seek (file) or not (pipe). - private readonly bool _isPipe; // Whether to disable async buffering code. protected long _filePosition; private long _appendStart; // When appending, prevent overwriting file. @@ -28,12 +26,9 @@ internal WindowsFileStreamStrategy(SafeFileHandle handle, FileAccess access, Fil _share = share; _exposedHandle = true; - _canSeek = handle.CanSeek; - _isPipe = handle.IsPipe; - handle.InitThreadPoolBindingIfNeeded(); - if (_canSeek) + if (handle.CanSeek) { // given strategy was created out of existing handle, so we have to perform // a syscall to get the current handle offset @@ -59,8 +54,6 @@ internal WindowsFileStreamStrategy(string path, FileMode mode, FileAccess access try { - _canSeek = true; - Init(mode, path); } catch @@ -73,7 +66,7 @@ internal WindowsFileStreamStrategy(string path, FileMode mode, FileAccess access } } - public sealed override bool CanSeek => _canSeek; + public sealed override bool CanSeek => _fileHandle.CanSeek; public sealed override bool CanRead => !_fileHandle.IsClosed && (_access & FileAccess.Read) != 0; @@ -126,7 +119,8 @@ public override long Position internal sealed override bool IsClosed => _fileHandle.IsClosed; - internal sealed override bool IsPipe => _isPipe; + internal sealed override bool IsPipe => _fileHandle.IsPipe; + // Flushing is the responsibility of BufferedFileStreamStrategy internal sealed override SafeFileHandle SafeFileHandle { From 8baf0bd994289b32f3540aa1bccb35f2c06825ec Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 12:55:19 +0200 Subject: [PATCH 22/71] unify CanSeek and IsPipe for the .NET 5 Windows strategy as well --- .../Net5CompatFileStreamStrategy.Windows.cs | 25 ++++++------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs index b0263e15e2694..fccd0232ecddc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs @@ -38,8 +38,6 @@ namespace System.IO.Strategies { internal sealed partial class Net5CompatFileStreamStrategy : FileStreamStrategy { - private bool _canSeek; - private bool _isPipe; // Whether to disable async buffering code. private long _appendStart; // When appending, prevent overwriting file. private Task _activeBufferOperation = Task.CompletedTask; // tracks in-progress async ops using the buffer @@ -61,8 +59,6 @@ private void Init(FileMode mode, string originalPath, FileOptions options) { _appendStart = -1; } - - _canSeek = true; } private void InitFromHandle(SafeFileHandle handle, FileAccess access, bool useAsyncIO) @@ -86,12 +82,9 @@ private void InitFromHandle(SafeFileHandle handle, FileAccess access, bool useAs private void InitFromHandleImpl(SafeFileHandle handle, bool useAsyncIO) { - _canSeek = handle.CanSeek; - _isPipe = handle.IsPipe; - handle.InitThreadPoolBindingIfNeeded(); - if (_canSeek) + if (handle.CanSeek) SeekCore(handle, 0, SeekOrigin.Current); else _filePosition = 0; @@ -99,7 +92,7 @@ private void InitFromHandleImpl(SafeFileHandle handle, bool useAsyncIO) private bool HasActiveBufferOperation => !_activeBufferOperation.IsCompleted; - public override bool CanSeek => _canSeek; + public override bool CanSeek => _fileHandle.CanSeek; public unsafe override long Length { @@ -154,7 +147,6 @@ protected override void Dispose(bool disposing) } _preallocatedOverlapped?.Dispose(); - _canSeek = false; // Don't set the buffer to null, to avoid a NullReferenceException // when users have a race condition in their code (i.e. they call @@ -182,7 +174,6 @@ public override async ValueTask DisposeAsync() } _preallocatedOverlapped?.Dispose(); - _canSeek = false; GC.SuppressFinalize(this); // the handle is closed; nothing further for the finalizer to do } } @@ -326,7 +317,7 @@ private int ReadSpan(Span destination) // If we are reading from a device with no clear EOF like a // serial port or a pipe, this will cause us to block incorrectly. - if (!_isPipe) + if (!_fileHandle.IsPipe) { // If we hit the end of the buffer and didn't have enough bytes, we must // read some more from the underlying stream. However, if we got @@ -479,7 +470,7 @@ public override long Seek(long offset, SeekOrigin origin) // internal position private long SeekCore(SafeFileHandle fileHandle, long offset, SeekOrigin origin, bool closeInvalidHandle = false) { - Debug.Assert(!fileHandle.IsClosed && _canSeek, "!fileHandle.IsClosed && _canSeek"); + Debug.Assert(fileHandle.CanSeek, "fileHandle.CanSeek"); return _filePosition = FileStreamHelpers.Seek(fileHandle, _path, offset, origin, closeInvalidHandle); } @@ -601,7 +592,7 @@ private unsafe void WriteCore(ReadOnlySpan source) Debug.Assert((_readPos == 0 && _readLength == 0 && _writePos >= 0) || (_writePos == 0 && _readPos <= _readLength), "We're either reading or writing, but not both."); - if (_isPipe) + if (_fileHandle.IsPipe) { // Pipes are tricky, at least when you have 2 different pipes // that you want to use simultaneously. When redirecting stdout @@ -632,7 +623,7 @@ private unsafe void WriteCore(ReadOnlySpan source) } } - Debug.Assert(!_isPipe, "Should not be a pipe."); + Debug.Assert(!_fileHandle.IsPipe, "Should not be a pipe."); // Handle buffering. if (_writePos > 0) FlushWriteBuffer(); @@ -811,12 +802,12 @@ private ValueTask WriteAsyncInternal(ReadOnlyMemory source, CancellationTo { Debug.Assert(_useAsyncIO); Debug.Assert((_readPos == 0 && _readLength == 0 && _writePos >= 0) || (_writePos == 0 && _readPos <= _readLength), "We're either reading or writing, but not both."); - Debug.Assert(!_isPipe || (_readPos == 0 && _readLength == 0), "Win32FileStream must not have buffered data here! Pipes should be unidirectional."); + Debug.Assert(!_fileHandle.IsPipe || (_readPos == 0 && _readLength == 0), "Win32FileStream must not have buffered data here! Pipes should be unidirectional."); if (!CanWrite) ThrowHelper.ThrowNotSupportedException_UnwritableStream(); bool writeDataStoredInBuffer = false; - if (!_isPipe) // avoid async buffering with pipes, as doing so can lead to deadlocks (see comments in ReadInternalAsyncCore) + if (!_fileHandle.IsPipe) // avoid async buffering with pipes, as doing so can lead to deadlocks (see comments in ReadInternalAsyncCore) { // Ensure the buffer is clear for writing if (_writePos == 0) From 50aba1f09b75f82e98f1ccdc9b5d0eb2b6c41be3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 13:05:57 +0200 Subject: [PATCH 23/71] move the !MS_IO_REDIST methods from `File.cs` to `File.netcoreapp.cs` file --- .../System.Private.CoreLib.Shared.projitems | 1 + .../src/System/IO/File.cs | 89 +----------------- .../src/System/IO/File.netcoreapp.cs | 94 +++++++++++++++++++ 3 files changed, 96 insertions(+), 88 deletions(-) create mode 100644 src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 69a6cbd913d20..c6c44f13afa18 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -407,6 +407,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs index de036ba2859d0..17e5f4c24fac3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.cs @@ -10,7 +10,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using Microsoft.Win32.SafeHandles; #if MS_IO_REDIST using System; @@ -23,7 +22,7 @@ namespace System.IO { // Class for creating FileStream objects, and some basic file management // routines such as Delete, etc. - public static class File + public static partial class File { private const int MaxByteArrayLength = 0x7FFFFFC7; private static Encoding? s_UTF8NoBOM; @@ -365,52 +364,6 @@ public static byte[] ReadAllBytes(string path) } } -#if !MS_IO_REDIST - private static byte[] ReadAllBytesUnknownLength(FileStream fs) - { - byte[]? rentedArray = null; - Span buffer = stackalloc byte[512]; - try - { - int bytesRead = 0; - while (true) - { - if (bytesRead == buffer.Length) - { - uint newLength = (uint)buffer.Length * 2; - if (newLength > MaxByteArrayLength) - { - newLength = (uint)Math.Max(MaxByteArrayLength, buffer.Length + 1); - } - - byte[] tmp = ArrayPool.Shared.Rent((int)newLength); - buffer.CopyTo(tmp); - if (rentedArray != null) - { - ArrayPool.Shared.Return(rentedArray); - } - buffer = rentedArray = tmp; - } - - Debug.Assert(bytesRead < buffer.Length); - int n = fs.Read(buffer.Slice(bytesRead)); - if (n == 0) - { - return buffer.Slice(0, bytesRead).ToArray(); - } - bytesRead += n; - } - } - finally - { - if (rentedArray != null) - { - ArrayPool.Shared.Return(rentedArray); - } - } - } -#endif - public static void WriteAllBytes(string path, byte[] bytes) { if (path == null) @@ -1054,45 +1007,5 @@ private static async Task InternalWriteAllTextAsync(StreamWriter sw, string cont ? Task.FromCanceled(cancellationToken) : InternalWriteAllLinesAsync(AsyncStreamWriter(path, encoding, append: true), contents, cancellationToken); } - -#if !MS_IO_REDIST - /// - /// Initializes a new instance of the class with the specified path, creation mode, read/write and sharing permission, the access other SafeFileHandles can have to the same file, additional file options and the allocation size. - /// - /// A relative or absolute path for the file that the current instance will encapsulate. - /// One of the enumeration values that determines how to open or create the file. The default value is - /// A bitwise combination of the enumeration values that determines how the file can be accessed. The default value is - /// A bitwise combination of the enumeration values that determines how the file will be shared by processes. The default value is . - /// The initial allocation size in bytes for the file. A positive value is effective only when a regular file is being created, overwritten, or replaced. - /// Negative values are not allowed. In other cases (including the default 0 value), it's ignored. - /// An object that describes optional parameters to use. - /// is . - /// is an empty string (""), contains only white space, or contains one or more invalid characters. - /// -or- - /// refers to a non-file device, such as CON:, COM1:, LPT1:, etc. in an NTFS environment. - /// refers to a non-file device, such as CON:, COM1:, LPT1:, etc. in a non-NTFS environment. - /// is negative. - /// -or- - /// , , or contain an invalid value. - /// The file cannot be found, such as when is or , and the file specified by does not exist. The file must already exist in these modes. - /// An I/O error, such as specifying when the file specified by already exists, occurred. - /// -or- - /// The disk was full (when was provided and was pointing to a regular file). - /// -or- - /// The file was too large (when was provided and was pointing to a regular file). - /// The caller does not have the required permission. - /// The specified path is invalid, such as being on an unmapped drive. - /// The requested is not permitted by the operating system for the specified , such as when is or and the file or directory is set for read-only access. - /// -or- - /// is specified for , but file encryption is not supported on the current platform. - /// The specified path, file name, or both exceed the system-defined maximum length. - public static SafeFileHandle OpenHandle(string path, FileMode mode = FileMode.Open, FileAccess access = FileAccess.Read, - FileShare share = FileShare.Read, FileOptions options = FileOptions.None, long preallocationSize = 0) - { - Strategies.FileStreamHelpers.ValidateArguments(path, mode, access, share, bufferSize: 0, options, preallocationSize); - - return SafeFileHandle.Open(Path.GetFullPath(path), mode, access, share, options, preallocationSize); - } -#endif } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs new file mode 100644 index 0000000000000..bff701b1b477a --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Diagnostics; +using Microsoft.Win32.SafeHandles; + +namespace System.IO +{ + public static partial class File + { + /// + /// Initializes a new instance of the class with the specified path, creation mode, read/write and sharing permission, the access other SafeFileHandles can have to the same file, additional file options and the allocation size. + /// + /// A relative or absolute path for the file that the current instance will encapsulate. + /// One of the enumeration values that determines how to open or create the file. The default value is + /// A bitwise combination of the enumeration values that determines how the file can be accessed. The default value is + /// A bitwise combination of the enumeration values that determines how the file will be shared by processes. The default value is . + /// The initial allocation size in bytes for the file. A positive value is effective only when a regular file is being created, overwritten, or replaced. + /// Negative values are not allowed. In other cases (including the default 0 value), it's ignored. + /// An object that describes optional parameters to use. + /// is . + /// is an empty string (""), contains only white space, or contains one or more invalid characters. + /// -or- + /// refers to a non-file device, such as CON:, COM1:, LPT1:, etc. in an NTFS environment. + /// refers to a non-file device, such as CON:, COM1:, LPT1:, etc. in a non-NTFS environment. + /// is negative. + /// -or- + /// , , or contain an invalid value. + /// The file cannot be found, such as when is or , and the file specified by does not exist. The file must already exist in these modes. + /// An I/O error, such as specifying when the file specified by already exists, occurred. + /// -or- + /// The disk was full (when was provided and was pointing to a regular file). + /// -or- + /// The file was too large (when was provided and was pointing to a regular file). + /// The caller does not have the required permission. + /// The specified path is invalid, such as being on an unmapped drive. + /// The requested is not permitted by the operating system for the specified , such as when is or and the file or directory is set for read-only access. + /// -or- + /// is specified for , but file encryption is not supported on the current platform. + /// The specified path, file name, or both exceed the system-defined maximum length. + public static SafeFileHandle OpenHandle(string path, FileMode mode = FileMode.Open, FileAccess access = FileAccess.Read, + FileShare share = FileShare.Read, FileOptions options = FileOptions.None, long preallocationSize = 0) + { + Strategies.FileStreamHelpers.ValidateArguments(path, mode, access, share, bufferSize: 0, options, preallocationSize); + + return SafeFileHandle.Open(Path.GetFullPath(path), mode, access, share, options, preallocationSize); + } + + private static byte[] ReadAllBytesUnknownLength(FileStream fs) + { + byte[]? rentedArray = null; + Span buffer = stackalloc byte[512]; + try + { + int bytesRead = 0; + while (true) + { + if (bytesRead == buffer.Length) + { + uint newLength = (uint)buffer.Length * 2; + if (newLength > MaxByteArrayLength) + { + newLength = (uint)Math.Max(MaxByteArrayLength, buffer.Length + 1); + } + + byte[] tmp = ArrayPool.Shared.Rent((int)newLength); + buffer.CopyTo(tmp); + if (rentedArray != null) + { + ArrayPool.Shared.Return(rentedArray); + } + buffer = rentedArray = tmp; + } + + Debug.Assert(bytesRead < buffer.Length); + int n = fs.Read(buffer.Slice(bytesRead)); + if (n == 0) + { + return buffer.Slice(0, bytesRead).ToArray(); + } + bytesRead += n; + } + } + finally + { + if (rentedArray != null) + { + ArrayPool.Shared.Return(rentedArray); + } + } + } + } +} From 143d31c0a31f311e638d0955566515a75b16a675 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 17:08:42 +0200 Subject: [PATCH 24/71] RandomAccess: GetLength, Read and Write single buffer --- .../Unix/System.Native/Interop.Pread.cs | 13 ++ .../Unix/System.Native/Interop.Pwrite.cs | 13 ++ .../Native/Unix/System.Native/pal_io.c | 24 +++ .../Native/Unix/System.Native/pal_io.h | 14 ++ .../ref/System.IO.FileSystem.Forwards.cs | 1 + .../tests/RandomAccess/GetLength.cs | 69 ++++++++ .../tests/RandomAccess/Read.cs | 98 +++++++++++ .../tests/RandomAccess/Write.cs | 92 +++++++++++ .../tests/System.IO.FileSystem.Tests.csproj | 3 + .../System.Private.CoreLib.Shared.projitems | 9 + .../src/System/IO/RandomAccess.Unix.cs | 41 +++++ .../src/System/IO/RandomAccess.Windows.cs | 156 ++++++++++++++++++ .../src/System/IO/RandomAccess.cs | 100 +++++++++++ .../Strategies/FileStreamHelpers.Windows.cs | 77 +-------- .../System/IO/Strategies/FileStreamHelpers.cs | 4 +- .../Net5CompatFileStreamStrategy.Unix.cs | 4 +- .../Net5CompatFileStreamStrategy.Windows.cs | 6 +- .../SyncWindowsFileStreamStrategy.cs | 55 +----- .../Strategies/WindowsFileStreamStrategy.cs | 4 +- .../src/System/ThrowHelper.cs | 15 ++ .../System.Runtime/ref/System.Runtime.cs | 6 + 21 files changed, 668 insertions(+), 136 deletions(-) create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs create mode 100644 src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs new file mode 100644 index 0000000000000..4b25d9f18e631 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Pread", SetLastError = true)] + internal static extern unsafe int Pread(SafeHandle fd, byte* buffer, int count, long fileOffset); + } +} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs new file mode 100644 index 0000000000000..8f1bfa032615b --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Pwrite", SetLastError = true)] + internal static extern unsafe int Pwrite(SafeHandle fd, byte* buffer, int bufferSize, long fileOffset); + } +} diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index c63eb8898cace..14d986d9715d6 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1454,3 +1454,27 @@ int32_t SystemNative_ReadProcessStatusInfo(pid_t pid, ProcessStatus* processStat return -1; #endif // __sun } + +int32_t SystemNative_Pread(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) +{ + assert(buffer != NULL); + assert(bufferSize >= 0); + + ssize_t count; + while ((count = pread(ToFileDescriptor(fd), buffer, (uint32_t)bufferSize, (off_t)fileOffset)) < 0 && errno == EINTR); + + assert(count >= -1 && count <= bufferSize); + return (int32_t)count; +} + +int32_t SystemNative_Pwrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) +{ + assert(buffer != NULL); + assert(bufferSize >= 0); + + ssize_t count; + while ((count = pwrite(ToFileDescriptor(fd), buffer, (uint32_t)bufferSize, (off_t)fileOffset)) < 0 && errno == EINTR); + + assert(count >= -1 && count <= bufferSize); + return (int32_t)count; +} \ No newline at end of file diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index e3d402d9be5f6..6faccdff5f05e 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -730,3 +730,17 @@ PALEXPORT int32_t SystemNative_LChflagsCanSetHiddenFlag(void); * Returns 1 if the process status was read; otherwise, 0. */ PALEXPORT int32_t SystemNative_ReadProcessStatusInfo(pid_t pid, ProcessStatus* processStatus); + +/** + * Reads the number of bytes specified into the provided buffer from the specified, opened file descriptor at specified offset. + * + * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. + */ +PALEXPORT int32_t SystemNative_Pread(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); + +/** + * Writes the number of bytes specified in the buffer into the specified, opened file descriptor at specified offset. + * + * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. + */ +PALEXPORT int32_t SystemNative_Pwrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); diff --git a/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs b/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs index f2ea51b0ddb0d..3f6524af8335b 100644 --- a/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs +++ b/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs @@ -16,6 +16,7 @@ [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.MatchCasing))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.MatchType))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.SearchOption))] +[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.RandomAccess))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.Enumeration.FileSystemEntry))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.Enumeration.FileSystemEnumerable<>))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.Enumeration.FileSystemEnumerator<>))] diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs new file mode 100644 index 0000000000000..a01e0becf885d --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Pipes; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_GetLength : FileSystemTest + { + [Fact] + public void ThrowsArgumentNullExceptionForNullHandle() + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.GetLength(null)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsArgumentExceptionForInvalidHandle() + { + SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); + + ArgumentException ex = Assert.Throws(() => RandomAccess.GetLength(handle)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsObjectDisposedExceptionForDisposedHandle() + { + SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); + handle.Dispose(); + + Assert.Throws(() => RandomAccess.GetLength(handle)); + } + + [Fact] + public void ThrowsNotSupportedExceptionForUnseekableFile() + { + using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) + using (SafeFileHandle handle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) + { + Assert.Throws(() => RandomAccess.GetLength(handle)); + } + } + + [Fact] + public void ReturnsZeroForEmptyFile() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + { + Assert.Equal(0, RandomAccess.GetLength(handle)); + } + } + + [Fact] + public void ReturnsExactSizeForNonEmptyFiles() + { + const int fileSize = 123; + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[fileSize]); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) + { + Assert.Equal(fileSize, RandomAccess.GetLength(handle)); + } + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs new file mode 100644 index 0000000000000..3b06c15322642 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs @@ -0,0 +1,98 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Pipes; +using System.Linq; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_Read : FileSystemTest + { + [Fact] + public void ThrowsArgumentNullExceptionForNullHandle() + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.Read(null, Array.Empty(), 0)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsArgumentExceptionForInvalidHandle() + { + SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); + + ArgumentException ex = Assert.Throws(() => RandomAccess.Read(handle, Array.Empty(), 0)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsObjectDisposedExceptionForDisposedHandle() + { + SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); + handle.Dispose(); + + Assert.Throws(() => RandomAccess.Read(handle, Array.Empty(), 0)); + } + + [Fact] + public void ThrowsNotSupportedExceptionForUnseekableFile() + { + using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) + using (SafeFileHandle pipeHandle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) + { + Assert.Throws(() => RandomAccess.Read(pipeHandle, Array.Empty(), 0)); + } + } + + [Fact] + public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + { + ArgumentOutOfRangeException ex = Assert.Throws(() => RandomAccess.Read(validHandle, Array.Empty(), -1)); + Assert.Equal("fileOffset", ex.ParamName); + } + } + + [Fact] + public void ThrowsArgumentExceptionForAsyncFileHandle() + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) + { + ArgumentException ex = Assert.Throws(() => RandomAccess.Read(validHandle, new byte[100], 0)); + Assert.Equal("handle", ex.ParamName); + } + } + + [Fact] + public void SupportsPartialReads() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] expected = new byte[fileSize]; + new Random().NextBytes(expected); + File.WriteAllBytes(filePath, expected); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) + { + byte[] actual = new byte[fileSize + 1]; + int current = 0; + int total = 0; + + do + { + current = RandomAccess.Read( + handle, + actual.AsSpan(total, Math.Min(actual.Length - total, fileSize / 4)), + fileOffset: total); + + total += current; + } while (current != 0); + + Assert.Equal(fileSize, total); + Assert.Equal(expected, actual.Take(total).ToArray()); + } + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs new file mode 100644 index 0000000000000..38c6f3ef32821 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs @@ -0,0 +1,92 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Pipes; +using System.Linq; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_Write : FileSystemTest + { + [Fact] + public void ThrowsArgumentNullExceptionForNullHandle() + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.Write(null, Array.Empty(), 0)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsArgumentExceptionForInvalidHandle() + { + SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); + + ArgumentException ex = Assert.Throws(() => RandomAccess.Write(handle, Array.Empty(), 0)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsObjectDisposedExceptionForDisposedHandle() + { + SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); + handle.Dispose(); + + Assert.Throws(() => RandomAccess.Write(handle, Array.Empty(), 0)); + } + + [Fact] + public void ThrowsNotSupportedExceptionForUnseekableFile() + { + using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) + using (SafeFileHandle pipeHandle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) + { + Assert.Throws(() => RandomAccess.Write(pipeHandle, Array.Empty(), 0)); + } + } + + [Fact] + public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + { + ArgumentOutOfRangeException ex = Assert.Throws(() => RandomAccess.Write(validHandle, Array.Empty(), -1)); + Assert.Equal("fileOffset", ex.ParamName); + } + } + + [Fact] + public void ThrowsArgumentExceptionForAsyncFileHandle() + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) + { + ArgumentException ex = Assert.Throws(() => RandomAccess.Write(validHandle, new byte[100], 0)); + Assert.Equal("handle", ex.ParamName); + } + } + + [Fact] + public void SupportsPartialWrites() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] content = new byte[fileSize]; + new Random().NextBytes(content); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + int total = 0; + + while (total != fileSize) + { + total += RandomAccess.Write( + handle, + content.AsSpan(total, Math.Min(content.Length - total, fileSize / 4)), + fileOffset: total); + } + } + + Assert.Equal(content, File.ReadAllBytes(filePath)); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 6f6597b8101ea..1fda5d01db858 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -51,6 +51,9 @@ + + + diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index c6c44f13afa18..a0ff9077393d3 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -430,6 +430,7 @@ + @@ -1775,6 +1776,7 @@ + @@ -1967,6 +1969,12 @@ Common\Interop\Unix\System.Native\Interop.PosixFAllocate.cs + + Common\Interop\Unix\System.Native\Interop.Pread.cs + + + Common\Interop\Unix\System.Native\Interop.Pwrite.cs + Common\Interop\Unix\System.Native\Interop.Read.cs @@ -2034,6 +2042,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs new file mode 100644 index 0000000000000..858c9a47453c3 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -0,0 +1,41 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Diagnostics; +using System.IO; +using System.IO.Strategies; +using Microsoft.Win32.SafeHandles; + +namespace System.IO +{ + public static partial class RandomAccess + { + internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) + { + int result = Interop.Sys.FStat(handle, out Interop.Sys.FileStatus status); + FileStreamHelpers.CheckFileCall(returnCode, path); + return status.Size + } + + private static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, long fileOffset) + { + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + return FileStreamHelpers.CheckFileCall( + Interop.Sys.Pread(handle, bufPtr, buffer.Length, fileOffset), + path: null); + } + } + + private static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) + { + fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) + { + return FileStreamHelpers.CheckFileCall( + Interop.Sys.Pwrite(handle, bufPtr, buffer.Length, fileOffset), + path: null); + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs new file mode 100644 index 0000000000000..389041e1af4e9 --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.IO.Strategies; +using System.Runtime.InteropServices; +using System.Threading; +using Microsoft.Win32.SafeHandles; + +namespace System.IO +{ + public static partial class RandomAccess + { + internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) + { + Interop.Kernel32.FILE_STANDARD_INFO info; + + if (!Interop.Kernel32.GetFileInformationByHandleEx(handle, Interop.Kernel32.FileStandardInfo, &info, (uint)sizeof(Interop.Kernel32.FILE_STANDARD_INFO))) + { + throw Win32Marshal.GetExceptionForLastWin32Error(path); + } + + return info.EndOfFile; + } + + internal static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, long fileOffset, string? path = null) + { + NativeOverlapped nativeOverlapped = GetNativeOverlapped(fileOffset); + int r = ReadFileNative(handle, buffer, true, &nativeOverlapped, out int errorCode); + + if (r == -1) + { + // For pipes, ERROR_BROKEN_PIPE is the normal end of the pipe. + if (errorCode == Interop.Errors.ERROR_BROKEN_PIPE) + { + r = 0; + } + else + { + if (errorCode == Interop.Errors.ERROR_INVALID_PARAMETER) + { + ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); + } + + throw Win32Marshal.GetExceptionForWin32Error(errorCode, path); + } + } + + return r; + } + + internal static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset, string? path = null) + { + NativeOverlapped nativeOverlapped = GetNativeOverlapped(fileOffset); + int r = WriteFileNative(handle, buffer, true, &nativeOverlapped, out int errorCode); + + if (r == -1) + { + // For pipes, ERROR_NO_DATA is not an error, but the pipe is closing. + if (errorCode == Interop.Errors.ERROR_NO_DATA) + { + r = 0; + } + else + { + // ERROR_INVALID_PARAMETER may be returned for writes + // where the position is too large or for synchronous writes + // to a handle opened asynchronously. + if (errorCode == Interop.Errors.ERROR_INVALID_PARAMETER) + { + throw new IOException(SR.IO_FileTooLongOrHandleNotSync); + } + + throw Win32Marshal.GetExceptionForWin32Error(errorCode, path); + } + } + + return r; + } + + internal static unsafe int ReadFileNative(SafeFileHandle handle, Span bytes, bool syncUsingOverlapped, NativeOverlapped* overlapped, out int errorCode) + { + Debug.Assert(handle != null, "handle != null"); + + int r; + int numBytesRead = 0; + + fixed (byte* p = &MemoryMarshal.GetReference(bytes)) + { + r = overlapped != null ? + (syncUsingOverlapped + ? Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, overlapped) + : Interop.Kernel32.ReadFile(handle, p, bytes.Length, IntPtr.Zero, overlapped)) + : Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, IntPtr.Zero); + } + + if (r == 0) + { + errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); + + if (syncUsingOverlapped && errorCode == Interop.Errors.ERROR_HANDLE_EOF) + { + // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfile#synchronization-and-file-position : + // "If lpOverlapped is not NULL, then when a synchronous read operation reaches the end of a file, + // ReadFile returns FALSE and GetLastError returns ERROR_HANDLE_EOF" + return numBytesRead; + } + + return -1; + } + else + { + errorCode = 0; + return numBytesRead; + } + } + + internal static unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan buffer, bool syncUsingOverlapped, NativeOverlapped* overlapped, out int errorCode) + { + Debug.Assert(handle != null, "handle != null"); + + int numBytesWritten = 0; + int r; + + fixed (byte* p = &MemoryMarshal.GetReference(buffer)) + { + r = overlapped != null ? + (syncUsingOverlapped + ? Interop.Kernel32.WriteFile(handle, p, buffer.Length, out numBytesWritten, overlapped) + : Interop.Kernel32.WriteFile(handle, p, buffer.Length, IntPtr.Zero, overlapped)) + : Interop.Kernel32.WriteFile(handle, p, buffer.Length, out numBytesWritten, IntPtr.Zero); + } + + if (r == 0) + { + errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); + return -1; + } + else + { + errorCode = 0; + return numBytesWritten; + } + } + + private static NativeOverlapped GetNativeOverlapped(long fileOffset) + { + NativeOverlapped nativeOverlapped = default; + // For pipes the offsets are ignored by the OS + nativeOverlapped.OffsetLow = unchecked((int)fileOffset); + nativeOverlapped.OffsetHigh = (int)(fileOffset >> 32); + + return nativeOverlapped; + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs new file mode 100644 index 0000000000000..36a08417a530a --- /dev/null +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -0,0 +1,100 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Win32.SafeHandles; + +namespace System.IO +{ + public static partial class RandomAccess + { + /// + /// Gets the length in bytes of the file. + /// + /// The file handle. + /// A long value representing the length of the file in bytes. + /// is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + public static long GetLength(SafeFileHandle handle) + { + ValidateInput(handle, fileOffset: 0, mustBeSync: false); + + return GetFileLength(handle, path: null); + } + + /// + /// Reads a sequence of bytes from given file at given offset. + /// + /// The file handle. + /// A region of memory. When this method returns, the contents of this region are replaced by the bytes read from the file. + /// The file position to read from. + /// The total number of bytes read into the buffer. This can be less than the number of bytes allocated in the buffer if that many bytes are not currently available, or zero (0) if the end of the file has been reached. + /// is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static int Read(SafeFileHandle handle, Span buffer, long fileOffset) + { + ValidateInput(handle, fileOffset, mustBeSync: true); + + return ReadAtOffset(handle, buffer, fileOffset); + } + + /// + /// Writes a sequence of bytes from given buffer to given file at given offset. + /// + /// The file handle. + /// A region of memory. This method copies the contents of this region to the file. + /// The file position to write to. + /// The total number of bytes written into the file. This can be less than the number of bytes provided in the buffer and it's not an error. + /// is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static int Write(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) + { + ValidateInput(handle, fileOffset, mustBeSync: true); + + return WriteAtOffset(handle, buffer, fileOffset); + } + + private static void ValidateInput(SafeFileHandle handle, long fileOffset, bool mustBeSync) + { + if (handle is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.handle); + } + else if (handle.IsInvalid) + { + ThrowHelper.ThrowArgumentException_InvalidHandle(nameof(handle)); + } + else if (!handle.CanSeek) + { + // CanSeek calls IsClosed, we don't want to call it twice for valid handles + if (handle.IsClosed) + { + ThrowHelper.ThrowObjectDisposedException_FileClosed(); + } + + ThrowHelper.ThrowNotSupportedException_UnseekableStream(); + } + else if (mustBeSync && handle.IsAsync) + { + ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); + } + else if (fileOffset < 0) + { + ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(fileOffset)); + } + } + } +} diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs index 2d7822a04b1ca..58732496ea847 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs @@ -60,18 +60,6 @@ private static FileStreamStrategy ChooseStrategyCore(string path, FileMode mode, internal static FileStreamStrategy EnableBufferingIfNeeded(WindowsFileStreamStrategy strategy, int bufferSize) => bufferSize > 1 ? new BufferedFileStreamStrategy(strategy, bufferSize) : strategy; - internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) - { - Interop.Kernel32.FILE_STANDARD_INFO info; - - if (!Interop.Kernel32.GetFileInformationByHandleEx(handle, Interop.Kernel32.FileStandardInfo, &info, (uint)sizeof(Interop.Kernel32.FILE_STANDARD_INFO))) - { - throw Win32Marshal.GetExceptionForLastWin32Error(path); - } - - return info.EndOfFile; - } - internal static void FlushToDisk(SafeFileHandle handle, string? path) { if (!Interop.Kernel32.FlushFileBuffers(handle)) @@ -200,70 +188,7 @@ internal static unsafe void SetFileLength(SafeFileHandle handle, string? path, l } } - internal static unsafe int ReadFileNative(SafeFileHandle handle, Span bytes, bool syncUsingOverlapped, NativeOverlapped* overlapped, out int errorCode) - { - Debug.Assert(handle != null, "handle != null"); - - int r; - int numBytesRead = 0; - - fixed (byte* p = &MemoryMarshal.GetReference(bytes)) - { - r = overlapped != null ? - (syncUsingOverlapped - ? Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, overlapped) - : Interop.Kernel32.ReadFile(handle, p, bytes.Length, IntPtr.Zero, overlapped)) - : Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, IntPtr.Zero); - } - - if (r == 0) - { - errorCode = GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); - - if (syncUsingOverlapped && errorCode == Interop.Errors.ERROR_HANDLE_EOF) - { - // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfile#synchronization-and-file-position : - // "If lpOverlapped is not NULL, then when a synchronous read operation reaches the end of a file, - // ReadFile returns FALSE and GetLastError returns ERROR_HANDLE_EOF" - return numBytesRead; - } - - return -1; - } - else - { - errorCode = 0; - return numBytesRead; - } - } - - internal static unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan buffer, bool syncUsingOverlapped, NativeOverlapped* overlapped, out int errorCode) - { - Debug.Assert(handle != null, "handle != null"); - - int numBytesWritten = 0; - int r; - fixed (byte* p = &MemoryMarshal.GetReference(buffer)) - { - r = overlapped != null ? - (syncUsingOverlapped - ? Interop.Kernel32.WriteFile(handle, p, buffer.Length, out numBytesWritten, overlapped) - : Interop.Kernel32.WriteFile(handle, p, buffer.Length, IntPtr.Zero, overlapped)) - : Interop.Kernel32.WriteFile(handle, p, buffer.Length, out numBytesWritten, IntPtr.Zero); - } - - if (r == 0) - { - errorCode = GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); - return -1; - } - else - { - errorCode = 0; - return numBytesWritten; - } - } internal static async Task AsyncModeCopyToAsync(SafeFileHandle handle, string? path, bool canSeek, long filePosition, Stream destination, int bufferSize, CancellationToken cancellationToken) { @@ -340,7 +265,7 @@ internal static async Task AsyncModeCopyToAsync(SafeFileHandle handle, string? p } // Kick off the read. - synchronousSuccess = ReadFileNative(handle, copyBuffer, false, readAwaitable._nativeOverlapped, out errorCode) >= 0; + synchronousSuccess = RandomAccess.ReadFileNative(handle, copyBuffer, false, readAwaitable._nativeOverlapped, out errorCode) >= 0; } // If the operation did not synchronously succeed, it either failed or initiated the asynchronous operation. diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs index 50a6bb262c9ba..c2b2bbfeb158a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs @@ -86,11 +86,11 @@ internal static void ValidateArguments(string path, FileMode mode, FileAccess ac } else if (bufferSize < 0) { - throw new ArgumentOutOfRangeException(nameof(bufferSize), SR.ArgumentOutOfRange_NeedPosNum); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(bufferSize)); } else if (preallocationSize < 0) { - throw new ArgumentOutOfRangeException(nameof(preallocationSize), SR.ArgumentOutOfRange_NeedNonNegNum); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(preallocationSize)); } // Write access validation diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs index 1100ff4329fc9..56716deb0ccbc 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs @@ -66,9 +66,7 @@ public override long Length get { // Get the length of the file as reported by the OS - Interop.Sys.FileStatus status; - CheckFileCall(Interop.Sys.FStat(_fileHandle, out status)); - long length = status.Size; + long length = RandomAccess.GetFileLength(_fileHandle, _path); // But we may have buffered some data to be written that puts our length // beyond what the OS is aware of. Update accordingly. diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs index fccd0232ecddc..ec74ccf1ac497 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Windows.cs @@ -98,7 +98,7 @@ public unsafe override long Length { get { - long len = FileStreamHelpers.GetFileLength(_fileHandle, _path); + long len = RandomAccess.GetFileLength(_fileHandle, _path); // If we're writing near the end of the file, we must include our // internal buffer in our Length calculation. Don't flush because @@ -1005,14 +1005,14 @@ private unsafe int ReadFileNative(SafeFileHandle handle, Span bytes, Nativ { Debug.Assert((_useAsyncIO && overlapped != null) || (!_useAsyncIO && overlapped == null), "Async IO and overlapped parameters inconsistent in call to ReadFileNative."); - return FileStreamHelpers.ReadFileNative(handle, bytes, false, overlapped, out errorCode); + return RandomAccess.ReadFileNative(handle, bytes, false, overlapped, out errorCode); } private unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan buffer, NativeOverlapped* overlapped, out int errorCode) { Debug.Assert((_useAsyncIO && overlapped != null) || (!_useAsyncIO && overlapped == null), "Async IO and overlapped parameters inconsistent in call to WriteFileNative."); - return FileStreamHelpers.WriteFileNative(handle, buffer, false, overlapped, out errorCode); + return RandomAccess.WriteFileNative(handle, buffer, false, overlapped, out errorCode); } public override Task CopyToAsync(Stream destination, int bufferSize, CancellationToken cancellationToken) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs index 37ac4ae622321..924e8e6778960 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/SyncWindowsFileStreamStrategy.cs @@ -90,25 +90,8 @@ private unsafe int ReadSpan(Span destination) Debug.Assert(!_fileHandle.IsClosed, "!_handle.IsClosed"); - NativeOverlapped nativeOverlapped = GetNativeOverlappedForCurrentPosition(); - int r = FileStreamHelpers.ReadFileNative(_fileHandle, destination, true, &nativeOverlapped, out int errorCode); - - if (r == -1) - { - // For pipes, ERROR_BROKEN_PIPE is the normal end of the pipe. - if (errorCode == Interop.Errors.ERROR_BROKEN_PIPE) - { - r = 0; - } - else - { - if (errorCode == Interop.Errors.ERROR_INVALID_PARAMETER) - ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(_fileHandle)); - - throw Win32Marshal.GetExceptionForWin32Error(errorCode, _path); - } - } - Debug.Assert(r >= 0, "FileStream's ReadNative is likely broken."); + int r = RandomAccess.ReadAtOffset(_fileHandle, destination, _filePosition, _path); + Debug.Assert(r >= 0, "RandomAccess.ReadAtOffsete is likely broken."); _filePosition += r; return r; @@ -123,39 +106,11 @@ private unsafe void WriteSpan(ReadOnlySpan source) Debug.Assert(!_fileHandle.IsClosed, "!_handle.IsClosed"); - NativeOverlapped nativeOverlapped = GetNativeOverlappedForCurrentPosition(); - int r = FileStreamHelpers.WriteFileNative(_fileHandle, source, true, &nativeOverlapped, out int errorCode); - - if (r == -1) - { - // For pipes, ERROR_NO_DATA is not an error, but the pipe is closing. - if (errorCode == Interop.Errors.ERROR_NO_DATA) - { - r = 0; - } - else - { - // ERROR_INVALID_PARAMETER may be returned for writes - // where the position is too large or for synchronous writes - // to a handle opened asynchronously. - if (errorCode == Interop.Errors.ERROR_INVALID_PARAMETER) - throw new IOException(SR.IO_FileTooLongOrHandleNotSync); - throw Win32Marshal.GetExceptionForWin32Error(errorCode, _path); - } - } - Debug.Assert(r >= 0, "FileStream's WriteCore is likely broken."); + int r = RandomAccess.WriteAtOffset(_fileHandle, source, _filePosition, _path); + Debug.Assert(r >= 0, "RandomAccess.WriteAtOffset is likely broken."); _filePosition += r; - UpdateLengthOnChangePosition(); - } - private NativeOverlapped GetNativeOverlappedForCurrentPosition() - { - NativeOverlapped nativeOverlapped = default; - // For pipes the offsets are ignored by the OS - nativeOverlapped.OffsetLow = unchecked((int)_filePosition); - nativeOverlapped.OffsetHigh = (int)(_filePosition >> 32); - - return nativeOverlapped; + UpdateLengthOnChangePosition(); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs index bd6e8a6013d98..dee1c5a954857 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/WindowsFileStreamStrategy.cs @@ -80,12 +80,12 @@ public unsafe sealed override long Length { if (_share > FileShare.Read || _exposedHandle) { - return FileStreamHelpers.GetFileLength(_fileHandle, _path); + return RandomAccess.GetFileLength(_fileHandle, _path); } if (_length < 0) { - _length = FileStreamHelpers.GetFileLength(_fileHandle, _path); + _length = RandomAccess.GetFileLength(_fileHandle, _path); } return _length; diff --git a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs index d061bddce9cab..97bd5e7c109d5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @@ -396,6 +396,12 @@ internal static void ThrowArgumentException_Argument_InvalidArrayType() throw new ArgumentException(SR.Argument_InvalidArrayType); } + [DoesNotReturn] + internal static void ThrowArgumentException_InvalidHandle(string? paramName) + { + throw new ArgumentException(SR.Arg_InvalidHandle, paramName); + } + [DoesNotReturn] internal static void ThrowInvalidOperationException_InvalidOperation_EnumNotStarted() { @@ -474,6 +480,12 @@ internal static void ThrowArgumentOutOfRangeException_SymbolDoesNotFit() throw new ArgumentOutOfRangeException("symbol", SR.Argument_BadFormatSpecifier); } + [DoesNotReturn] + internal static void ThrowArgumentOutOfRangeException_NeedPosNum(string? paramName) + { + throw new ArgumentOutOfRangeException(paramName, SR.ArgumentOutOfRange_NeedPosNum); + } + private static Exception GetArraySegmentCtorValidationFailedException(Array? array, int offset, int count) { if (array == null) @@ -733,6 +745,8 @@ private static string GetArgumentName(ExceptionArgument argument) return "destinationArray"; case ExceptionArgument.pHandle: return "pHandle"; + case ExceptionArgument.handle: + return "handle"; case ExceptionArgument.other: return "other"; case ExceptionArgument.newSize: @@ -1022,6 +1036,7 @@ internal enum ExceptionArgument destinationIndex, destinationArray, pHandle, + handle, other, newSize, lowerBounds, diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 0faf10fbc23ab..554bfdee9ca91 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -8073,6 +8073,12 @@ public override void Write(System.ReadOnlySpan buffer) { } public override System.Threading.Tasks.ValueTask WriteAsync(System.ReadOnlyMemory buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public override void WriteByte(byte value) { } } + public static partial class RandomAccess + { + public static long GetLength(Microsoft.Win32.SafeHandles.SafeFileHandle handle) { throw null; } + public static int Read(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Span buffer, long fileOffset) { throw null; } + public static int Write(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlySpan buffer, long fileOffset) { throw null; } + } } namespace System.IO.Enumeration { From 1d41d65697a1ca2d98390cc22f0839f6339fc8fc Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 31 May 2021 21:52:50 +0200 Subject: [PATCH 25/71] fix Unix build --- .../{Interop.Pread.cs => Interop.PRead.cs} | 4 ++-- .../{Interop.Pwrite.cs => Interop.PWrite.cs} | 4 ++-- .../Native/Unix/System.Native/entrypoints.c | 2 ++ .../Native/Unix/System.Native/pal_io.c | 10 +++++----- .../Native/Unix/System.Native/pal_io.h | 4 ++-- .../System.Private.CoreLib.Shared.projitems | 8 ++++---- .../src/System/IO/RandomAccess.Unix.cs | 20 +++++++++---------- 7 files changed, 26 insertions(+), 26 deletions(-) rename src/libraries/Common/src/Interop/Unix/System.Native/{Interop.Pread.cs => Interop.PRead.cs} (78%) rename src/libraries/Common/src/Interop/Unix/System.Native/{Interop.Pwrite.cs => Interop.PWrite.cs} (79%) diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs similarity index 78% rename from src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs rename to src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs index 4b25d9f18e631..7b95ded24a54c 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pread.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs @@ -7,7 +7,7 @@ internal static partial class Interop { internal static partial class Sys { - [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Pread", SetLastError = true)] - internal static extern unsafe int Pread(SafeHandle fd, byte* buffer, int count, long fileOffset); + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PRead", SetLastError = true)] + internal static extern unsafe int PRead(SafeHandle fd, byte* buffer, int count, long fileOffset); } } diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWrite.cs similarity index 79% rename from src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs rename to src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWrite.cs index 8f1bfa032615b..721a1c8706fd7 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.Pwrite.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWrite.cs @@ -7,7 +7,7 @@ internal static partial class Interop { internal static partial class Sys { - [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_Pwrite", SetLastError = true)] - internal static extern unsafe int Pwrite(SafeHandle fd, byte* buffer, int bufferSize, long fileOffset); + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PWrite", SetLastError = true)] + internal static extern unsafe int PWrite(SafeHandle fd, byte* buffer, int bufferSize, long fileOffset); } } diff --git a/src/libraries/Native/Unix/System.Native/entrypoints.c b/src/libraries/Native/Unix/System.Native/entrypoints.c index 0f7b1a12fa507..fbaff8cfae7e9 100644 --- a/src/libraries/Native/Unix/System.Native/entrypoints.c +++ b/src/libraries/Native/Unix/System.Native/entrypoints.c @@ -242,6 +242,8 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_iOSSupportVersion) DllImportEntry(SystemNative_GetErrNo) DllImportEntry(SystemNative_SetErrNo) + DllImportEntry(SystemNative_PRead) + DllImportEntry(SystemNative_PWrite) }; EXTERN_C const void* SystemResolveDllImport(const char* name); diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 14d986d9715d6..ae8cc24c19df4 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1455,11 +1455,11 @@ int32_t SystemNative_ReadProcessStatusInfo(pid_t pid, ProcessStatus* processStat #endif // __sun } -int32_t SystemNative_Pread(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) +int32_t SystemNative_PRead(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) { assert(buffer != NULL); assert(bufferSize >= 0); - + ssize_t count; while ((count = pread(ToFileDescriptor(fd), buffer, (uint32_t)bufferSize, (off_t)fileOffset)) < 0 && errno == EINTR); @@ -1467,14 +1467,14 @@ int32_t SystemNative_Pread(intptr_t fd, void* buffer, int32_t bufferSize, int64_ return (int32_t)count; } -int32_t SystemNative_Pwrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) +int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset) { assert(buffer != NULL); assert(bufferSize >= 0); - + ssize_t count; while ((count = pwrite(ToFileDescriptor(fd), buffer, (uint32_t)bufferSize, (off_t)fileOffset)) < 0 && errno == EINTR); assert(count >= -1 && count <= bufferSize); return (int32_t)count; -} \ No newline at end of file +} diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index 6faccdff5f05e..95a0c89bcbf9f 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -736,11 +736,11 @@ PALEXPORT int32_t SystemNative_ReadProcessStatusInfo(pid_t pid, ProcessStatus* p * * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. */ -PALEXPORT int32_t SystemNative_Pread(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); +PALEXPORT int32_t SystemNative_PRead(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); /** * Writes the number of bytes specified in the buffer into the specified, opened file descriptor at specified offset. * * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. */ -PALEXPORT int32_t SystemNative_Pwrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); +PALEXPORT int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index a0ff9077393d3..e07893b4d7fdd 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1969,11 +1969,11 @@ Common\Interop\Unix\System.Native\Interop.PosixFAllocate.cs - - Common\Interop\Unix\System.Native\Interop.Pread.cs + + Common\Interop\Unix\System.Native\Interop.PRead.cs - - Common\Interop\Unix\System.Native\Interop.Pwrite.cs + + Common\Interop\Unix\System.Native\Interop.PWrite.cs Common\Interop\Unix\System.Native\Interop.Read.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 858c9a47453c3..0ccc823926453 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -1,10 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System; -using System.Diagnostics; -using System.IO; using System.IO.Strategies; +using System.Runtime.InteropServices; using Microsoft.Win32.SafeHandles; namespace System.IO @@ -14,17 +12,17 @@ public static partial class RandomAccess internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) { int result = Interop.Sys.FStat(handle, out Interop.Sys.FileStatus status); - FileStreamHelpers.CheckFileCall(returnCode, path); - return status.Size + FileStreamHelpers.CheckFileCall(result, path); + return status.Size; } private static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, long fileOffset) { fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) { - return FileStreamHelpers.CheckFileCall( - Interop.Sys.Pread(handle, bufPtr, buffer.Length, fileOffset), - path: null); + int result = Interop.Sys.PRead(handle, bufPtr, buffer.Length, fileOffset); + FileStreamHelpers.CheckFileCall(result, path: null); + return result; } } @@ -32,9 +30,9 @@ private static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan Date: Tue, 1 Jun 2021 09:10:54 +0200 Subject: [PATCH 26/71] refactor the tests --- .../tests/RandomAccess/Base.cs | 94 +++++++++++++++++++ .../tests/RandomAccess/GetLength.cs | 39 +------- .../tests/RandomAccess/Read.cs | 62 ++---------- .../tests/RandomAccess/Write.cs | 63 ++----------- .../tests/System.IO.FileSystem.Tests.csproj | 1 + .../src/System/IO/RandomAccess.cs | 4 +- 6 files changed, 113 insertions(+), 150 deletions(-) create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs new file mode 100644 index 0000000000000..8d585b72c1443 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.IO.Pipes; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public abstract class RandomAccess_Base : FileSystemTest + { + protected abstract T MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset); + + protected virtual bool ShouldThrowForSyncHandle => false; + + protected virtual bool ShouldThrowForAsyncHandle => false; + + protected virtual bool UsesOffsets => true; + + [Fact] + public void ThrowsArgumentNullExceptionForNullHandle() + { + ArgumentNullException ex = Assert.Throws(() => MethodUnderTest(null, Array.Empty(), 0)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsArgumentExceptionForInvalidHandle() + { + SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); + + ArgumentException ex = Assert.Throws(() => MethodUnderTest(handle, Array.Empty(), 0)); + Assert.Equal("handle", ex.ParamName); + } + + [Fact] + public void ThrowsObjectDisposedExceptionForDisposedHandle() + { + SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); + handle.Dispose(); + + Assert.Throws(() => MethodUnderTest(handle, Array.Empty(), 0)); + } + + [Fact] + public void ThrowsNotSupportedExceptionForUnseekableFile() + { + using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) + using (SafeFileHandle pipeHandle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) + { + Assert.Throws(() => MethodUnderTest(pipeHandle, Array.Empty(), 0)); + } + } + + [Fact] + public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() + { + if (UsesOffsets) + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + { + ArgumentOutOfRangeException ex = Assert.Throws(() => MethodUnderTest(validHandle, Array.Empty(), -1)); + Assert.Equal("fileOffset", ex.ParamName); + } + } + } + + [Fact] + public void ThrowsArgumentExceptionForAsyncFileHandle() + { + if (ShouldThrowForAsyncHandle) + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) + { + ArgumentException ex = Assert.Throws(() => MethodUnderTest(validHandle, new byte[100], 0)); + Assert.Equal("handle", ex.ParamName); + } + } + } + + [Fact] + public void ThrowsArgumentExceptionForSyncFileHandle() + { + if (ShouldThrowForSyncHandle) + { + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.None)) + { + ArgumentException ex = Assert.Throws(() => MethodUnderTest(validHandle, new byte[100], 0)); + Assert.Equal("handle", ex.ParamName); + } + } + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs index a01e0becf885d..46e2914aed167 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/GetLength.cs @@ -1,48 +1,17 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.IO.Pipes; using Microsoft.Win32.SafeHandles; using Xunit; namespace System.IO.Tests { - public class RandomAccess_GetLength : FileSystemTest + public class RandomAccess_GetLength : RandomAccess_Base { - [Fact] - public void ThrowsArgumentNullExceptionForNullHandle() - { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.GetLength(null)); - Assert.Equal("handle", ex.ParamName); - } + protected override long MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.GetLength(handle); - [Fact] - public void ThrowsArgumentExceptionForInvalidHandle() - { - SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); - - ArgumentException ex = Assert.Throws(() => RandomAccess.GetLength(handle)); - Assert.Equal("handle", ex.ParamName); - } - - [Fact] - public void ThrowsObjectDisposedExceptionForDisposedHandle() - { - SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); - handle.Dispose(); - - Assert.Throws(() => RandomAccess.GetLength(handle)); - } - - [Fact] - public void ThrowsNotSupportedExceptionForUnseekableFile() - { - using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) - using (SafeFileHandle handle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) - { - Assert.Throws(() => RandomAccess.GetLength(handle)); - } - } + protected override bool UsesOffsets => false; [Fact] public void ReturnsZeroForEmptyFile() diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs index 3b06c15322642..659e3df42d345 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs @@ -1,72 +1,22 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.IO.Pipes; using System.Linq; using Microsoft.Win32.SafeHandles; using Xunit; namespace System.IO.Tests { - public class RandomAccess_Read : FileSystemTest + public class RandomAccess_Read : RandomAccess_Base { - [Fact] - public void ThrowsArgumentNullExceptionForNullHandle() - { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.Read(null, Array.Empty(), 0)); - Assert.Equal("handle", ex.ParamName); - } - - [Fact] - public void ThrowsArgumentExceptionForInvalidHandle() - { - SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); - - ArgumentException ex = Assert.Throws(() => RandomAccess.Read(handle, Array.Empty(), 0)); - Assert.Equal("handle", ex.ParamName); - } - - [Fact] - public void ThrowsObjectDisposedExceptionForDisposedHandle() - { - SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); - handle.Dispose(); - - Assert.Throws(() => RandomAccess.Read(handle, Array.Empty(), 0)); - } - - [Fact] - public void ThrowsNotSupportedExceptionForUnseekableFile() - { - using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) - using (SafeFileHandle pipeHandle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) - { - Assert.Throws(() => RandomAccess.Read(pipeHandle, Array.Empty(), 0)); - } - } + protected override int MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.Read(handle, bytes, fileOffset); - [Fact] - public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() - { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) - { - ArgumentOutOfRangeException ex = Assert.Throws(() => RandomAccess.Read(validHandle, Array.Empty(), -1)); - Assert.Equal("fileOffset", ex.ParamName); - } - } - - [Fact] - public void ThrowsArgumentExceptionForAsyncFileHandle() - { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) - { - ArgumentException ex = Assert.Throws(() => RandomAccess.Read(validHandle, new byte[100], 0)); - Assert.Equal("handle", ex.ParamName); - } - } + protected override bool ShouldThrowForAsyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform sync IO using async handle [Fact] - public void SupportsPartialReads() + public void HappyPath() { const int fileSize = 4_001; string filePath = GetTestFilePath(); diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs index 38c6f3ef32821..ee03631d8de0b 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs @@ -1,72 +1,21 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.IO.Pipes; -using System.Linq; using Microsoft.Win32.SafeHandles; using Xunit; namespace System.IO.Tests { - public class RandomAccess_Write : FileSystemTest + public class RandomAccess_Write : RandomAccess_Base { - [Fact] - public void ThrowsArgumentNullExceptionForNullHandle() - { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.Write(null, Array.Empty(), 0)); - Assert.Equal("handle", ex.ParamName); - } - - [Fact] - public void ThrowsArgumentExceptionForInvalidHandle() - { - SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); - - ArgumentException ex = Assert.Throws(() => RandomAccess.Write(handle, Array.Empty(), 0)); - Assert.Equal("handle", ex.ParamName); - } - - [Fact] - public void ThrowsObjectDisposedExceptionForDisposedHandle() - { - SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write); - handle.Dispose(); - - Assert.Throws(() => RandomAccess.Write(handle, Array.Empty(), 0)); - } - - [Fact] - public void ThrowsNotSupportedExceptionForUnseekableFile() - { - using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) - using (SafeFileHandle pipeHandle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) - { - Assert.Throws(() => RandomAccess.Write(pipeHandle, Array.Empty(), 0)); - } - } + protected override int MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.Write(handle, bytes, fileOffset); - [Fact] - public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() - { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) - { - ArgumentOutOfRangeException ex = Assert.Throws(() => RandomAccess.Write(validHandle, Array.Empty(), -1)); - Assert.Equal("fileOffset", ex.ParamName); - } - } - - [Fact] - public void ThrowsArgumentExceptionForAsyncFileHandle() - { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) - { - ArgumentException ex = Assert.Throws(() => RandomAccess.Write(validHandle, new byte[100], 0)); - Assert.Equal("handle", ex.ParamName); - } - } + protected override bool ShouldThrowForAsyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform sync IO using async handle [Fact] - public void SupportsPartialWrites() + public void HappyPath() { const int fileSize = 4_001; string filePath = GetTestFilePath(); diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 1fda5d01db858..d8ba62917e902 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -51,6 +51,7 @@ + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 36a08417a530a..d10b8d5e83158 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -40,7 +40,7 @@ public static long GetLength(SafeFileHandle handle) /// Position of the file is not advanced. public static int Read(SafeFileHandle handle, Span buffer, long fileOffset) { - ValidateInput(handle, fileOffset, mustBeSync: true); + ValidateInput(handle, fileOffset, mustBeSync: OperatingSystem.IsWindows()); return ReadAtOffset(handle, buffer, fileOffset); } @@ -62,7 +62,7 @@ public static int Read(SafeFileHandle handle, Span buffer, long fileOffset /// Position of the file is not advanced. public static int Write(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) { - ValidateInput(handle, fileOffset, mustBeSync: true); + ValidateInput(handle, fileOffset, mustBeSync: OperatingSystem.IsWindows()); return WriteAtOffset(handle, buffer, fileOffset); } From bbbd2677f1bc24cc6f08ceb26c24a8ff8bfe1d48 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 1 Jun 2021 09:11:54 +0200 Subject: [PATCH 27/71] ReadAsync and WriteAsync --- .../tests/RandomAccess/Base.cs | 3 +- .../tests/RandomAccess/ReadAsync.cs | 49 ++++++++++++++++ .../tests/RandomAccess/WriteAsync.cs | 42 ++++++++++++++ .../tests/System.IO.FileSystem.Tests.csproj | 2 + .../src/System/IO/FileStream.cs | 2 +- .../src/System/IO/RandomAccess.Windows.cs | 11 ++++ .../src/System/IO/RandomAccess.cs | 56 ++++++++++++++++++- .../src/System/ThrowHelper.cs | 6 ++ .../System.Runtime/ref/System.Runtime.cs | 2 + 9 files changed, 169 insertions(+), 4 deletions(-) create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index 8d585b72c1443..a4044b1442128 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -57,7 +57,8 @@ public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() { if (UsesOffsets) { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + FileOptions options = ShouldThrowForAsyncHandle ? FileOptions.None : FileOptions.Asynchronous; + using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: options)) { ArgumentOutOfRangeException ex = Assert.Throws(() => MethodUnderTest(validHandle, Array.Empty(), -1)); Assert.Equal("fileOffset", ex.ParamName); diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs new file mode 100644 index 0000000000000..7b8a1011c5968 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs @@ -0,0 +1,49 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_ReadAsync : RandomAccess_Base> + { + protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.ReadAsync(handle, bytes, fileOffset); + + protected override bool ShouldThrowForSyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle + + [Fact] + public async Task HappyPath() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] expected = new byte[fileSize]; + new Random().NextBytes(expected); + File.WriteAllBytes(filePath, expected); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) + { + byte[] actual = new byte[fileSize + 1]; + int current = 0; + int total = 0; + + do + { + current = await RandomAccess.ReadAsync( + handle, + actual.AsMemory(total, Math.Min(actual.Length - total, fileSize / 4)), + fileOffset: total); + + total += current; + } while (current != 0); + + Assert.Equal(fileSize, total); + Assert.Equal(expected, actual.Take(total).ToArray()); + } + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs new file mode 100644 index 0000000000000..a0f96cf31d108 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs @@ -0,0 +1,42 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading.Tasks; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_WriteAsync : RandomAccess_Base> + { + protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.WriteAsync(handle, bytes, fileOffset); + + protected override bool ShouldThrowForSyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle + + [Fact] + public async Task HappyPath() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] content = new byte[fileSize]; + new Random().NextBytes(content); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous)) + { + int total = 0; + + while (total != fileSize) + { + total += await RandomAccess.WriteAsync( + handle, + content.AsMemory(total, Math.Min(content.Length - total, fileSize / 4)), + fileOffset: total); + } + } + + Assert.Equal(content, File.ReadAllBytes(filePath)); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index d8ba62917e902..4ca822791aee7 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -54,7 +54,9 @@ + + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs index 1a149d6f295ad..18de4e27955e5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs @@ -90,7 +90,7 @@ private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int if (isAsync && !handle.IsAsync) { - throw new ArgumentException(SR.Arg_HandleNotAsync, nameof(handle)); + ThrowHelper.ThrowArgumentException_HandleNotAsync(nameof(handle)); } else if (!isAsync && handle.IsAsync) { diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 389041e1af4e9..6f25011badae8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -5,6 +5,7 @@ using System.IO.Strategies; using System.Runtime.InteropServices; using System.Threading; +using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; namespace System.IO @@ -143,6 +144,16 @@ internal static unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) + { + + } + + private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) + { + + } + private static NativeOverlapped GetNativeOverlapped(long fileOffset) { NativeOverlapped nativeOverlapped = default; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index d10b8d5e83158..8508a5d748293 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Threading; +using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; namespace System.IO @@ -18,7 +20,7 @@ public static partial class RandomAccess /// The file does not support seeking (pipe or socket). public static long GetLength(SafeFileHandle handle) { - ValidateInput(handle, fileOffset: 0, mustBeSync: false); + ValidateInput(handle, fileOffset: 0); return GetFileLength(handle, path: null); } @@ -67,7 +69,53 @@ public static int Write(SafeFileHandle handle, ReadOnlySpan buffer, long f return WriteAtOffset(handle, buffer, fileOffset); } - private static void ValidateInput(SafeFileHandle handle, long fileOffset, bool mustBeSync) + /// + /// Reads a sequence of bytes from given file at given offset. + /// + /// The file handle. + /// A region of memory. When this method returns, the contents of this region are replaced by the bytes read from the file. + /// The file position to read from. + /// The token to monitor for cancellation requests. The default value is . + /// The total number of bytes read into the buffer. This can be less than the number of bytes allocated in the buffer if that many bytes are not currently available, or zero (0) if the end of the file has been reached. + /// is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was not opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken = default) + { + ValidateInput(handle, fileOffset, mustBeAsync: true); + + return ReadAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); + } + + /// + /// Writes a sequence of bytes from given buffer to given file at given offset. + /// + /// The file handle. + /// A region of memory. This method copies the contents of this region to the file. + /// The file position to write to. + /// The token to monitor for cancellation requests. The default value is . + /// The total number of bytes written into the file. This can be less than the number of bytes provided in the buffer and it's not an error. + /// is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was not opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken = default) + { + ValidateInput(handle, fileOffset, mustBeAsync: true); + + return WriteAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); + } + + private static void ValidateInput(SafeFileHandle handle, long fileOffset, bool mustBeSync = false, bool mustBeAsync = false) { if (handle is null) { @@ -91,6 +139,10 @@ private static void ValidateInput(SafeFileHandle handle, long fileOffset, bool m { ThrowHelper.ThrowArgumentException_HandleNotSync(nameof(handle)); } + else if (mustBeAsync && !handle.IsAsync) + { + ThrowHelper.ThrowArgumentException_HandleNotAsync(nameof(handle)); + } else if (fileOffset < 0) { ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(fileOffset)); diff --git a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs index 97bd5e7c109d5..62716599d0460 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @@ -231,6 +231,12 @@ internal static void ThrowArgumentException_HandleNotSync(string paramName) throw new ArgumentException(SR.Arg_HandleNotSync, paramName); } + [DoesNotReturn] + internal static void ThrowArgumentException_HandleNotAsync(string paramName) + { + throw new ArgumentException(SR.Arg_HandleNotAsync, paramName); + } + [DoesNotReturn] internal static void ThrowArgumentNullException(ExceptionArgument argument) { diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 554bfdee9ca91..2f6e0abd48721 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -8077,7 +8077,9 @@ public static partial class RandomAccess { public static long GetLength(Microsoft.Win32.SafeHandles.SafeFileHandle handle) { throw null; } public static int Read(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Span buffer, long fileOffset) { throw null; } + public static System.Threading.Tasks.ValueTask ReadAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Memory buffer, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static int Write(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlySpan buffer, long fileOffset) { throw null; } + public static System.Threading.Tasks.ValueTask WriteAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlyMemory buffer, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } } } namespace System.IO.Enumeration From 46d9b6fa6667ca0c365436d7bb96927e5dbd55ab Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 1 Jun 2021 10:07:52 +0200 Subject: [PATCH 28/71] move AsyncWindowsFileStreamStrategy.ValueTaskSource to SafeFileHandle --- ...SafeFileHandle.ValueTaskSource.Windows.cs} | 62 +++++++++++--- .../SafeHandles/SafeFileHandle.Windows.cs | 4 +- .../System.Private.CoreLib.Shared.projitems | 2 +- .../src/System/IO/RandomAccess.Windows.cs | 81 ++++++++++++++++++- .../AsyncWindowsFileStreamStrategy.cs | 63 ++------------- 5 files changed, 138 insertions(+), 74 deletions(-) rename src/libraries/System.Private.CoreLib/src/{System/IO/Strategies/AsyncWindowsFileStreamStrategy.ValueTaskSource.cs => Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs} (75%) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.ValueTaskSource.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs similarity index 75% rename from src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.ValueTaskSource.cs rename to src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs index f1153a5cc041c..6d20f76b6b175 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.ValueTaskSource.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs @@ -1,22 +1,49 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System; using System.Buffers; using System.Diagnostics; +using System.IO; using System.Threading; using System.Threading.Tasks.Sources; -namespace System.IO.Strategies +namespace Microsoft.Win32.SafeHandles { - internal sealed partial class AsyncWindowsFileStreamStrategy : WindowsFileStreamStrategy + public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { + private ValueTaskSource? _reusableValueTaskSource; // reusable ValueTaskSource that is currently NOT being used + + // Rent the reusable ValueTaskSource, or create a new one to use if we couldn't get one (which + // should only happen on first use or if the FileStream is being used concurrently). + internal ValueTaskSource GetValueTaskSource() => Interlocked.Exchange(ref _reusableValueTaskSource, null) ?? new ValueTaskSource(this); + + protected override bool ReleaseHandle() + { + bool result = Interop.Kernel32.CloseHandle(handle); + + Interlocked.Exchange(ref _reusableValueTaskSource, null)?.Dispose(); + + return result; + } + + private void TryToReuse(ValueTaskSource source) + { + source._source.Reset(); + + if (Interlocked.CompareExchange(ref _reusableValueTaskSource, source, null) is not null) + { + source._preallocatedOverlapped.Dispose(); + } + } + /// Reusable IValueTaskSource for FileStream ValueTask-returning async operations. - private sealed unsafe class ValueTaskSource : IValueTaskSource, IValueTaskSource + internal sealed unsafe class ValueTaskSource : IValueTaskSource, IValueTaskSource { internal static readonly IOCompletionCallback s_ioCallback = IOCallback; internal readonly PreAllocatedOverlapped _preallocatedOverlapped; - private readonly AsyncWindowsFileStreamStrategy _strategy; + private readonly SafeFileHandle _fileHandle; internal MemoryHandle _memoryHandle; internal ManualResetValueTaskSourceCore _source; // mutable struct; do not make this readonly private NativeOverlapped* _overlapped; @@ -28,9 +55,9 @@ private sealed unsafe class ValueTaskSource : IValueTaskSource, IValueTaskS /// internal ulong _result; - internal ValueTaskSource(AsyncWindowsFileStreamStrategy strategy) + internal ValueTaskSource(SafeFileHandle fileHandle) { - _strategy = strategy; + _fileHandle = fileHandle; _source.RunContinuationsAsynchronously = true; _preallocatedOverlapped = PreAllocatedOverlapped.UnsafeCreate(s_ioCallback, this, null); } @@ -41,11 +68,18 @@ internal void Dispose() _preallocatedOverlapped.Dispose(); } - internal NativeOverlapped* PrepareForOperation(ReadOnlyMemory memory) + internal static Exception GetIOError(int errorCode, string? path) + => errorCode == Interop.Errors.ERROR_HANDLE_EOF + ? ThrowHelper.CreateEndOfFileException() + : Win32Marshal.GetExceptionForWin32Error(errorCode, path); + + internal NativeOverlapped* PrepareForOperation(ReadOnlyMemory memory, long fileOffset) { _result = 0; _memoryHandle = memory.Pin(); - _overlapped = _strategy._fileHandle.ThreadPoolBinding!.AllocateNativeOverlapped(_preallocatedOverlapped); + _overlapped = _fileHandle.ThreadPoolBinding!.AllocateNativeOverlapped(_preallocatedOverlapped); + _overlapped->OffsetLow = (int)fileOffset; + _overlapped->OffsetHigh = (int)(fileOffset >> 32); return _overlapped; } @@ -63,7 +97,7 @@ private int GetResultAndRelease(short token) finally { // The instance is ready to be reused - _strategy.TryToReuse(this); + _fileHandle.TryToReuse(this); } } @@ -79,11 +113,11 @@ internal void RegisterForCancellation(CancellationToken cancellationToken) _cancellationRegistration = cancellationToken.UnsafeRegister(static (s, token) => { ValueTaskSource vts = (ValueTaskSource)s!; - if (!vts._strategy._fileHandle.IsInvalid) + if (!vts._fileHandle.IsInvalid) { try { - Interop.Kernel32.CancelIoEx(vts._strategy._fileHandle, vts._overlapped); + Interop.Kernel32.CancelIoEx(vts._fileHandle, vts._overlapped); // Ignore all failures: no matter whether it succeeds or fails, completion is handled via the IOCallback. } catch (ObjectDisposedException) { } // in case the SafeHandle is (erroneously) closed concurrently @@ -112,7 +146,7 @@ internal void ReleaseResources() // Free the overlapped. if (_overlapped != null) { - _strategy._fileHandle.ThreadPoolBinding!.FreeNativeOverlapped(_overlapped); + _fileHandle.ThreadPoolBinding!.FreeNativeOverlapped(_overlapped); _overlapped = null; } } @@ -161,6 +195,10 @@ internal void Complete(uint errorCode, uint numBytes) case 0: case Interop.Errors.ERROR_BROKEN_PIPE: case Interop.Errors.ERROR_NO_DATA: + // AsyncWindowsFileStreamStrategy checks the Length before calling ReadFile + // and tries to perform a 0-byte read at EOF. + // RandomAccess does not and we need to treat EOF as a successfull operation. + case Interop.Errors.ERROR_HANDLE_EOF: // Success _source.SetResult((int)numBytes); break; diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index c475bf8f5c04c..1b81e456994f4 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -9,7 +9,7 @@ namespace Microsoft.Win32.SafeHandles { - public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid + public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { private const FileOptions NoBuffering = (FileOptions)(0x20000000); private FileOptions _fileOptions = (FileOptions)(-1); @@ -40,8 +40,6 @@ private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fi internal ThreadPoolBoundHandle? ThreadPoolBinding { get; set; } - protected override bool ReleaseHandle() => Interop.Kernel32.CloseHandle(handle); - internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) { using (DisableMediaInsertionPrompt.Create()) diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index e07893b4d7fdd..6a90fd69c190d 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1750,6 +1750,7 @@ + @@ -1781,7 +1782,6 @@ - diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 6f25011badae8..3e4cef9449bb1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -144,14 +144,91 @@ internal static unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) + private static unsafe ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) { + SafeFileHandle.ValueTaskSource vts = handle.GetValueTaskSource(); + try + { + NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(buffer, fileOffset); + Debug.Assert(vts._memoryHandle.Pointer != null); + + // Queue an async ReadFile operation. + if (Interop.Kernel32.ReadFile(handle, (byte*)vts._memoryHandle.Pointer, buffer.Length, IntPtr.Zero, nativeOverlapped) == 0) + { + // The operation failed, or it's pending. + int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); + switch (errorCode) + { + case Interop.Errors.ERROR_IO_PENDING: + // Common case: IO was initiated, completion will be handled by callback. + // Register for cancellation now that the operation has been initiated. + vts.RegisterForCancellation(cancellationToken); + break; + + case Interop.Errors.ERROR_BROKEN_PIPE: + // EOF on a pipe. Callback will not be called. + // We clear the overlapped status bit for this special case (failure + // to do so looks like we are freeing a pending overlapped later). + nativeOverlapped->InternalLow = IntPtr.Zero; + vts.Dispose(); + return ValueTask.FromResult(0); + + default: + // Error. Callback will not be called. + vts.Dispose(); + return ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + } + } + } + catch + { + vts.Dispose(); + throw; + } + // Completion handled by callback. + vts.FinishedScheduling(); + return new ValueTask(vts, vts.Version); } - private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) + private static unsafe ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) { + SafeFileHandle.ValueTaskSource vts = handle.GetValueTaskSource(); + try + { + NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(buffer, fileOffset); + Debug.Assert(vts._memoryHandle.Pointer != null); + + // Queue an async WriteFile operation. + if (Interop.Kernel32.WriteFile(handle, (byte*)vts._memoryHandle.Pointer, buffer.Length, IntPtr.Zero, nativeOverlapped) == 0) + { + // The operation failed, or it's pending. + int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); + if (errorCode == Interop.Errors.ERROR_IO_PENDING) + { + // Common case: IO was initiated, completion will be handled by callback. + // Register for cancellation now that the operation has been initiated. + vts.RegisterForCancellation(cancellationToken); + } + else + { + // Error. Callback will not be invoked. + vts.Dispose(); + return errorCode == Interop.Errors.ERROR_NO_DATA // EOF on a pipe. IO callback will not be called. + ? ValueTask.FromResult(0) + : ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + } + } + } + catch + { + vts.Dispose(); + throw; + } + // Completion handled by callback. + vts.FinishedScheduling(); + return new ValueTask(vts, vts.Version); } private static NativeOverlapped GetNativeOverlapped(long fileOffset) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs index 47e4025c11b8e..96f078c414378 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs @@ -10,8 +10,6 @@ namespace System.IO.Strategies { internal sealed partial class AsyncWindowsFileStreamStrategy : WindowsFileStreamStrategy { - private ValueTaskSource? _reusableValueTaskSource; // reusable ValueTaskSource that is currently NOT being used - internal AsyncWindowsFileStreamStrategy(SafeFileHandle handle, FileAccess access, FileShare share) : base(handle, access, share) { @@ -24,37 +22,6 @@ internal AsyncWindowsFileStreamStrategy(string path, FileMode mode, FileAccess a internal override bool IsAsync => true; - public override ValueTask DisposeAsync() - { - // the base class must dispose ThreadPoolBinding and FileHandle - // before _preallocatedOverlapped is disposed - ValueTask result = base.DisposeAsync(); - Debug.Assert(result.IsCompleted, "the method must be sync, as it performs no flushing"); - - Interlocked.Exchange(ref _reusableValueTaskSource, null)?.Dispose(); - - return result; - } - - protected override void Dispose(bool disposing) - { - // the base class must dispose ThreadPoolBinding and FileHandle - // before _preallocatedOverlapped is disposed - base.Dispose(disposing); - - Interlocked.Exchange(ref _reusableValueTaskSource, null)?.Dispose(); - } - - private void TryToReuse(ValueTaskSource source) - { - source._source.Reset(); - - if (Interlocked.CompareExchange(ref _reusableValueTaskSource, source, null) is not null) - { - source._preallocatedOverlapped.Dispose(); - } - } - public override int Read(byte[] buffer, int offset, int count) { ValueTask vt = ReadAsyncInternal(new Memory(buffer, offset, count), CancellationToken.None); @@ -76,16 +43,14 @@ private unsafe ValueTask ReadAsyncInternal(Memory destination, Cancel ThrowHelper.ThrowNotSupportedException_UnreadableStream(); } - // Rent the reusable ValueTaskSource, or create a new one to use if we couldn't get one (which - // should only happen on first use or if the FileStream is being used concurrently). - ValueTaskSource vts = Interlocked.Exchange(ref _reusableValueTaskSource, null) ?? new ValueTaskSource(this); + SafeFileHandle.ValueTaskSource vts = _fileHandle.GetValueTaskSource(); try { - NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(destination); + long positionBefore = _filePosition; + NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(destination, positionBefore); Debug.Assert(vts._memoryHandle.Pointer != null); // Calculate position in the file we should be at after the read is done - long positionBefore = _filePosition; if (CanSeek) { long len = Length; @@ -97,11 +62,6 @@ private unsafe ValueTask ReadAsyncInternal(Memory destination, Cancel default; } - // Now set the position to read from in the NativeOverlapped struct - // For pipes, we should leave the offset fields set to 0. - nativeOverlapped->OffsetLow = unchecked((int)positionBefore); - nativeOverlapped->OffsetHigh = (int)(positionBefore >> 32); - // When using overlapped IO, the OS is not supposed to // touch the file pointer location at all. We will adjust it // ourselves, but only in memory. This isn't threadsafe. @@ -163,22 +123,15 @@ private unsafe ValueTask WriteAsyncInternal(ReadOnlyMemory source, Cancell ThrowHelper.ThrowNotSupportedException_UnwritableStream(); } - // Rent the reusable ValueTaskSource, or create a new one to use if we couldn't get one (which - // should only happen on first use or if the FileStream is being used concurrently). - ValueTaskSource vts = Interlocked.Exchange(ref _reusableValueTaskSource, null) ?? new ValueTaskSource(this); + SafeFileHandle.ValueTaskSource vts = _fileHandle.GetValueTaskSource(); try { - NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(source); + long positionBefore = _filePosition; + NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(source, positionBefore); Debug.Assert(vts._memoryHandle.Pointer != null); - long positionBefore = _filePosition; if (CanSeek) { - // Now set the position to read from in the NativeOverlapped struct - // For pipes, we should leave the offset fields set to 0. - nativeOverlapped->OffsetLow = (int)positionBefore; - nativeOverlapped->OffsetHigh = (int)(positionBefore >> 32); - // When using overlapped IO, the OS is not supposed to // touch the file pointer location at all. We will adjust it // ourselves, but only in memory. This isn't threadsafe. @@ -226,9 +179,7 @@ private Exception HandleIOError(long positionBefore, int errorCode) _filePosition = positionBefore; } - return errorCode == Interop.Errors.ERROR_HANDLE_EOF ? - ThrowHelper.CreateEndOfFileException() : - Win32Marshal.GetExceptionForWin32Error(errorCode, _path); + return SafeFileHandle.ValueTaskSource.GetIOError(errorCode, _path); } public override Task FlushAsync(CancellationToken cancellationToken) => Task.CompletedTask; // no buffering = nothing to flush From a946c5d754cdc21f3f486f0372d14a28c5b309cf Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 1 Jun 2021 14:39:01 +0200 Subject: [PATCH 29/71] ReadScatter and WriteGather --- .../tests/RandomAccess/Base.cs | 16 ++--- .../tests/RandomAccess/ReadScatter.cs | 64 +++++++++++++++++++ .../tests/RandomAccess/WriteGather.cs | 57 +++++++++++++++++ .../tests/System.IO.FileSystem.Tests.csproj | 2 + .../src/System/IO/RandomAccess.Windows.cs | 39 +++++++++++ .../src/System/IO/RandomAccess.cs | 55 ++++++++++++++++ .../src/System/ThrowHelper.cs | 3 + .../System.Runtime/ref/System.Runtime.cs | 2 + 8 files changed, 230 insertions(+), 8 deletions(-) create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index a4044b1442128..13088d326cf47 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -46,9 +46,9 @@ public void ThrowsObjectDisposedExceptionForDisposedHandle() public void ThrowsNotSupportedExceptionForUnseekableFile() { using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) - using (SafeFileHandle pipeHandle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) + using (SafeFileHandle handle = new SafeFileHandle(server.SafePipeHandle.DangerousGetHandle(), true)) { - Assert.Throws(() => MethodUnderTest(pipeHandle, Array.Empty(), 0)); + Assert.Throws(() => MethodUnderTest(handle, Array.Empty(), 0)); } } @@ -58,9 +58,9 @@ public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() if (UsesOffsets) { FileOptions options = ShouldThrowForAsyncHandle ? FileOptions.None : FileOptions.Asynchronous; - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: options)) + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: options)) { - ArgumentOutOfRangeException ex = Assert.Throws(() => MethodUnderTest(validHandle, Array.Empty(), -1)); + ArgumentOutOfRangeException ex = Assert.Throws(() => MethodUnderTest(handle, Array.Empty(), -1)); Assert.Equal("fileOffset", ex.ParamName); } } @@ -71,9 +71,9 @@ public void ThrowsArgumentExceptionForAsyncFileHandle() { if (ShouldThrowForAsyncHandle) { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) { - ArgumentException ex = Assert.Throws(() => MethodUnderTest(validHandle, new byte[100], 0)); + ArgumentException ex = Assert.Throws(() => MethodUnderTest(handle, new byte[100], 0)); Assert.Equal("handle", ex.ParamName); } } @@ -84,9 +84,9 @@ public void ThrowsArgumentExceptionForSyncFileHandle() { if (ShouldThrowForSyncHandle) { - using (SafeFileHandle validHandle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.None)) + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.None)) { - ArgumentException ex = Assert.Throws(() => MethodUnderTest(validHandle, new byte[100], 0)); + ArgumentException ex = Assert.Throws(() => MethodUnderTest(handle, new byte[100], 0)); Assert.Equal("handle", ex.ParamName); } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs new file mode 100644 index 0000000000000..6cfa74b306a94 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs @@ -0,0 +1,64 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_ReadScatter : RandomAccess_Base + { + protected override long MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.Read(handle, new Memory[] { bytes }, fileOffset); + + protected override bool ShouldThrowForAsyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform sync IO using async handle + + [Fact] + public void ThrowsArgumentNullExceptionForNullBuffers() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.Read(handle, buffers: null, 0)); + Assert.Equal("buffers", ex.ParamName); + } + } + + [Fact] + public void HappyPath() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] expected = new byte[fileSize]; + new Random().NextBytes(expected); + File.WriteAllBytes(filePath, expected); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) + { + byte[] actual = new byte[fileSize + 1]; + long current = 0; + long total = 0; + + do + { + int firstBufferLength = (int)Math.Min(actual.Length - total, fileSize / 4); + + current = RandomAccess.Read( + handle, + new Memory[] + { + actual.AsMemory((int)total, firstBufferLength), + actual.AsMemory((int)total + firstBufferLength) + }, + fileOffset: total); + + total += current; + } while (current != 0); + + Assert.Equal(fileSize, total); + Assert.Equal(expected, actual.Take((int)total).ToArray()); + } + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs new file mode 100644 index 0000000000000..9568213a16fbb --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_WriteGather : RandomAccess_Base + { + protected override long MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.Write(handle, new ReadOnlyMemory[] { bytes }, fileOffset); + + protected override bool ShouldThrowForAsyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform sync IO using async handle + + [Fact] + public void ThrowsArgumentNullExceptionForNullBuffers() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.Write(handle, buffers: null, 0)); + Assert.Equal("buffers", ex.ParamName); + } + } + + [Fact] + public void HappyPath() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] content = new byte[fileSize]; + new Random().NextBytes(content); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) + { + long total = 0; + + while (total != fileSize) + { + int firstBufferLength = (int)Math.Min(content.Length - total, fileSize / 4); + + total += RandomAccess.Write( + handle, + new ReadOnlyMemory[] + { + content.AsMemory((int)total, firstBufferLength), + content.AsMemory((int)total + firstBufferLength) + }, + fileOffset: total); + } + } + + Assert.Equal(content, File.ReadAllBytes(filePath)); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 4ca822791aee7..58e4ac954cce8 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -55,8 +55,10 @@ + + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 3e4cef9449bb1..26a224fdf3b74 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Diagnostics; using System.IO.Strategies; using System.Runtime.InteropServices; @@ -231,6 +232,44 @@ private static unsafe ValueTask WriteAtOffsetAsync(SafeFileHandle handle, R return new ValueTask(vts, vts.Version); } + private static long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) + { + long total = 0; + + for (int i = 0; i < buffers.Count; i++) + { + Span span = buffers[i].Span; + int read = ReadAtOffset(handle, span, fileOffset + total); + total += read; + + if (read != span.Length) + { + break; + } + } + + return total; + } + + private static long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) + { + long total = 0; + + for (int i = 0; i < buffers.Count; i++) + { + ReadOnlySpan span = buffers[i].Span; + int written = WriteAtOffset(handle, span, fileOffset + total); + total += written; + + if (written != span.Length) + { + break; + } + } + + return total; + } + private static NativeOverlapped GetNativeOverlapped(long fileOffset) { NativeOverlapped nativeOverlapped = default; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 8508a5d748293..0a9acb3e11871 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -47,6 +48,29 @@ public static int Read(SafeFileHandle handle, Span buffer, long fileOffset return ReadAtOffset(handle, buffer, fileOffset); } + /// + /// Reads a sequence of bytes from given file at given offset. + /// + /// The file handle. + /// A list of memory buffers. When this method returns, the contents of the buffers are replaced by the bytes read from the file. + /// The file position to read from. + /// The total number of bytes read into the buffers. This can be less than the number of bytes allocated in the buffers if that many bytes are not currently available, or zero (0) if the end of the file has been reached. + /// or is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static long Read(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) + { + ValidateInput(handle, fileOffset, mustBeSync: OperatingSystem.IsWindows()); + ValidateBuffers(buffers); + + return ReadScatterAtOffset(handle, buffers, fileOffset); + } + /// /// Writes a sequence of bytes from given buffer to given file at given offset. /// @@ -69,6 +93,29 @@ public static int Write(SafeFileHandle handle, ReadOnlySpan buffer, long f return WriteAtOffset(handle, buffer, fileOffset); } + /// + /// Writes a sequence of bytes from given buffers to given file at given offset. + /// + /// The file handle. + /// A list of memory buffers. This method copies the contents of these buffers to the file. + /// The file position to write to. + /// The total number of bytes written into the file. This can be less than the number of bytes provided in the buffers and it's not an error. + /// or is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static long Write(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) + { + ValidateInput(handle, fileOffset, mustBeSync: OperatingSystem.IsWindows()); + ValidateBuffers(buffers); + + return WriteGatherAtOffset(handle, buffers, fileOffset); + } + /// /// Reads a sequence of bytes from given file at given offset. /// @@ -148,5 +195,13 @@ private static void ValidateInput(SafeFileHandle handle, long fileOffset, bool m ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(fileOffset)); } } + + private static void ValidateBuffers(IReadOnlyList buffers) + { + if (buffers is null) + { + ThrowHelper.ThrowArgumentNullException(ExceptionArgument.buffers); + } + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs index 62716599d0460..ae242caef9cfe 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @@ -799,6 +799,8 @@ private static string GetArgumentName(ExceptionArgument argument) return "suffix"; case ExceptionArgument.buffer: return "buffer"; + case ExceptionArgument.buffers: + return "buffers"; case ExceptionArgument.offset: return "offset"; case ExceptionArgument.stream: @@ -1066,6 +1068,7 @@ internal enum ExceptionArgument prefix, suffix, buffer, + buffers, offset, stream } diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 2f6e0abd48721..1d1cfeeac44b7 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -8077,8 +8077,10 @@ public static partial class RandomAccess { public static long GetLength(Microsoft.Win32.SafeHandles.SafeFileHandle handle) { throw null; } public static int Read(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Span buffer, long fileOffset) { throw null; } + public static long Read(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Collections.Generic.IReadOnlyList> buffers, long fileOffset) { throw null; } public static System.Threading.Tasks.ValueTask ReadAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Memory buffer, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static int Write(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlySpan buffer, long fileOffset) { throw null; } + public static long Write(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Collections.Generic.IReadOnlyList> buffers, long fileOffset) { throw null; } public static System.Threading.Tasks.ValueTask WriteAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlyMemory buffer, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } } } From 4f807ef6c2fab1711c0528e8c7041b429185a609 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 1 Jun 2021 14:50:18 +0200 Subject: [PATCH 30/71] some polishing --- .../src/System/IO/RandomAccess.cs | 50 +++++++++---------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 0a9acb3e11871..030c72e1175f8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -11,7 +11,7 @@ namespace System.IO public static partial class RandomAccess { /// - /// Gets the length in bytes of the file. + /// Gets the length of the file in bytes. /// /// The file handle. /// A long value representing the length of the file in bytes. @@ -71,6 +71,29 @@ public static long Read(SafeFileHandle handle, IReadOnlyList> buffe return ReadScatterAtOffset(handle, buffers, fileOffset); } + /// + /// Reads a sequence of bytes from given file at given offset. + /// + /// The file handle. + /// A region of memory. When this method returns, the contents of this region are replaced by the bytes read from the file. + /// The file position to read from. + /// The token to monitor for cancellation requests. The default value is . + /// The total number of bytes read into the buffer. This can be less than the number of bytes allocated in the buffer if that many bytes are not currently available, or zero (0) if the end of the file has been reached. + /// is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was not opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken = default) + { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + + return ReadAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); + } + /// /// Writes a sequence of bytes from given buffer to given file at given offset. /// @@ -116,29 +139,6 @@ public static long Write(SafeFileHandle handle, IReadOnlyList - /// Reads a sequence of bytes from given file at given offset. - /// - /// The file handle. - /// A region of memory. When this method returns, the contents of this region are replaced by the bytes read from the file. - /// The file position to read from. - /// The token to monitor for cancellation requests. The default value is . - /// The total number of bytes read into the buffer. This can be less than the number of bytes allocated in the buffer if that many bytes are not currently available, or zero (0) if the end of the file has been reached. - /// is . - /// is invalid. - /// The file is closed. - /// The file does not support seeking (pipe or socket). - /// was not opened for async IO. - /// is negative. - /// An I/O error occurred. - /// Position of the file is not advanced. - public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken = default) - { - ValidateInput(handle, fileOffset, mustBeAsync: true); - - return ReadAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); - } - /// /// Writes a sequence of bytes from given buffer to given file at given offset. /// @@ -157,7 +157,7 @@ public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffe /// Position of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken = default) { - ValidateInput(handle, fileOffset, mustBeAsync: true); + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); return WriteAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); } From 638e67ad04f62b67de51e0305d58ce095baf4e3c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 1 Jun 2021 15:09:34 +0200 Subject: [PATCH 31/71] ReadScatterAsync and WriteGatherAsync --- .../tests/RandomAccess/ReadScatterAsync.cs | 65 +++++++++++++++++++ .../tests/RandomAccess/WriteGatherAsync.cs | 58 +++++++++++++++++ .../tests/System.IO.FileSystem.Tests.csproj | 2 + .../src/System/IO/RandomAccess.Windows.cs | 40 ++++++++++++ .../src/System/IO/RandomAccess.cs | 48 ++++++++++++++ .../System.Runtime/ref/System.Runtime.cs | 2 + 6 files changed, 215 insertions(+) create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs new file mode 100644 index 0000000000000..f417e30cf6927 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -0,0 +1,65 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_ReadScatterAsync : RandomAccess_Base> + { + protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.ReadAsync(handle, new Memory[] { bytes }, fileOffset); + + protected override bool ShouldThrowForSyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle + + [Fact] + public void ThrowsArgumentNullExceptionForNullBuffers() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.ReadAsync(handle, buffers: null, 0)); + Assert.Equal("buffers", ex.ParamName); + } + } + + [Fact] + public async Task HappyPath() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] expected = new byte[fileSize]; + new Random().NextBytes(expected); + File.WriteAllBytes(filePath, expected); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) + { + byte[] actual = new byte[fileSize + 1]; + long current = 0; + long total = 0; + + do + { + int firstBufferLength = (int)Math.Min(actual.Length - total, fileSize / 4); + + current = await RandomAccess.ReadAsync( + handle, + new Memory[] + { + actual.AsMemory((int)total, firstBufferLength), + actual.AsMemory((int)total + firstBufferLength) + }, + fileOffset: total); + + total += current; + } while (current != 0); + + Assert.Equal(fileSize, total); + Assert.Equal(expected, actual.Take((int)total).ToArray()); + } + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs new file mode 100644 index 0000000000000..ccede935a2e95 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -0,0 +1,58 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Threading.Tasks; +using Microsoft.Win32.SafeHandles; +using Xunit; + +namespace System.IO.Tests +{ + public class RandomAccess_WriteGatherAsync : RandomAccess_Base> + { + protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) + => RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { bytes }, fileOffset); + + protected override bool ShouldThrowForSyncHandle + => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle + + [Fact] + public void ThrowsArgumentNullExceptionForNullBuffers() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous)) + { + ArgumentNullException ex = Assert.Throws(() => RandomAccess.WriteAsync(handle, buffers: null, 0)); + Assert.Equal("buffers", ex.ParamName); + } + } + + [Fact] + public async Task HappyPath() + { + const int fileSize = 4_001; + string filePath = GetTestFilePath(); + byte[] content = new byte[fileSize]; + new Random().NextBytes(content); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous)) + { + long total = 0; + + while (total != fileSize) + { + int firstBufferLength = (int)Math.Min(content.Length - total, fileSize / 4); + + total += await RandomAccess.WriteAsync( + handle, + new ReadOnlyMemory[] + { + content.AsMemory((int)total, firstBufferLength), + content.AsMemory((int)total + firstBufferLength) + }, + fileOffset: total); + } + } + + Assert.Equal(content, File.ReadAllBytes(filePath)); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 58e4ac954cce8..8ae316cb512b5 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -56,9 +56,11 @@ + + diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 26a224fdf3b74..351cad7071679 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -270,6 +270,46 @@ private static long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyList ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + long total = 0; + + for (int i = 0; i < buffers.Count; i++) + { + Memory buffer = buffers[i]; + int read = await ReadAtOffsetAsync(handle, buffer, fileOffset + total, cancellationToken).ConfigureAwait(false); + total += read; + + if (read != buffer.Length) + { + break; + } + } + + return total; + } + + private static async ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + long total = 0; + + for (int i = 0; i < buffers.Count; i++) + { + ReadOnlyMemory buffer = buffers[i]; + int written = await WriteAtOffsetAsync(handle, buffer, fileOffset + total, cancellationToken).ConfigureAwait(false); + total += written; + + if (written != buffer.Length) + { + break; + } + } + + return total; + } + private static NativeOverlapped GetNativeOverlapped(long fileOffset) { NativeOverlapped nativeOverlapped = default; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 030c72e1175f8..524d61d14b7ab 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -94,6 +94,30 @@ public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffe return ReadAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); } + /// + /// Reads a sequence of bytes from given file at given offset. + /// + /// The file handle. + /// A list of memory buffers. When this method returns, the contents of these buffers are replaced by the bytes read from the file. + /// The file position to read from. + /// The token to monitor for cancellation requests. The default value is . + /// The total number of bytes read into the buffers. This can be less than the number of bytes allocated in the buffers if that many bytes are not currently available, or zero (0) if the end of the file has been reached. + /// or is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was not opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static ValueTask ReadAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) + { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + ValidateBuffers(buffers); + + return ReadScatterAtOffsetAsync(handle, buffers, fileOffset, cancellationToken); + } + /// /// Writes a sequence of bytes from given buffer to given file at given offset. /// @@ -162,6 +186,30 @@ public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemory + /// Writes a sequence of bytes from given buffers to given file at given offset. + /// + /// The file handle. + /// A list of memory buffers. This method copies the contents of these buffers to the file. + /// The file position to write to. + /// The token to monitor for cancellation requests. The default value is . + /// The total number of bytes written into the file. This can be less than the number of bytes provided in the buffers and it's not an error. + /// or is . + /// is invalid. + /// The file is closed. + /// The file does not support seeking (pipe or socket). + /// was not opened for async IO. + /// is negative. + /// An I/O error occurred. + /// Position of the file is not advanced. + public static ValueTask WriteAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) + { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + ValidateBuffers(buffers); + + return WriteGatherAtOffsetAsync(handle, buffers, fileOffset, cancellationToken); + } + private static void ValidateInput(SafeFileHandle handle, long fileOffset, bool mustBeSync = false, bool mustBeAsync = false) { if (handle is null) diff --git a/src/libraries/System.Runtime/ref/System.Runtime.cs b/src/libraries/System.Runtime/ref/System.Runtime.cs index 1d1cfeeac44b7..612fdbc6252c7 100644 --- a/src/libraries/System.Runtime/ref/System.Runtime.cs +++ b/src/libraries/System.Runtime/ref/System.Runtime.cs @@ -8079,9 +8079,11 @@ public static partial class RandomAccess public static int Read(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Span buffer, long fileOffset) { throw null; } public static long Read(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Collections.Generic.IReadOnlyList> buffers, long fileOffset) { throw null; } public static System.Threading.Tasks.ValueTask ReadAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Memory buffer, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Threading.Tasks.ValueTask ReadAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Collections.Generic.IReadOnlyList> buffers, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } public static int Write(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlySpan buffer, long fileOffset) { throw null; } public static long Write(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Collections.Generic.IReadOnlyList> buffers, long fileOffset) { throw null; } public static System.Threading.Tasks.ValueTask WriteAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.ReadOnlyMemory buffer, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } + public static System.Threading.Tasks.ValueTask WriteAsync(Microsoft.Win32.SafeHandles.SafeFileHandle handle, System.Collections.Generic.IReadOnlyList> buffers, long fileOffset, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; } } } namespace System.IO.Enumeration From fe8fbd153f60ffe77ef2fd44f81d9d67f725d135 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 1 Jun 2021 15:36:51 +0200 Subject: [PATCH 32/71] use ReadFileScatter and WriteFileGather when possible --- .../Kernel32/Interop.FileScatterGather.cs | 37 +++ ...stem.IO.FileSystem.Net5Compat.Tests.csproj | 12 +- .../tests/RandomAccess/NoBuffering.Windows.cs | 264 ++++++++++++++++++ .../SectorAlignedMemory.Windows.cs | 109 ++++++++ .../tests/System.IO.FileSystem.Tests.csproj | 14 +- .../SafeHandles/SafeFileHandle.Windows.cs | 4 +- .../System.Private.CoreLib.Shared.projitems | 3 + .../src/System/IO/RandomAccess.Windows.cs | 245 +++++++++++++++- 8 files changed, 676 insertions(+), 12 deletions(-) create mode 100644 src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs create mode 100644 src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs new file mode 100644 index 0000000000000..8cdb8877187f6 --- /dev/null +++ b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Runtime.InteropServices; +using System.Threading; + +internal static partial class Interop +{ + internal static partial class Kernel32 + { + [StructLayout(LayoutKind.Explicit, Size = 8)] + internal unsafe struct FILE_SEGMENT_ELEMENT + { + [FieldOffset(0)] + public IntPtr Buffer; + [FieldOffset(0)] + public ulong Alignment; + } + + [DllImport(Libraries.Kernel32, SetLastError = true)] + internal static extern unsafe int ReadFileScatter( + SafeHandle handle, + FILE_SEGMENT_ELEMENT* segments, + int numBytesToRead, + IntPtr reserved_mustBeZero, + NativeOverlapped* overlapped); + + [DllImport(Libraries.Kernel32, SetLastError = true)] + internal static extern unsafe int WriteFileGather( + SafeHandle handle, + FILE_SEGMENT_ELEMENT* segments, + int numBytesToWrite, + IntPtr reserved_mustBeZero, + NativeOverlapped* overlapped); + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj index dfabeb02694b1..50ff724192a09 100644 --- a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj @@ -19,10 +19,14 @@ - - - - + + + + + + + + diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs new file mode 100644 index 0000000000000..0b8bddfce0153 --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs @@ -0,0 +1,264 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; +using System.Threading.Tasks; +using Microsoft.Win32.SafeHandles; +using Xunit; +using static Interop; +using static Interop.Kernel32; + +namespace System.IO.Tests +{ + public class RandomAccess_NoBuffering : FileSystemTest + { + private const FileOptions NoBuffering = (FileOptions)0x20000000; + + // https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfile2 + // we need it to open a device handle (File.OpenHandle does not allow for opening directories or devices) + [DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, SetLastError = true, ExactSpelling = true)] + private static extern SafeFileHandle CreateFile2( + ref char lpFileName, + int dwDesiredAccess, + int dwShareMode, + int dwCreationDisposition, + ref CREATEFILE2_EXTENDED_PARAMETERS pCreateExParams); + + // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363216.aspx + [DllImport(Libraries.Kernel32, SetLastError = true, ExactSpelling = true)] + private static unsafe extern bool DeviceIoControl( + SafeHandle hDevice, + uint dwIoControlCode, + void* lpInBuffer, + uint nInBufferSize, + void* lpOutBuffer, + uint nOutBufferSize, + out uint lpBytesReturned, + void* lpOverlapped); + + [Fact] + public async Task ReadAsyncUsingSingleBuffer() + { + const int fileSize = 1_000_000; // 1 MB + string filePath = GetTestFilePath(); + byte[] expected = new byte[fileSize]; + new Random().NextBytes(expected); + File.WriteAllBytes(filePath, expected); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous | NoBuffering)) + using (SectorAlignedMemory buffer = SectorAlignedMemory.Allocate(GetBufferSize(filePath))) + { + int current = 0; + int total = 0; + + do + { + current = await RandomAccess.ReadAsync(handle, buffer.Memory, fileOffset: total); + + Assert.True(expected.AsSpan(total, current).SequenceEqual(buffer.GetSpan().Slice(0, current))); + + total += current; + } + // From https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering: + // "File access sizes, including the optional file offset in the OVERLAPPED structure, + // if specified, must be for a number of bytes that is an integer multiple of the volume sector size." + // So if buffer and physical sector size is 4096 and the file size is 4097: + // the read from offset=0 reads 4096 bytes + // the read from offset=4096 reads 1 byte + // the read from offset=4097 THROWS (Invalid argument, offset is not a multiple of sector size!) + // That is why we stop at the first incomplete read (the next one would throw). + // It's possible to get 0 if we are lucky and file size is a multiple of physical sector size. + while (current == buffer.Memory.Length); + + Assert.Equal(fileSize, total); + } + } + + private int GetBufferSize(string filePath) + { + // From https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfilescatter: + // "Each buffer must be at least the size of a system memory page and must be aligned on a system memory page size boundary" + // "Because the file must be opened with FILE_FLAG_NO_BUFFERING, the number of bytes must be a multiple of the sector size + // of the file system where the file is located." + // Sector size is typically 512 to 4,096 bytes for direct-access storage devices (hard drives) and 2,048 bytes for CD-ROMs. + int physicalSectorSize = GetPhysicalSectorSize(filePath); + + // From https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering: + // "VirtualAlloc allocates memory that is aligned on addresses that are integer multiples of the system's page size. + // Page size is 4,096 bytes on x64 and x86 or 8,192 bytes for Itanium-based systems. For additional information, see the GetSystemInfo function." + int systemPageSize = Environment.SystemPageSize; + + // the following assumption is crucial for all NoBuffering tests and should always be true + Assert.True(systemPageSize % physicalSectorSize == 0); + + return systemPageSize; + } + + [Fact] + public async Task ReadAsyncUsingMultipleBuffers() + { + const int fileSize = 1_000_000; // 1 MB + string filePath = GetTestFilePath(); + byte[] expected = new byte[fileSize]; + new Random().NextBytes(expected); + File.WriteAllBytes(filePath, expected); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous | NoBuffering)) + using (SectorAlignedMemory buffer_1 = SectorAlignedMemory.Allocate(GetBufferSize(filePath))) + using (SectorAlignedMemory buffer_2 = SectorAlignedMemory.Allocate(GetBufferSize(filePath))) + { + long current = 0; + long total = 0; + + do + { + current = await RandomAccess.ReadAsync( + handle, + new Memory[] + { + buffer_1.Memory, + buffer_2.Memory, + }, + fileOffset: total); + + int takeFromFirst = Math.Min(buffer_1.Memory.Length, (int)current); + Assert.True(expected.AsSpan((int)total, takeFromFirst).SequenceEqual(buffer_1.GetSpan().Slice(0, takeFromFirst))); + int takeFromSecond = (int)current - takeFromFirst; + Assert.True(expected.AsSpan((int)total + takeFromFirst, takeFromSecond).SequenceEqual(buffer_2.GetSpan().Slice(0, takeFromSecond))); + + total += current; + } while (current == buffer_1.Memory.Length + buffer_2.Memory.Length); + + Assert.Equal(fileSize, total); + } + } + + [Fact] + public async Task WriteAsyncUsingSingleBuffer() + { + string filePath = GetTestFilePath(); + int bufferSize = GetBufferSize(filePath); + int fileSize = bufferSize * 10; + byte[] content = new byte[fileSize]; + new Random().NextBytes(content); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous | NoBuffering)) + using (SectorAlignedMemory buffer = SectorAlignedMemory.Allocate(bufferSize)) + { + int total = 0; + + while (total != fileSize) + { + int take = Math.Min(content.Length - total, bufferSize); + content.AsSpan(total, take).CopyTo(buffer.GetSpan()); + + total += await RandomAccess.WriteAsync( + handle, + buffer.Memory, + fileOffset: total); + } + } + + Assert.Equal(content, File.ReadAllBytes(filePath)); + } + + [Fact] + public async Task WriteAsyncUsingMultipleBuffers() + { + string filePath = GetTestFilePath(); + int bufferSize = GetBufferSize(filePath); + int fileSize = bufferSize * 10; + byte[] content = new byte[fileSize]; + new Random().NextBytes(content); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous | NoBuffering)) + using (SectorAlignedMemory buffer_1 = SectorAlignedMemory.Allocate(bufferSize)) + using (SectorAlignedMemory buffer_2 = SectorAlignedMemory.Allocate(bufferSize)) + { + long total = 0; + + while (total != fileSize) + { + content.AsSpan((int)total, bufferSize).CopyTo(buffer_1.GetSpan()); + content.AsSpan((int)total + bufferSize, bufferSize).CopyTo(buffer_2.GetSpan()); + + total += await RandomAccess.WriteAsync( + handle, + new ReadOnlyMemory[] + { + buffer_1.Memory, + buffer_2.Memory, + }, + fileOffset: total); + } + } + + Assert.Equal(content, File.ReadAllBytes(filePath)); + } + + private static unsafe int GetPhysicalSectorSize(string fullPath) + { + string devicePath = @"\\.\" + Path.GetPathRoot(Path.GetFullPath(fullPath)); + CREATEFILE2_EXTENDED_PARAMETERS extended = new CREATEFILE2_EXTENDED_PARAMETERS() + { + dwSize = (uint)sizeof(CREATEFILE2_EXTENDED_PARAMETERS), + dwFileAttributes = 0, + dwFileFlags = 0, + dwSecurityQosFlags = 0 + }; + + ReadOnlySpan span = Path.EndsInDirectorySeparator(devicePath) + ? devicePath.Remove(devicePath.Length - 1) // CreateFile2 does not like a `\` at the end of device path.. + : devicePath; + + using (SafeFileHandle deviceHandle = CreateFile2( + lpFileName: ref MemoryMarshal.GetReference(span), + dwDesiredAccess: 0, + dwShareMode: (int)FileShare.ReadWrite, + dwCreationDisposition: (int)FileMode.Open, + pCreateExParams: ref extended)) + { + Assert.False(deviceHandle.IsInvalid); + + STORAGE_PROPERTY_QUERY input = new STORAGE_PROPERTY_QUERY + { + PropertyId = 6, // StorageAccessAlignmentProperty + QueryType = 0 // PropertyStandardQuery + }; + STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR output = default; + + Assert.True(DeviceIoControl( + hDevice: deviceHandle, + dwIoControlCode: 0x2D1400, // IOCTL_STORAGE_QUERY_PROPERTY + lpInBuffer: &input, + nInBufferSize: (uint)Marshal.SizeOf(input), + lpOutBuffer: &output, + nOutBufferSize: (uint)Marshal.SizeOf(), + lpBytesReturned: out _, + lpOverlapped: null)); + + return (int)output.BytesPerPhysicalSector; + } + } + + [StructLayout(LayoutKind.Sequential)] + private unsafe struct STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR + { + internal uint Version; + internal uint Size; + internal uint BytesPerCacheLine; + internal uint BytesOffsetForCacheAlignment; + internal uint BytesPerLogicalSector; + internal uint BytesPerPhysicalSector; + internal uint BytesOffsetForSectorAlignment; + } + + [StructLayout(LayoutKind.Sequential)] + private unsafe struct STORAGE_PROPERTY_QUERY + { + internal int PropertyId; + internal int QueryType; + internal byte AdditionalParameters; + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs new file mode 100644 index 0000000000000..cede38d4b18ca --- /dev/null +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs @@ -0,0 +1,109 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Buffers; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Threading; +using static Interop.Kernel32; + +namespace System.IO.Tests +{ + internal class SectorAlignedMemory : MemoryManager + { + private bool disposed = false; + private int refCount = 0; + private IntPtr memory; + private int length; + + private unsafe SectorAlignedMemory(void* memory, int length) + { + this.memory = (IntPtr)memory; + this.length = length; + } + + ~SectorAlignedMemory() + { + Dispose(false); + } + + public static unsafe SectorAlignedMemory Allocate(int length) + { + void* memory = VirtualAlloc( + IntPtr.Zero.ToPointer(), + new UIntPtr((uint)(Marshal.SizeOf() * length)), + MemOptions.MEM_COMMIT | MemOptions.MEM_RESERVE, + PageOptions.PAGE_READWRITE); + + return new SectorAlignedMemory(memory, length); + } + + public bool IsDisposed => disposed; + + public unsafe override Span GetSpan() => new Span((void*)memory, length); + + protected bool IsRetained => refCount > 0; + + public override MemoryHandle Pin(int elementIndex = 0) + { + unsafe + { + Retain(); + if ((uint)elementIndex > length) throw new ArgumentOutOfRangeException(nameof(elementIndex)); + void* pointer = Unsafe.Add((void*)memory, elementIndex); + return new MemoryHandle(pointer, default, this); + } + } + + public bool Release() + { + int newRefCount = Interlocked.Decrement(ref refCount); + + if (newRefCount < 0) + { + throw new InvalidOperationException("Unmatched Release/Retain"); + } + + return newRefCount != 0; + } + + public void Retain() + { + if (disposed) + { + throw new ObjectDisposedException(nameof(SectorAlignedMemory)); + } + + Interlocked.Increment(ref refCount); + } + + protected override unsafe void Dispose(bool disposing) + { + if (disposed) + { + return; + } + + VirtualAlloc( + memory.ToPointer(), + new UIntPtr((uint)(Marshal.SizeOf() * length)), + MemOptions.MEM_FREE, + PageOptions.PAGE_READWRITE); + memory = IntPtr.Zero; + + disposed = true; + } + + protected override bool TryGetArray(out ArraySegment arraySegment) + { + // cannot expose managed array + arraySegment = default; + return false; + } + + public override void Unpin() + { + Release(); + } + } +} diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 8ae316cb512b5..d4cb4a418ab95 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -71,10 +71,16 @@ - - - - + + + + + + + + + + diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 1b81e456994f4..be9d1e228e3f4 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -11,7 +11,7 @@ namespace Microsoft.Win32.SafeHandles { public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private const FileOptions NoBuffering = (FileOptions)(0x20000000); + internal const FileOptions NoBuffering = (FileOptions)(0x20000000); private FileOptions _fileOptions = (FileOptions)(-1); private int _fileType = -1; @@ -132,7 +132,7 @@ internal void InitThreadPoolBindingIfNeeded() } } - private unsafe FileOptions GetFileOptions() + internal unsafe FileOptions GetFileOptions() { if (_fileOptions != (FileOptions)(-1)) { diff --git a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems index 6a90fd69c190d..438b1c3ff169e 100644 --- a/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems +++ b/src/libraries/System.Private.CoreLib/src/System.Private.CoreLib.Shared.projitems @@ -1594,6 +1594,9 @@ Common\Interop\Windows\Kernel32\Interop.ReadFile_SafeHandle_NativeOverlapped.cs + + Common\Interop\Windows\Kernel32\Interop.FileScatterGather.cs + Common\Interop\Windows\Kernel32\Interop.RemoveDirectory.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 351cad7071679..80f1e01a7469e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Collections.Generic; using System.Diagnostics; using System.IO.Strategies; @@ -236,12 +237,15 @@ private static long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList span = buffers[i].Span; int read = ReadAtOffset(handle, span, fileOffset + total); total += read; + // We stop on the first incomplete read. + // Most probably there is no more data available and the next read is going to return 0 (EOF). if (read != span.Length) { break; @@ -255,12 +259,15 @@ private static long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyList span = buffers[i].Span; int written = WriteAtOffset(handle, span, fileOffset + total); total += written; + // We stop on the first incomplete write. + // Most probably the disk became full and the next write is going to throw. if (written != span.Length) { break; @@ -270,7 +277,107 @@ private static long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyList ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + if (CanUseScatterGatherWindowsAPIs(handle)) + { + long totalBytes = 0; + for (int i = 0; i < buffers.Count; i++) + { + totalBytes += buffers[i].Length; + } + + if (totalBytes < int.MaxValue) // the ReadFileScatter API uses int, not long + { + return ReadScatterAtOffsetSingleSyscallAsync(handle, buffers, fileOffset, cancellationToken); + } + } + + return ReadScatterAtOffsetMultipleSyscallsAsync(handle, buffers, fileOffset, cancellationToken); + } + + // From https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfilescatter: + // "The file handle must be created with the GENERIC_READ right, and the FILE_FLAG_OVERLAPPED and FILE_FLAG_NO_BUFFERING flags." + // "This function is not supported for 32-bit applications by WOW64 on Itanium-based systems." + private static bool CanUseScatterGatherWindowsAPIs(SafeFileHandle handle) + => IntPtr.Size != sizeof(int) + && handle.IsAsync + && ((handle.GetFileOptions() & SafeFileHandle.NoBuffering) != 0); + + private static async ValueTask ReadScatterAtOffsetSingleSyscallAsync(SafeFileHandle handle, + IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) + { + if (buffers.Count == 1) + { + // we have to await it because we can't cast a VT to VT + return await ReadAtOffsetAsync(handle, buffers[0], fileOffset, cancellationToken).ConfigureAwait(false); + } + + (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) = PrepareMemorySegments(buffers); + + try + { + return await ReadFileScatterAsync(handle, pinnedSegments, totalBytes, fileOffset, cancellationToken).ConfigureAwait(false); + } + finally + { + foreach (MemoryHandle memoryHandle in memoryHandles) + { + memoryHandle.Dispose(); + } + pinnedSegments.Dispose(); + } + } + + private static unsafe ValueTask ReadFileScatterAsync(SafeFileHandle handle, MemoryHandle pinnedSegments, + int bytesToRead, long fileOffset, CancellationToken cancellationToken) + { + SafeFileHandle.ValueTaskSource vts = handle.GetValueTaskSource(); + try + { + NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(Memory.Empty, fileOffset); + Debug.Assert(pinnedSegments.Pointer != null); + + if (Interop.Kernel32.ReadFileScatter(handle, (Interop.Kernel32.FILE_SEGMENT_ELEMENT*)pinnedSegments.Pointer, bytesToRead, IntPtr.Zero, nativeOverlapped) == 0) + { + // The operation failed, or it's pending. + int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); + switch (errorCode) + { + case Interop.Errors.ERROR_IO_PENDING: + // Common case: IO was initiated, completion will be handled by callback. + // Register for cancellation now that the operation has been initiated. + vts.RegisterForCancellation(cancellationToken); + break; + + case Interop.Errors.ERROR_BROKEN_PIPE: + // EOF on a pipe. Callback will not be called. + // We clear the overlapped status bit for this special case (failure + // to do so looks like we are freeing a pending overlapped later). + nativeOverlapped->InternalLow = IntPtr.Zero; + vts.Dispose(); + return ValueTask.FromResult(0); + + default: + // Error. Callback will not be called. + vts.Dispose(); + return ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + } + } + } + catch + { + vts.Dispose(); + throw; + } + + // Completion handled by callback. + vts.FinishedScheduling(); + return new ValueTask(vts, vts.Version); + } + + private static async ValueTask ReadScatterAtOffsetMultipleSyscallsAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) { long total = 0; @@ -290,7 +397,27 @@ private static async ValueTask ReadScatterAtOffsetAsync(SafeFileHandle han return total; } - private static async ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + if (CanUseScatterGatherWindowsAPIs(handle)) + { + long totalBytes = 0; + for (int i = 0; i < buffers.Count; i++) + { + totalBytes += buffers[i].Length; + } + + if (totalBytes < int.MaxValue) // the ReadFileScatter API uses int, not long + { + return WriteGatherAtOffsetSingleSyscallAsync(handle, buffers, fileOffset, cancellationToken); + } + } + + return WriteGatherAtOffsetMultipleSyscallsAsync(handle, buffers, fileOffset, cancellationToken); + } + + private static async ValueTask WriteGatherAtOffsetMultipleSyscallsAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) { long total = 0; @@ -310,6 +437,120 @@ private static async ValueTask WriteGatherAtOffsetAsync(SafeFileHandle han return total; } + private static async ValueTask WriteGatherAtOffsetSingleSyscallAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + if (buffers.Count == 1) + { + return await WriteAtOffsetAsync(handle, buffers[0], fileOffset, cancellationToken).ConfigureAwait(false); + } + + (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) = PrepareMemorySegments(buffers); + + try + { + return await WriteFileGatherAsync(handle, pinnedSegments, totalBytes, fileOffset, cancellationToken).ConfigureAwait(false); + } + finally + { + foreach (MemoryHandle memoryHandle in memoryHandles) + { + memoryHandle.Dispose(); + } + pinnedSegments.Dispose(); + } + } + + private static unsafe ValueTask WriteFileGatherAsync(SafeFileHandle handle, MemoryHandle pinnedSegments, + int bytesToWrite, long fileOffset, CancellationToken cancellationToken) + { + SafeFileHandle.ValueTaskSource vts = handle.GetValueTaskSource(); + try + { + NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(ReadOnlyMemory.Empty, fileOffset); + Debug.Assert(vts._memoryHandle.Pointer != null); + + // Queue an async WriteFile operation. + if (Interop.Kernel32.WriteFileGather(handle, (Interop.Kernel32.FILE_SEGMENT_ELEMENT*)pinnedSegments.Pointer, bytesToWrite, IntPtr.Zero, nativeOverlapped) == 0) + { + // The operation failed, or it's pending. + int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); + if (errorCode == Interop.Errors.ERROR_IO_PENDING) + { + // Common case: IO was initiated, completion will be handled by callback. + // Register for cancellation now that the operation has been initiated. + vts.RegisterForCancellation(cancellationToken); + } + else + { + // Error. Callback will not be invoked. + vts.Dispose(); + return errorCode == Interop.Errors.ERROR_NO_DATA // EOF on a pipe. IO callback will not be called. + ? ValueTask.FromResult(0) + : ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + } + } + } + catch + { + vts.Dispose(); + throw; + } + + // Completion handled by callback. + vts.FinishedScheduling(); + return new ValueTask(vts, vts.Version); + } + + private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) PrepareMemorySegments(IReadOnlyList> buffers) + { + // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " + Interop.Kernel32.FILE_SEGMENT_ELEMENT[] fileSegments = new Interop.Kernel32.FILE_SEGMENT_ELEMENT[buffers.Count + 1]; + fileSegments[buffers.Count].Buffer = IntPtr.Zero; + + MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; + int totalBytes = 0; + for (int i = 0; i < buffers.Count; i++) + { + Memory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + memoryHandles[i] = memoryHandle; + fileSegments[i] = new Interop.Kernel32.FILE_SEGMENT_ELEMENT { Buffer = new IntPtr(memoryHandle.Pointer) }; + + checked + { + totalBytes += buffer.Length; + } + } + + return (fileSegments.AsMemory().Pin(), memoryHandles, totalBytes); + } + + // any ideas on geting rid of the code duplication are more than welcomed + private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) PrepareMemorySegments(IReadOnlyList> buffers) + { + // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " + Interop.Kernel32.FILE_SEGMENT_ELEMENT[] fileSegments = new Interop.Kernel32.FILE_SEGMENT_ELEMENT[buffers.Count + 1]; + fileSegments[buffers.Count].Buffer = IntPtr.Zero; + + MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; + int totalBytes = 0; + for (int i = 0; i < buffers.Count; i++) + { + ReadOnlyMemory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + memoryHandles[i] = memoryHandle; + fileSegments[i] = new Interop.Kernel32.FILE_SEGMENT_ELEMENT { Buffer = new IntPtr(memoryHandle.Pointer) }; + + checked + { + totalBytes += buffer.Length; + } + } + + return (fileSegments.AsMemory().Pin(), memoryHandles, totalBytes); + } + private static NativeOverlapped GetNativeOverlapped(long fileOffset) { NativeOverlapped nativeOverlapped = default; From 381776b8ef311c684d0810671697ae85cb738f41 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 10:38:26 +0200 Subject: [PATCH 33/71] complete Unix implementation --- .../Unix/System.Native/Interop.IOVector.cs | 16 +++ .../System.Native/Interop.MessageHeader.cs | 6 - .../Unix/System.Native/Interop.PReadV.cs | 13 ++ .../Unix/System.Native/Interop.PWriteV.cs | 13 ++ .../Native/Unix/System.Native/entrypoints.c | 2 + .../Native/Unix/System.Native/pal_io.c | 25 ++++ .../Native/Unix/System.Native/pal_io.h | 14 +++ .../Unix/System.Native/pal_networking.h | 2 +- .../src/System.Net.Sockets.csproj | 2 + .../System.Private.CoreLib.Shared.projitems | 9 ++ .../src/System/IO/RandomAccess.Unix.cs | 112 +++++++++++++++++- .../src/System/IO/RandomAccess.cs | 20 ++++ 12 files changed, 226 insertions(+), 8 deletions(-) create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.IOVector.cs create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs create mode 100644 src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.IOVector.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.IOVector.cs new file mode 100644 index 0000000000000..9cbf1ee2c3478 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.IOVector.cs @@ -0,0 +1,16 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; + +internal static partial class Interop +{ + internal static partial class Sys + { + internal unsafe struct IOVector + { + public byte* Base; + public UIntPtr Count; + } + } +} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MessageHeader.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MessageHeader.cs index 5e327a6479362..4b71e0e3d3724 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MessageHeader.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.MessageHeader.cs @@ -8,12 +8,6 @@ internal static partial class Interop { internal static partial class Sys { - internal unsafe struct IOVector - { - public byte* Base; - public UIntPtr Count; - } - internal unsafe struct MessageHeader { public byte* SocketAddress; diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs new file mode 100644 index 0000000000000..c0daf42b0fbe6 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PReadV", SetLastError = true)] + internal static extern unsafe long PReadV(SafeHandle fd, IOVector* buffer, int vectorCount, long fileOffset); + } +} diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs new file mode 100644 index 0000000000000..c710840ee93b4 --- /dev/null +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs @@ -0,0 +1,13 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Runtime.InteropServices; + +internal static partial class Interop +{ + internal static partial class Sys + { + [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PWriteV", SetLastError = true)] + internal static extern unsafe long PWriteV(SafeHandle fd, IOVector* buffer, int vectorCount, long fileOffset); + } +} diff --git a/src/libraries/Native/Unix/System.Native/entrypoints.c b/src/libraries/Native/Unix/System.Native/entrypoints.c index fbaff8cfae7e9..8a1438b6c3d18 100644 --- a/src/libraries/Native/Unix/System.Native/entrypoints.c +++ b/src/libraries/Native/Unix/System.Native/entrypoints.c @@ -244,6 +244,8 @@ static const Entry s_sysNative[] = DllImportEntry(SystemNative_SetErrNo) DllImportEntry(SystemNative_PRead) DllImportEntry(SystemNative_PWrite) + DllImportEntry(SystemNative_PReadV) + DllImportEntry(SystemNative_PWriteV) }; EXTERN_C const void* SystemResolveDllImport(const char* name); diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index ae8cc24c19df4..d18e8db7970f7 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -22,6 +22,7 @@ #include #include #include +#include #include #include #include @@ -1478,3 +1479,27 @@ int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64 assert(count >= -1 && count <= bufferSize); return (int32_t)count; } + +int64_t SystemNative_PReadV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset) +{ + assert(vectors != NULL); + assert(vectorCount >= 0); + + ssize_t count; + while ((count = preadv(ToFileDescriptor(fd), (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + + assert(count >= -1); + return (int64_t)count; +} + +int64_t SystemNative_PWriteV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset) +{ + assert(vectors != NULL); + assert(vectorCount >= 0); + + ssize_t count; + while ((count = pwritev(ToFileDescriptor(fd), (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + + assert(count >= -1); + return (int64_t)count; +} diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index 95a0c89bcbf9f..c5ceadaa60b3b 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -744,3 +744,17 @@ PALEXPORT int32_t SystemNative_PRead(intptr_t fd, void* buffer, int32_t bufferSi * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. */ PALEXPORT int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64_t fileOffset); + +/** + * Reads the number of bytes specified into the provided buffers from the specified, opened file descriptor at specified offset. + * + * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. + */ +PALEXPORT int64_t SystemNative_PReadV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset); + +/** + * Writes the number of bytes specified in the buffers into the specified, opened file descriptor at specified offset. + * + * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. + */ +PALEXPORT int64_t SystemNative_PWriteV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset); diff --git a/src/libraries/Native/Unix/System.Native/pal_networking.h b/src/libraries/Native/Unix/System.Native/pal_networking.h index bbb0bc0785cce..cd2cf91e45eec 100644 --- a/src/libraries/Native/Unix/System.Native/pal_networking.h +++ b/src/libraries/Native/Unix/System.Native/pal_networking.h @@ -276,7 +276,7 @@ typedef struct } LingerOption; // NOTE: the layout of this type is intended to exactly match the layout of a `struct iovec`. There are -// assertions in pal_networking.cpp that validate this. +// assertions in pal_networking.c that validate this. typedef struct { uint8_t* Base; diff --git a/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj b/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj index 9f56ca5250a52..35d10bb679f16 100644 --- a/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj +++ b/src/libraries/System.Net.Sockets/src/System.Net.Sockets.csproj @@ -236,6 +236,8 @@ Link="Common\Interop\Unix\Interop.Stat.cs" /> + Common\Interop\Unix\System.Native\Interop.GetUnixRelease.cs + + Common\Interop\Unix\System.Native\Interop.IOVector.cs + Common\Interop\Unix\System.Native\Interop.LChflags.cs @@ -1975,9 +1978,15 @@ Common\Interop\Unix\System.Native\Interop.PRead.cs + + Common\Interop\Unix\System.Native\Interop.PReadV.cs + Common\Interop\Unix\System.Native\Interop.PWrite.cs + + Common\Interop\Unix\System.Native\Interop.PWriteV.cs + Common\Interop\Unix\System.Native\Interop.Read.cs diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 0ccc823926453..d79444e20f633 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -1,15 +1,23 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Collections.Generic; using System.IO.Strategies; using System.Runtime.InteropServices; +using System.Threading; +using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; namespace System.IO { public static partial class RandomAccess { - internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) + // IovStackThreshold matches Linux's UIO_FASTIOV, which is the number of 'struct iovec' + // that get stackalloced in the Linux kernel. + private const int IovStackThreshold = 8; + + internal static long GetFileLength(SafeFileHandle handle, string? path) { int result = Interop.Sys.FStat(handle, out Interop.Sys.FileStatus status); FileStreamHelpers.CheckFileCall(result, path); @@ -26,6 +34,57 @@ private static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, } } + private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) + { + MemoryHandle[] handles = new MemoryHandle[buffers.Count]; + Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count ]; + + for (int i = 0; i < buffers.Count; i++) + { + Memory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; + handles[i] = memoryHandle; + } + + long result; + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + { + result = Interop.Sys.PReadV(handle, pinnedVectors, buffers.Count, fileOffset); + } + + foreach (MemoryHandle memoryHandle in handles) + { + memoryHandle.Dispose(); + } + + return FileStreamHelpers.CheckFileCall(result, path: null); + } + + private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, + CancellationToken cancellationToken) + { + return new ValueTask( + new TaskFactory( + cancellationToken, + TaskCreationOptions.None, + TaskContinuationOptions.DenyChildAttach, + TaskScheduler.Default) + .StartNew(() => ReadAtOffset(handle, buffer.Span, fileOffset), cancellationToken)); + } + + private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + return new ValueTask( + new TaskFactory( + cancellationToken, + TaskCreationOptions.None, + TaskContinuationOptions.DenyChildAttach, + TaskScheduler.Default) + .StartNew(() => ReadScatterAtOffset(handle, buffers, fileOffset), cancellationToken)); + } + private static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) { fixed (byte* bufPtr = &MemoryMarshal.GetReference(buffer)) @@ -35,5 +94,56 @@ private static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan> buffers, long fileOffset) + { + MemoryHandle[] handles = new MemoryHandle[buffers.Count ]; + Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count ]; + + for (int i = 0; i < buffers.Count; i++) + { + ReadOnlyMemory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector {Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length}; + handles[i] = memoryHandle; + } + + long result; + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + { + result = Interop.Sys.PWriteV(handle, pinnedVectors, buffers.Count, fileOffset); + } + + foreach (MemoryHandle memoryHandle in handles) + { + memoryHandle.Dispose(); + } + + return FileStreamHelpers.CheckFileCall(result, path: null); + } + + private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, + CancellationToken cancellationToken) + { + return new ValueTask( + new TaskFactory( + cancellationToken, + TaskCreationOptions.None, + TaskContinuationOptions.DenyChildAttach, + TaskScheduler.Default) + .StartNew(() => WriteAtOffset(handle, buffer.Span, fileOffset), cancellationToken)); + } + + private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, + long fileOffset, CancellationToken cancellationToken) + { + return new ValueTask( + new TaskFactory( + cancellationToken, + TaskCreationOptions.None, + TaskContinuationOptions.DenyChildAttach, + TaskScheduler.Default) + .StartNew(() => WriteGatherAtOffset(handle, buffers, fileOffset), cancellationToken)); + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 524d61d14b7ab..20034d3dc0ce3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -89,6 +89,11 @@ public static long Read(SafeFileHandle handle, IReadOnlyList> buffe /// Position of the file is not advanced. public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken = default) { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); return ReadAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); @@ -112,6 +117,11 @@ public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffe /// Position of the file is not advanced. public static ValueTask ReadAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); ValidateBuffers(buffers); @@ -181,6 +191,11 @@ public static long Write(SafeFileHandle handle, IReadOnlyListPosition of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken = default) { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); return WriteAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); @@ -204,6 +219,11 @@ public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemoryPosition of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) { + if (cancellationToken.IsCancellationRequested) + { + return ValueTask.FromCanceled(cancellationToken); + } + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); ValidateBuffers(buffers); From 1febba1e6ec6a04fbe283923f8d52eb476ed975f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 11:02:34 +0200 Subject: [PATCH 34/71] add tests for handling already cancelled tokens --- .../System.IO.FileSystem/tests/RandomAccess/Base.cs | 10 ++++++++++ .../tests/RandomAccess/ReadAsync.cs | 12 ++++++++++++ .../tests/RandomAccess/ReadScatterAsync.cs | 12 ++++++++++++ .../tests/RandomAccess/WriteAsync.cs | 12 ++++++++++++ .../tests/RandomAccess/WriteGatherAsync.cs | 12 ++++++++++++ 5 files changed, 58 insertions(+) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index 13088d326cf47..7a1e591d9e9a8 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.IO.Pipes; +using System.Threading; using Microsoft.Win32.SafeHandles; using Xunit; @@ -91,5 +92,14 @@ public void ThrowsArgumentExceptionForSyncFileHandle() } } } + + protected static CancellationToken GetCancelledToken() + { + CancellationTokenSource source = new CancellationTokenSource(); + CancellationToken token = source.Token; + source.Cancel(); + + return token; + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs index 7b8a1011c5968..7b4525b4d4424 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; using Xunit; @@ -16,6 +17,17 @@ protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] protected override bool ShouldThrowForSyncHandle => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle + [Fact] + public Task TaskAlreadyCanceledAsync() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) + { + CancellationToken token = GetCancelledToken(); + Assert.True(RandomAccess.ReadAsync(handle, new byte[1], 0, token).IsCanceled); + return Assert.ThrowsAsync(async () => await RandomAccess.ReadAsync(handle, new byte[1], 0, token)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs index f417e30cf6927..c7f4b8f6ffe4e 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Linq; +using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; using Xunit; @@ -26,6 +27,17 @@ public void ThrowsArgumentNullExceptionForNullBuffers() } } + [Fact] + public Task TaskAlreadyCanceledAsync() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) + { + CancellationToken token = GetCancelledToken(); + Assert.True(RandomAccess.ReadAsync(handle, new Memory[] { new byte[1] }, 0, token).IsCanceled); + return Assert.ThrowsAsync(async () => await RandomAccess.ReadAsync(handle, new Memory[] { new byte[1] }, 0, token)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs index a0f96cf31d108..c27f553aaf626 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; using Xunit; @@ -15,6 +16,17 @@ protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] protected override bool ShouldThrowForSyncHandle => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle + [Fact] + public Task TaskAlreadyCanceledAsync() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) + { + CancellationToken token = GetCancelledToken(); + Assert.True(RandomAccess.WriteAsync(handle, new byte[1], 0, token).IsCanceled); + return Assert.ThrowsAsync(async () => await RandomAccess.WriteAsync(handle, new byte[1], 0, token)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index ccede935a2e95..8548ba7fc72ca 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; using Xunit; @@ -25,6 +26,17 @@ public void ThrowsArgumentNullExceptionForNullBuffers() } } + [Fact] + public Task TaskAlreadyCanceledAsync() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) + { + CancellationToken token = GetCancelledToken(); + Assert.True(RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { new byte[1] }, 0, token).IsCanceled); + return Assert.ThrowsAsync(async () => await RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { new byte[1] }, 0, token)); + } + } + [Fact] public async Task HappyPath() { From 8db3f7afefaeed33e64616c723b4a594c87eaf92 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 11:41:41 +0200 Subject: [PATCH 35/71] add tests for unauthorized access --- .../System.IO.FileSystem/tests/RandomAccess/Base.cs | 9 +++++++++ .../System.IO.FileSystem/tests/RandomAccess/Read.cs | 9 +++++++++ .../System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs | 9 +++++++++ .../tests/RandomAccess/ReadScatter.cs | 9 +++++++++ .../tests/RandomAccess/ReadScatterAsync.cs | 9 +++++++++ .../System.IO.FileSystem/tests/RandomAccess/Write.cs | 9 +++++++++ .../tests/RandomAccess/WriteAsync.cs | 9 +++++++++ .../tests/RandomAccess/WriteGather.cs | 9 +++++++++ .../tests/RandomAccess/WriteGatherAsync.cs | 9 +++++++++ .../System.Private.CoreLib/src/System/IO/RandomAccess.cs | 8 ++++++++ 10 files changed, 89 insertions(+) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index 7a1e591d9e9a8..e337b10719160 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -101,5 +101,14 @@ protected static CancellationToken GetCancelledToken() return token; } + + protected SafeFileHandle GetHandleToExistingFile(FileAccess access) + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[1]); + + FileOptions options = ShouldThrowForAsyncHandle ? FileOptions.None : FileOptions.Asynchronous; + return File.OpenHandle(filePath, FileMode.Open, access, FileShare.None, options); + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs index 659e3df42d345..00e8953645038 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs @@ -15,6 +15,15 @@ protected override int MethodUnderTest(SafeFileHandle handle, byte[] bytes, long protected override bool ShouldThrowForAsyncHandle => OperatingSystem.IsWindows(); // on Windows we can NOT perform sync IO using async handle + [Fact] + public void ThrowsOnWriteAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Write)) + { + Assert.Throws(() => RandomAccess.Read(handle, new byte[1], 0)); + } + } + [Fact] public void HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs index 7b4525b4d4424..18c0d0e840dbc 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs @@ -28,6 +28,15 @@ public Task TaskAlreadyCanceledAsync() } } + [Fact] + public async Task ThrowsOnWriteAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Write)) + { + await Assert.ThrowsAsync(async () => await RandomAccess.ReadAsync(handle, new byte[1], 0)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs index 6cfa74b306a94..a87af17357ee3 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs @@ -25,6 +25,15 @@ public void ThrowsArgumentNullExceptionForNullBuffers() } } + [Fact] + public void ThrowsOnWriteAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Write)) + { + Assert.Throws(() => RandomAccess.Read(handle, new Memory[] { new byte[1] }, 0)); + } + } + [Fact] public void HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs index c7f4b8f6ffe4e..fe5388090d960 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -38,6 +38,15 @@ public Task TaskAlreadyCanceledAsync() } } + [Fact] + public async Task ThrowsOnWriteAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Write)) + { + await Assert.ThrowsAsync(async () => await RandomAccess.ReadAsync(handle, new Memory[] { new byte[1] }, 0)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs index ee03631d8de0b..c959973aafd8d 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs @@ -14,6 +14,15 @@ protected override int MethodUnderTest(SafeFileHandle handle, byte[] bytes, long protected override bool ShouldThrowForAsyncHandle => OperatingSystem.IsWindows(); // on Windows we can NOT perform sync IO using async handle + [Fact] + public void ThrowsOnReadAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Read)) + { + Assert.Throws(() => RandomAccess.Write(handle, new byte[1], 0)); + } + } + [Fact] public void HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs index c27f553aaf626..61983a54651ba 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs @@ -27,6 +27,15 @@ public Task TaskAlreadyCanceledAsync() } } + [Fact] + public async Task ThrowsOnReadAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Read)) + { + await Assert.ThrowsAsync(async () => await RandomAccess.WriteAsync(handle, new byte[1], 0)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs index 9568213a16fbb..911432f685571 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs @@ -24,6 +24,15 @@ public void ThrowsArgumentNullExceptionForNullBuffers() } } + [Fact] + public void ThrowsOnReadAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Read)) + { + Assert.Throws(() => RandomAccess.Write(handle, new ReadOnlyMemory[] { new byte[1] }, 0)); + } + } + [Fact] public void HappyPath() { diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index 8548ba7fc72ca..926b53fe24fdc 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -37,6 +37,15 @@ public Task TaskAlreadyCanceledAsync() } } + [Fact] + public async Task ThrowsOnReadAccess() + { + using (SafeFileHandle handle = GetHandleToExistingFile(FileAccess.Read)) + { + await Assert.ThrowsAsync(async () => await RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { new byte[1] }, 0)); + } + } + [Fact] public async Task HappyPath() { diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 20034d3dc0ce3..943c17e8fb549 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -39,6 +39,7 @@ public static long GetLength(SafeFileHandle handle) /// The file does not support seeking (pipe or socket). /// was opened for async IO. /// is negative. + /// was not opened for reading. /// An I/O error occurred. /// Position of the file is not advanced. public static int Read(SafeFileHandle handle, Span buffer, long fileOffset) @@ -61,6 +62,7 @@ public static int Read(SafeFileHandle handle, Span buffer, long fileOffset /// The file does not support seeking (pipe or socket). /// was opened for async IO. /// is negative. + /// was not opened for reading. /// An I/O error occurred. /// Position of the file is not advanced. public static long Read(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) @@ -85,6 +87,7 @@ public static long Read(SafeFileHandle handle, IReadOnlyList> buffe /// The file does not support seeking (pipe or socket). /// was not opened for async IO. /// is negative. + /// was not opened for reading. /// An I/O error occurred. /// Position of the file is not advanced. public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken = default) @@ -113,6 +116,7 @@ public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffe /// The file does not support seeking (pipe or socket). /// was not opened for async IO. /// is negative. + /// was not opened for reading. /// An I/O error occurred. /// Position of the file is not advanced. public static ValueTask ReadAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) @@ -141,6 +145,7 @@ public static ValueTask ReadAsync(SafeFileHandle handle, IReadOnlyListThe file does not support seeking (pipe or socket). /// was opened for async IO. /// is negative. + /// was not opened for writing. /// An I/O error occurred. /// Position of the file is not advanced. public static int Write(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) @@ -163,6 +168,7 @@ public static int Write(SafeFileHandle handle, ReadOnlySpan buffer, long f /// The file does not support seeking (pipe or socket). /// was opened for async IO. /// is negative. + /// was not opened for writing. /// An I/O error occurred. /// Position of the file is not advanced. public static long Write(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) @@ -187,6 +193,7 @@ public static long Write(SafeFileHandle handle, IReadOnlyListThe file does not support seeking (pipe or socket). /// was not opened for async IO. /// is negative. + /// was not opened for writing. /// An I/O error occurred. /// Position of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken = default) @@ -215,6 +222,7 @@ public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemoryThe file does not support seeking (pipe or socket). /// was not opened for async IO. /// is negative. + /// was not opened for writing. /// An I/O error occurred. /// Position of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) From f33d4ca84094be7b1b09054588e5594a0a7d2343 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 12:33:43 +0200 Subject: [PATCH 36/71] some polishing before sending PR --- .../Common/src/Interop/Unix/System.Native/Interop.PReadV.cs | 2 +- .../Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs | 2 +- src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs | 2 +- .../System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs index c0daf42b0fbe6..5d93078161f05 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PReadV.cs @@ -8,6 +8,6 @@ internal static partial class Interop internal static partial class Sys { [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PReadV", SetLastError = true)] - internal static extern unsafe long PReadV(SafeHandle fd, IOVector* buffer, int vectorCount, long fileOffset); + internal static extern unsafe long PReadV(SafeHandle fd, IOVector* vectors, int vectorCount, long fileOffset); } } diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs index c710840ee93b4..c17e9964d8fe9 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PWriteV.cs @@ -8,6 +8,6 @@ internal static partial class Interop internal static partial class Sys { [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PWriteV", SetLastError = true)] - internal static extern unsafe long PWriteV(SafeHandle fd, IOVector* buffer, int vectorCount, long fileOffset); + internal static extern unsafe long PWriteV(SafeHandle fd, IOVector* vectors, int vectorCount, long fileOffset); } } diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index b3a394549536e..1ebea9ec1042c 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -32,7 +32,7 @@ public override void NegativePreallocationSizeThrows() public override void FileModeAppendExisting(string streamSpecifier) { // currently not enabled due to https://github.com/dotnet/runtime/issues/53432 - GC.KeepAlive(streamSpecifier); // to keep the xUnit analyser happy + _ = streamSpecifier; // to keep the xUnit analyser happy } [Theory] diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index d79444e20f633..2b90b80bd9b66 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -30,7 +30,7 @@ private static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, { int result = Interop.Sys.PRead(handle, bufPtr, buffer.Length, fileOffset); FileStreamHelpers.CheckFileCall(result, path: null); - return result; + return result; } } From f8c8423d11e5a108cf4ba6a83e0c38899f81aea2 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 15:56:44 +0200 Subject: [PATCH 37/71] always treat EOF as success with 0 bytes read (read at end of file) hopefully fix an edge case bug hit in the CI but not reproducible locally --- .../SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs | 5 +---- .../src/System/IO/RandomAccess.Windows.cs | 6 ++++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs index 6d20f76b6b175..69371a45aff9b 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.ValueTaskSource.Windows.cs @@ -195,10 +195,7 @@ internal void Complete(uint errorCode, uint numBytes) case 0: case Interop.Errors.ERROR_BROKEN_PIPE: case Interop.Errors.ERROR_NO_DATA: - // AsyncWindowsFileStreamStrategy checks the Length before calling ReadFile - // and tries to perform a 0-byte read at EOF. - // RandomAccess does not and we need to treat EOF as a successfull operation. - case Interop.Errors.ERROR_HANDLE_EOF: + case Interop.Errors.ERROR_HANDLE_EOF: // logically success with 0 bytes read (read at end of file) // Success _source.SetResult((int)numBytes); break; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 80f1e01a7469e..cefb134d33461 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -167,6 +167,7 @@ private static unsafe ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Me vts.RegisterForCancellation(cancellationToken); break; + case Interop.Errors.ERROR_HANDLE_EOF: // logically success with 0 bytes read (read at end of file) case Interop.Errors.ERROR_BROKEN_PIPE: // EOF on a pipe. Callback will not be called. // We clear the overlapped status bit for this special case (failure @@ -178,7 +179,7 @@ private static unsafe ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Me default: // Error. Callback will not be called. vts.Dispose(); - return ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + return ValueTask.FromException(Win32Marshal.GetExceptionForWin32Error(errorCode)); } } } @@ -351,6 +352,7 @@ private static unsafe ValueTask ReadFileScatterAsync(SafeFileHandle handle, vts.RegisterForCancellation(cancellationToken); break; + case Interop.Errors.ERROR_HANDLE_EOF: // logically success with 0 bytes read (read at end of file) case Interop.Errors.ERROR_BROKEN_PIPE: // EOF on a pipe. Callback will not be called. // We clear the overlapped status bit for this special case (failure @@ -362,7 +364,7 @@ private static unsafe ValueTask ReadFileScatterAsync(SafeFileHandle handle, default: // Error. Callback will not be called. vts.Dispose(); - return ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + return ValueTask.FromException(Win32Marshal.GetExceptionForWin32Error(errorCode)); } } } From 2126f15ebe4f6cc21fa908756072643c65ca658e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 16:09:55 +0200 Subject: [PATCH 38/71] simplify the NoBuffering tests logic --- ...stem.IO.FileSystem.Net5Compat.Tests.csproj | 2 - .../tests/RandomAccess/NoBuffering.Windows.cs | 120 +----------------- .../tests/System.IO.FileSystem.Tests.csproj | 2 - 3 files changed, 5 insertions(+), 119 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj index 50ff724192a09..628bbb4324aea 100644 --- a/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/Net5CompatTests/System.IO.FileSystem.Net5Compat.Tests.csproj @@ -21,10 +21,8 @@ - - diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs index 0b8bddfce0153..201fe85803de4 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs @@ -1,12 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Runtime.InteropServices; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; using Xunit; -using static Interop; -using static Interop.Kernel32; namespace System.IO.Tests { @@ -14,28 +11,6 @@ public class RandomAccess_NoBuffering : FileSystemTest { private const FileOptions NoBuffering = (FileOptions)0x20000000; - // https://docs.microsoft.com/en-us/windows/desktop/api/fileapi/nf-fileapi-createfile2 - // we need it to open a device handle (File.OpenHandle does not allow for opening directories or devices) - [DllImport(Libraries.Kernel32, CharSet = CharSet.Unicode, SetLastError = true, ExactSpelling = true)] - private static extern SafeFileHandle CreateFile2( - ref char lpFileName, - int dwDesiredAccess, - int dwShareMode, - int dwCreationDisposition, - ref CREATEFILE2_EXTENDED_PARAMETERS pCreateExParams); - - // https://msdn.microsoft.com/en-us/library/windows/desktop/aa363216.aspx - [DllImport(Libraries.Kernel32, SetLastError = true, ExactSpelling = true)] - private static unsafe extern bool DeviceIoControl( - SafeHandle hDevice, - uint dwIoControlCode, - void* lpInBuffer, - uint nInBufferSize, - void* lpOutBuffer, - uint nOutBufferSize, - out uint lpBytesReturned, - void* lpOverlapped); - [Fact] public async Task ReadAsyncUsingSingleBuffer() { @@ -46,7 +21,7 @@ public async Task ReadAsyncUsingSingleBuffer() File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous | NoBuffering)) - using (SectorAlignedMemory buffer = SectorAlignedMemory.Allocate(GetBufferSize(filePath))) + using (SectorAlignedMemory buffer = SectorAlignedMemory.Allocate(Environment.SystemPageSize)) { int current = 0; int total = 0; @@ -74,26 +49,6 @@ public async Task ReadAsyncUsingSingleBuffer() } } - private int GetBufferSize(string filePath) - { - // From https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfilescatter: - // "Each buffer must be at least the size of a system memory page and must be aligned on a system memory page size boundary" - // "Because the file must be opened with FILE_FLAG_NO_BUFFERING, the number of bytes must be a multiple of the sector size - // of the file system where the file is located." - // Sector size is typically 512 to 4,096 bytes for direct-access storage devices (hard drives) and 2,048 bytes for CD-ROMs. - int physicalSectorSize = GetPhysicalSectorSize(filePath); - - // From https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering: - // "VirtualAlloc allocates memory that is aligned on addresses that are integer multiples of the system's page size. - // Page size is 4,096 bytes on x64 and x86 or 8,192 bytes for Itanium-based systems. For additional information, see the GetSystemInfo function." - int systemPageSize = Environment.SystemPageSize; - - // the following assumption is crucial for all NoBuffering tests and should always be true - Assert.True(systemPageSize % physicalSectorSize == 0); - - return systemPageSize; - } - [Fact] public async Task ReadAsyncUsingMultipleBuffers() { @@ -104,8 +59,8 @@ public async Task ReadAsyncUsingMultipleBuffers() File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous | NoBuffering)) - using (SectorAlignedMemory buffer_1 = SectorAlignedMemory.Allocate(GetBufferSize(filePath))) - using (SectorAlignedMemory buffer_2 = SectorAlignedMemory.Allocate(GetBufferSize(filePath))) + using (SectorAlignedMemory buffer_1 = SectorAlignedMemory.Allocate(Environment.SystemPageSize)) + using (SectorAlignedMemory buffer_2 = SectorAlignedMemory.Allocate(Environment.SystemPageSize)) { long current = 0; long total = 0; @@ -137,7 +92,7 @@ public async Task ReadAsyncUsingMultipleBuffers() public async Task WriteAsyncUsingSingleBuffer() { string filePath = GetTestFilePath(); - int bufferSize = GetBufferSize(filePath); + int bufferSize = Environment.SystemPageSize; int fileSize = bufferSize * 10; byte[] content = new byte[fileSize]; new Random().NextBytes(content); @@ -166,7 +121,7 @@ public async Task WriteAsyncUsingSingleBuffer() public async Task WriteAsyncUsingMultipleBuffers() { string filePath = GetTestFilePath(); - int bufferSize = GetBufferSize(filePath); + int bufferSize = Environment.SystemPageSize; int fileSize = bufferSize * 10; byte[] content = new byte[fileSize]; new Random().NextBytes(content); @@ -195,70 +150,5 @@ public async Task WriteAsyncUsingMultipleBuffers() Assert.Equal(content, File.ReadAllBytes(filePath)); } - - private static unsafe int GetPhysicalSectorSize(string fullPath) - { - string devicePath = @"\\.\" + Path.GetPathRoot(Path.GetFullPath(fullPath)); - CREATEFILE2_EXTENDED_PARAMETERS extended = new CREATEFILE2_EXTENDED_PARAMETERS() - { - dwSize = (uint)sizeof(CREATEFILE2_EXTENDED_PARAMETERS), - dwFileAttributes = 0, - dwFileFlags = 0, - dwSecurityQosFlags = 0 - }; - - ReadOnlySpan span = Path.EndsInDirectorySeparator(devicePath) - ? devicePath.Remove(devicePath.Length - 1) // CreateFile2 does not like a `\` at the end of device path.. - : devicePath; - - using (SafeFileHandle deviceHandle = CreateFile2( - lpFileName: ref MemoryMarshal.GetReference(span), - dwDesiredAccess: 0, - dwShareMode: (int)FileShare.ReadWrite, - dwCreationDisposition: (int)FileMode.Open, - pCreateExParams: ref extended)) - { - Assert.False(deviceHandle.IsInvalid); - - STORAGE_PROPERTY_QUERY input = new STORAGE_PROPERTY_QUERY - { - PropertyId = 6, // StorageAccessAlignmentProperty - QueryType = 0 // PropertyStandardQuery - }; - STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR output = default; - - Assert.True(DeviceIoControl( - hDevice: deviceHandle, - dwIoControlCode: 0x2D1400, // IOCTL_STORAGE_QUERY_PROPERTY - lpInBuffer: &input, - nInBufferSize: (uint)Marshal.SizeOf(input), - lpOutBuffer: &output, - nOutBufferSize: (uint)Marshal.SizeOf(), - lpBytesReturned: out _, - lpOverlapped: null)); - - return (int)output.BytesPerPhysicalSector; - } - } - - [StructLayout(LayoutKind.Sequential)] - private unsafe struct STORAGE_ACCESS_ALIGNMENT_DESCRIPTOR - { - internal uint Version; - internal uint Size; - internal uint BytesPerCacheLine; - internal uint BytesOffsetForCacheAlignment; - internal uint BytesPerLogicalSector; - internal uint BytesPerPhysicalSector; - internal uint BytesOffsetForSectorAlignment; - } - - [StructLayout(LayoutKind.Sequential)] - private unsafe struct STORAGE_PROPERTY_QUERY - { - internal int PropertyId; - internal int QueryType; - internal byte AdditionalParameters; - } } } diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index d4cb4a418ab95..0961f29469716 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -75,10 +75,8 @@ - - From e613e72603e1acc7a1b25748b549d2c562441b28 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 16:06:34 +0200 Subject: [PATCH 39/71] handle lack of preadv pwritev on OSX --- .../Native/Unix/Common/pal_config.h.in | 2 + .../Native/Unix/System.Native/pal_io.c | 62 +++++++++++++++++-- .../Native/Unix/System.Native/pal_io.h | 4 +- src/libraries/Native/Unix/configure.cmake | 10 +++ 4 files changed, 70 insertions(+), 8 deletions(-) diff --git a/src/libraries/Native/Unix/Common/pal_config.h.in b/src/libraries/Native/Unix/Common/pal_config.h.in index b7bd484e629ff..034d57b1e1c98 100644 --- a/src/libraries/Native/Unix/Common/pal_config.h.in +++ b/src/libraries/Native/Unix/Common/pal_config.h.in @@ -36,6 +36,8 @@ #cmakedefine01 HAVE_POSIX_ADVISE #cmakedefine01 HAVE_POSIX_FALLOCATE #cmakedefine01 HAVE_POSIX_FALLOCATE64 +#cmakedefine01 HAVE_PREADV +#cmakedefine01 HAVE_PWRITEV #cmakedefine01 PRIORITY_REQUIRES_INT_WHO #cmakedefine01 KEVENT_REQUIRES_INT_PARAMS #cmakedefine01 HAVE_IOCTL diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index d18e8db7970f7..ad5e405b38e5d 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1480,25 +1480,75 @@ int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64 return (int32_t)count; } -int64_t SystemNative_PReadV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset) +int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset) { assert(vectors != NULL); assert(vectorCount >= 0); - ssize_t count; - while ((count = preadv(ToFileDescriptor(fd), (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + ssize_t count = 0; + int fileDescriptor = ToFileDescriptor(fd); +#if HAVE_PREADV + while ((count = preadv(fileDescriptor, vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); +#else + ssize_t current; + for (int i = 0; i < vectorCount; i++) + { + struct iovec vector = vectors[i]; + while ((current = pread(fileDescriptor, vector.iov_base, vector.iov_len, (off_t)(fileOffset + count))) < 0 && errno == EINTR); + + if (current < 0) + { + // if previous calls were succesfull, we return what we got so far + // otherwise, we return the error code + return count > 0 ? count : current; + } + + count += current; + + // we stop on the first incomplete operation (the next call would fail) + if (current != vector.iov_len) + { + return (int64_t)count; + } + } +#endif assert(count >= -1); return (int64_t)count; } -int64_t SystemNative_PWriteV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset) +int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset) { assert(vectors != NULL); assert(vectorCount >= 0); - ssize_t count; - while ((count = pwritev(ToFileDescriptor(fd), (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + ssize_t count = 0; + int fileDescriptor = ToFileDescriptor(fd); +#if HAVE_PWRITEV + while ((count = pwritev(fileDescriptor, vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); +#else + ssize_t current; + for (int i = 0; i < vectorCount; i++) + { + struct iovec vector = vectors[i]; + while ((current = pwrite(fileDescriptor, vector.iov_base, vector.iov_len, (off_t)(fileOffset + count))) < 0 && errno == EINTR); + + if (current < 0) + { + // if previous calls were succesfull, we return what we got so far + // otherwise, we return the error code + return count > 0 ? count : current; + } + + count += current; + + // we stop on the first incomplete operation (the next call would fail) + if (current != vector.iov_len) + { + return (int64_t)count; + } + } +#endif assert(count >= -1); return (int64_t)count; diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index c5ceadaa60b3b..52fb8522e2de5 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -750,11 +750,11 @@ PALEXPORT int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferS * * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. */ -PALEXPORT int64_t SystemNative_PReadV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset); +PALEXPORT int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset); /** * Writes the number of bytes specified in the buffers into the specified, opened file descriptor at specified offset. * * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. */ -PALEXPORT int64_t SystemNative_PWriteV(intptr_t fd, void* vectors, int32_t vectorCount, int64_t fileOffset); +PALEXPORT int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset); diff --git a/src/libraries/Native/Unix/configure.cmake b/src/libraries/Native/Unix/configure.cmake index 1b85454db4af2..a63f83dbdcccf 100644 --- a/src/libraries/Native/Unix/configure.cmake +++ b/src/libraries/Native/Unix/configure.cmake @@ -219,6 +219,16 @@ check_symbol_exists( fcntl.h HAVE_POSIX_FALLOCATE64) +check_symbol_exists( + preadv + uio.h + HAVE_PREADV) + +check_symbol_exists( + pwritev + uio.h + HAVE_PWRITEV) + check_symbol_exists( ioctl sys/ioctl.h From 73f07511127e4a47d5f57e8964519787ee735398 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 3 Jun 2021 16:52:33 +0200 Subject: [PATCH 40/71] fix the Linux build --- src/libraries/Native/Unix/configure.cmake | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Native/Unix/configure.cmake b/src/libraries/Native/Unix/configure.cmake index a63f83dbdcccf..b5d3b9aecd188 100644 --- a/src/libraries/Native/Unix/configure.cmake +++ b/src/libraries/Native/Unix/configure.cmake @@ -221,12 +221,12 @@ check_symbol_exists( check_symbol_exists( preadv - uio.h + sys/uio.h HAVE_PREADV) check_symbol_exists( pwritev - uio.h + sys/uio.h HAVE_PWRITEV) check_symbol_exists( From 940609c8e098d4ed2143c9f9a811f7e341ff551f Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 10:43:16 +0200 Subject: [PATCH 41/71] Apply suggestions from code review Co-authored-by: Stephen Toub --- .../Win32/SafeHandles/SafeFileHandle.Windows.cs | 2 +- .../src/System/IO/RandomAccess.Unix.cs | 17 ++++++++--------- .../src/System/IO/RandomAccess.Windows.cs | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index be9d1e228e3f4..97e532a448bf2 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -48,7 +48,7 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA NtCreateFile(fullPath, mode, access, share, options, preallocationSize), ownsHandle: true, options, - Interop.Kernel32.FileTypes.FILE_TYPE_DISK); // similarly to FileStream, we assume that only disk files can be referenced by path on Windows + Interop.Kernel32.FileTypes.FILE_TYPE_DISK); // similar to FileStream, we assume that only disk files can be referenced by path on Windows fileHandle.InitThreadPoolBindingIfNeeded(); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 2b90b80bd9b66..447391b32d624 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -37,7 +37,7 @@ private static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) { MemoryHandle[] handles = new MemoryHandle[buffers.Count]; - Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count ]; + Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count]; for (int i = 0; i < buffers.Count; i++) { @@ -64,12 +64,11 @@ private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyL private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask( - new TaskFactory( - cancellationToken, - TaskCreationOptions.None, - TaskContinuationOptions.DenyChildAttach, - TaskScheduler.Default) + return new ValueTask(Task.Factory.StartNew(state => + { + var args = ((SafeFileHandle handle, Memory buffer, long fileOffset))state; + return ReadAtOffset(args.handle, args.buffer.Span, args.fileOffset); + }, (handle, buffer, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); .StartNew(() => ReadAtOffset(handle, buffer.Span, fileOffset), cancellationToken)); } @@ -97,14 +96,14 @@ private static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan> buffers, long fileOffset) { - MemoryHandle[] handles = new MemoryHandle[buffers.Count ]; + MemoryHandle[] handles = new MemoryHandle[buffers.Count]; Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count ]; for (int i = 0; i < buffers.Count; i++) { ReadOnlyMemory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector {Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length}; + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; handles[i] = memoryHandle; } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index cefb134d33461..f5b39e84ddde0 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -29,7 +29,7 @@ internal static unsafe long GetFileLength(SafeFileHandle handle, string? path) internal static unsafe int ReadAtOffset(SafeFileHandle handle, Span buffer, long fileOffset, string? path = null) { NativeOverlapped nativeOverlapped = GetNativeOverlapped(fileOffset); - int r = ReadFileNative(handle, buffer, true, &nativeOverlapped, out int errorCode); + int r = ReadFileNative(handle, buffer, syncUsingOverlapped: true, &nativeOverlapped, out int errorCode); if (r == -1) { From 891c2b3b108ca68455ea5406df61c3e0b6e168f1 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 11:13:25 +0200 Subject: [PATCH 42/71] code review fixes and suggestions --- .../Windows/NtDll/Interop.NtCreateFile.cs | 2 +- .../SafeHandles/SafeFileHandle.Windows.cs | 10 ++--- .../src/System/IO/File.netcoreapp.cs | 4 +- .../src/System/IO/RandomAccess.Unix.cs | 37 ++++++++----------- .../src/System/IO/RandomAccess.Windows.cs | 37 ++++++------------- 5 files changed, 35 insertions(+), 55 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs index 84263c71e4016..1841ba558eb65 100644 --- a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs +++ b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs @@ -72,7 +72,7 @@ internal static unsafe (uint status, IntPtr handle) CreateFile( } } - internal static unsafe (uint status, IntPtr handle) CreateFile(ReadOnlySpan path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) + internal static unsafe (uint status, IntPtr handle) NtCreateFile(ReadOnlySpan path, FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize) { // For mitigating local elevation of privilege attack through named pipes // make sure we always call NtCreateFile with SECURITY_ANONYMOUS so that the diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 97e532a448bf2..ac582f0a8e025 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -61,15 +61,15 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac uint ntStatus; IntPtr fileHandle; - const string mandatoryNtPrefix = @"\??\"; - if (fullPath.StartsWith(mandatoryNtPrefix, StringComparison.Ordinal)) + const string MandatoryNtPrefix = @"\??\"; + if (fullPath.StartsWith(MandatoryNtPrefix, StringComparison.Ordinal)) { - (ntStatus, fileHandle) = Interop.NtDll.CreateFile(fullPath, mode, access, share, options, preallocationSize); + (ntStatus, fileHandle) = Interop.NtDll.NtCreateFile(fullPath, mode, access, share, options, preallocationSize); } else { var vsb = new ValueStringBuilder(stackalloc char[1024]); - vsb.Append(mandatoryNtPrefix); + vsb.Append(MandatoryNtPrefix); if (fullPath.StartsWith(@"\\?\", StringComparison.Ordinal)) // NtCreateFile does not support "\\?\" prefix, only "\??\" { @@ -80,7 +80,7 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac vsb.Append(fullPath); } - (ntStatus, fileHandle) = Interop.NtDll.CreateFile(vsb.AsSpan(), mode, access, share, options, preallocationSize); + (ntStatus, fileHandle) = Interop.NtDll.NtCreateFile(vsb.AsSpan(), mode, access, share, options, preallocationSize); vsb.Dispose(); } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs index bff701b1b477a..ee9d78f61fbfa 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs @@ -59,9 +59,9 @@ private static byte[] ReadAllBytesUnknownLength(FileStream fs) if (bytesRead == buffer.Length) { uint newLength = (uint)buffer.Length * 2; - if (newLength > MaxByteArrayLength) + if (newLength > Array.MaxLength) { - newLength = (uint)Math.Max(MaxByteArrayLength, buffer.Length + 1); + newLength = (uint)Math.Max(Array.MaxLength, buffer.Length + 1); } byte[] tmp = ArrayPool.Shared.Rent((int)newLength); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 447391b32d624..548e30711a3be 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -69,19 +69,16 @@ private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset))state; return ReadAtOffset(args.handle, args.buffer.Span, args.fileOffset); }, (handle, buffer, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); - .StartNew(() => ReadAtOffset(handle, buffer.Span, fileOffset), cancellationToken)); } private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask( - new TaskFactory( - cancellationToken, - TaskCreationOptions.None, - TaskContinuationOptions.DenyChildAttach, - TaskScheduler.Default) - .StartNew(() => ReadScatterAtOffset(handle, buffers, fileOffset), cancellationToken)); + return new ValueTask(Task.Factory.StartNew(state => + { + var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state; + return ReadScatterAtOffset(args.handle, args.buffers, args.fileOffset); + }, (handle, buffers, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } private static unsafe int WriteAtOffset(SafeFileHandle handle, ReadOnlySpan buffer, long fileOffset) @@ -124,25 +121,21 @@ private static unsafe long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyL private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask( - new TaskFactory( - cancellationToken, - TaskCreationOptions.None, - TaskContinuationOptions.DenyChildAttach, - TaskScheduler.Default) - .StartNew(() => WriteAtOffset(handle, buffer.Span, fileOffset), cancellationToken)); + return new ValueTask(Task.Factory.StartNew(state => + { + var args = ((SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset))state; + return WriteAtOffset(args.handle, args.buffer.Span, args.fileOffset); + }, (handle, buffer, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask( - new TaskFactory( - cancellationToken, - TaskCreationOptions.None, - TaskContinuationOptions.DenyChildAttach, - TaskScheduler.Default) - .StartNew(() => WriteGatherAtOffset(handle, buffers, fileOffset), cancellationToken)); + return new ValueTask(Task.Factory.StartNew(state => + { + var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state; + return WriteGatherAtOffset(args.handle, args.buffers, args.fileOffset); + }, (handle, buffers, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index f5b39e84ddde0..813f6f0ec302a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -289,9 +289,9 @@ private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, I totalBytes += buffers[i].Length; } - if (totalBytes < int.MaxValue) // the ReadFileScatter API uses int, not long + if (totalBytes <= int.MaxValue) // the ReadFileScatter API uses int, not long { - return ReadScatterAtOffsetSingleSyscallAsync(handle, buffers, fileOffset, cancellationToken); + return ReadScatterAtOffsetSingleSyscallAsync(handle, buffers, fileOffset, (int)totalBytes, cancellationToken); } } @@ -307,7 +307,7 @@ private static bool CanUseScatterGatherWindowsAPIs(SafeFileHandle handle) && ((handle.GetFileOptions() & SafeFileHandle.NoBuffering) != 0); private static async ValueTask ReadScatterAtOffsetSingleSyscallAsync(SafeFileHandle handle, - IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) + IReadOnlyList> buffers, long fileOffset, int totalBytes, CancellationToken cancellationToken) { if (buffers.Count == 1) { @@ -315,7 +315,7 @@ private static async ValueTask ReadScatterAtOffsetSingleSyscallAsync(SafeF return await ReadAtOffsetAsync(handle, buffers[0], fileOffset, cancellationToken).ConfigureAwait(false); } - (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) = PrepareMemorySegments(buffers); + (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) = PrepareMemorySegments(buffers); try { @@ -410,9 +410,9 @@ private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, I totalBytes += buffers[i].Length; } - if (totalBytes < int.MaxValue) // the ReadFileScatter API uses int, not long + if (totalBytes <= int.MaxValue) // the ReadFileScatter API uses int, not long { - return WriteGatherAtOffsetSingleSyscallAsync(handle, buffers, fileOffset, cancellationToken); + return WriteGatherAtOffsetSingleSyscallAsync(handle, buffers, fileOffset, (int)totalBytes, cancellationToken); } } @@ -440,14 +440,14 @@ private static async ValueTask WriteGatherAtOffsetMultipleSyscallsAsync(Sa } private static async ValueTask WriteGatherAtOffsetSingleSyscallAsync(SafeFileHandle handle, IReadOnlyList> buffers, - long fileOffset, CancellationToken cancellationToken) + long fileOffset, int totalBytes, CancellationToken cancellationToken) { if (buffers.Count == 1) { return await WriteAtOffsetAsync(handle, buffers[0], fileOffset, cancellationToken).ConfigureAwait(false); } - (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) = PrepareMemorySegments(buffers); + (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) = PrepareMemorySegments(buffers); try { @@ -504,53 +504,40 @@ private static unsafe ValueTask WriteFileGatherAsync(SafeFileHandle handle, return new ValueTask(vts, vts.Version); } - private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) PrepareMemorySegments(IReadOnlyList> buffers) + private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) PrepareMemorySegments(IReadOnlyList> buffers) { // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " Interop.Kernel32.FILE_SEGMENT_ELEMENT[] fileSegments = new Interop.Kernel32.FILE_SEGMENT_ELEMENT[buffers.Count + 1]; fileSegments[buffers.Count].Buffer = IntPtr.Zero; MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; - int totalBytes = 0; for (int i = 0; i < buffers.Count; i++) { Memory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); memoryHandles[i] = memoryHandle; fileSegments[i] = new Interop.Kernel32.FILE_SEGMENT_ELEMENT { Buffer = new IntPtr(memoryHandle.Pointer) }; - - checked - { - totalBytes += buffer.Length; - } } - return (fileSegments.AsMemory().Pin(), memoryHandles, totalBytes); + return (fileSegments.AsMemory().Pin(), memoryHandles); } - // any ideas on geting rid of the code duplication are more than welcomed - private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles, int totalBytes) PrepareMemorySegments(IReadOnlyList> buffers) + private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) PrepareMemorySegments(IReadOnlyList> buffers) { // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " Interop.Kernel32.FILE_SEGMENT_ELEMENT[] fileSegments = new Interop.Kernel32.FILE_SEGMENT_ELEMENT[buffers.Count + 1]; fileSegments[buffers.Count].Buffer = IntPtr.Zero; MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; - int totalBytes = 0; for (int i = 0; i < buffers.Count; i++) { ReadOnlyMemory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); memoryHandles[i] = memoryHandle; fileSegments[i] = new Interop.Kernel32.FILE_SEGMENT_ELEMENT { Buffer = new IntPtr(memoryHandle.Pointer) }; - - checked - { - totalBytes += buffer.Length; - } } - return (fileSegments.AsMemory().Pin(), memoryHandles, totalBytes); + return (fileSegments.AsMemory().Pin(), memoryHandles); } private static NativeOverlapped GetNativeOverlapped(long fileOffset) From e462d4980471f7d33ddf04a57feea3f8e8d1f533 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 13:38:57 +0200 Subject: [PATCH 43/71] Unix fixes --- src/libraries/Native/Unix/System.Native/pal_io.c | 10 ++++++++-- .../src/System/IO/RandomAccess.Unix.cs | 8 ++++---- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index ad5e405b38e5d..31d497947e467 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1505,7 +1505,10 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo count += current; - // we stop on the first incomplete operation (the next call would fail) + // Incomplete pread operation may happen for two reasons: + // a) We have reached EOF (the next call to pread would return 0). + // b) The operation was interrupted by a signal handler. + // To mimic preadv, we stop on the first incomplete operation. if (current != vector.iov_len) { return (int64_t)count; @@ -1542,7 +1545,10 @@ int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorC count += current; - // we stop on the first incomplete operation (the next call would fail) + // Incomplete write operation may happen for few reasons: + // a) There was not enough space available or the file is too large for given file system. + // b) The operation was interrupted by a signal handler. + // To mimic pwritev, we stop on the first incomplete operation. if (current != vector.iov_len) { return (int64_t)count; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 548e30711a3be..4cac1441b82cf 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -66,7 +66,7 @@ private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory(Task.Factory.StartNew(state => { - var args = ((SafeFileHandle handle, Memory buffer, long fileOffset))state; + var args = ((SafeFileHandle handle, Memory buffer, long fileOffset))state!; return ReadAtOffset(args.handle, args.buffer.Span, args.fileOffset); }, (handle, buffer, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } @@ -76,7 +76,7 @@ private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, I { return new ValueTask(Task.Factory.StartNew(state => { - var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state; + var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state!; return ReadScatterAtOffset(args.handle, args.buffers, args.fileOffset); }, (handle, buffers, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } @@ -123,7 +123,7 @@ private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnly { return new ValueTask(Task.Factory.StartNew(state => { - var args = ((SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset))state; + var args = ((SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset))state!; return WriteAtOffset(args.handle, args.buffer.Span, args.fileOffset); }, (handle, buffer, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } @@ -133,7 +133,7 @@ private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, I { return new ValueTask(Task.Factory.StartNew(state => { - var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state; + var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state!; return WriteGatherAtOffset(args.handle, args.buffers, args.fileOffset); }, (handle, buffers, fileOffset), cancellationToken, TaskCreationOptions.DenyChildAttach, TaskScheduler.Default)); } From 253faa1fe1030eea52297ca4a266f7b69ae2cebf Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 14:18:26 +0200 Subject: [PATCH 44/71] skip the test that uses System.IO.Pipes for Browser (it's not supported) --- src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index e337b10719160..88040476c6f4e 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -44,6 +44,7 @@ public void ThrowsObjectDisposedExceptionForDisposedHandle() } [Fact] + [SkipOnPlatform(TestPlatforms.Browser, "System.IO.Pipes aren't supported on browser")] public void ThrowsNotSupportedExceptionForUnseekableFile() { using (var server = new AnonymousPipeServerStream(PipeDirection.Out)) From fd86735075250ecc35fe68f7b7fe98d2d2892a3e Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 15:02:25 +0200 Subject: [PATCH 45/71] remove the invalid assumption that every file file opened from path must be a regular file --- .../Win32/SafeHandles/SafeFileHandle.Windows.cs | 12 +++++------- .../IO/Strategies/FileStreamHelpers.Windows.cs | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index ac582f0a8e025..066c0b152ddb5 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -11,7 +11,7 @@ namespace Microsoft.Win32.SafeHandles { public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - internal const FileOptions NoBuffering = (FileOptions)(0x20000000); + internal const FileOptions NoBuffering = (FileOptions)0x20000000; private FileOptions _fileOptions = (FileOptions)(-1); private int _fileType = -1; @@ -24,12 +24,11 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : base(ownsHand SetHandle(preexistingHandle); } - private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fileOptions, int fileType) : base(ownsHandle) + private SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle, FileOptions fileOptions) : base(ownsHandle) { SetHandle(preexistingHandle); _fileOptions = fileOptions; - _fileType = fileType; } public bool IsAsync => (GetFileOptions() & FileOptions.Asynchronous) != 0; @@ -47,8 +46,7 @@ internal static unsafe SafeFileHandle Open(string fullPath, FileMode mode, FileA SafeFileHandle fileHandle = new SafeFileHandle( NtCreateFile(fullPath, mode, access, share, options, preallocationSize), ownsHandle: true, - options, - Interop.Kernel32.FileTypes.FILE_TYPE_DISK); // similar to FileStream, we assume that only disk files can be referenced by path on Windows + options); fileHandle.InitThreadPoolBindingIfNeeded(); @@ -183,9 +181,9 @@ internal unsafe FileOptions GetFileOptions() return _fileOptions = result; } - private int GetFileType() + internal int GetFileType() { - if (_fileType != -1) + if (_fileType == -1) { _fileType = Interop.Kernel32.GetFileType(this); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs index 58732496ea847..39275721010fb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.Windows.cs @@ -150,7 +150,7 @@ internal static void ValidateFileTypeForNonExtendedPaths(SafeFileHandle handle, // we were explicitly passed a path that has \\?\. GetFullPath() will turn paths like C:\foo\con.txt into // \\.\CON, so we'll only allow the \\?\ syntax. - int fileType = Interop.Kernel32.GetFileType(handle); + int fileType = handle.GetFileType(); if (fileType != Interop.Kernel32.FileTypes.FILE_TYPE_DISK) { int errorCode = fileType == Interop.Kernel32.FileTypes.FILE_TYPE_UNKNOWN From 6241b088c5e7722613d0e641933355dbc5d69aa5 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 15:16:41 +0200 Subject: [PATCH 46/71] support ReadFileScatter and WriteFileGather on 32-bit systems as we don't support Itanium anymore and don't need to worry about it --- .../Kernel32/Interop.FileScatterGather.cs | 13 ++---------- .../src/System/IO/RandomAccess.Windows.cs | 21 ++++++++----------- 2 files changed, 11 insertions(+), 23 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs index 8cdb8877187f6..f573232523d08 100644 --- a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs +++ b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs @@ -9,19 +9,10 @@ internal static partial class Interop { internal static partial class Kernel32 { - [StructLayout(LayoutKind.Explicit, Size = 8)] - internal unsafe struct FILE_SEGMENT_ELEMENT - { - [FieldOffset(0)] - public IntPtr Buffer; - [FieldOffset(0)] - public ulong Alignment; - } - [DllImport(Libraries.Kernel32, SetLastError = true)] internal static extern unsafe int ReadFileScatter( SafeHandle handle, - FILE_SEGMENT_ELEMENT* segments, + long* segments, int numBytesToRead, IntPtr reserved_mustBeZero, NativeOverlapped* overlapped); @@ -29,7 +20,7 @@ internal static extern unsafe int ReadFileScatter( [DllImport(Libraries.Kernel32, SetLastError = true)] internal static extern unsafe int WriteFileGather( SafeHandle handle, - FILE_SEGMENT_ELEMENT* segments, + long* segments, int numBytesToWrite, IntPtr reserved_mustBeZero, NativeOverlapped* overlapped); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 813f6f0ec302a..42055daf26b16 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -300,11 +300,8 @@ private static ValueTask ReadScatterAtOffsetAsync(SafeFileHandle handle, I // From https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-readfilescatter: // "The file handle must be created with the GENERIC_READ right, and the FILE_FLAG_OVERLAPPED and FILE_FLAG_NO_BUFFERING flags." - // "This function is not supported for 32-bit applications by WOW64 on Itanium-based systems." private static bool CanUseScatterGatherWindowsAPIs(SafeFileHandle handle) - => IntPtr.Size != sizeof(int) - && handle.IsAsync - && ((handle.GetFileOptions() & SafeFileHandle.NoBuffering) != 0); + => handle.IsAsync && ((handle.GetFileOptions() & SafeFileHandle.NoBuffering) != 0); private static async ValueTask ReadScatterAtOffsetSingleSyscallAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, int totalBytes, CancellationToken cancellationToken) @@ -340,7 +337,7 @@ private static unsafe ValueTask ReadFileScatterAsync(SafeFileHandle handle, NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(Memory.Empty, fileOffset); Debug.Assert(pinnedSegments.Pointer != null); - if (Interop.Kernel32.ReadFileScatter(handle, (Interop.Kernel32.FILE_SEGMENT_ELEMENT*)pinnedSegments.Pointer, bytesToRead, IntPtr.Zero, nativeOverlapped) == 0) + if (Interop.Kernel32.ReadFileScatter(handle, (long*)pinnedSegments.Pointer, bytesToRead, IntPtr.Zero, nativeOverlapped) == 0) { // The operation failed, or it's pending. int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); @@ -473,7 +470,7 @@ private static unsafe ValueTask WriteFileGatherAsync(SafeFileHandle handle, Debug.Assert(vts._memoryHandle.Pointer != null); // Queue an async WriteFile operation. - if (Interop.Kernel32.WriteFileGather(handle, (Interop.Kernel32.FILE_SEGMENT_ELEMENT*)pinnedSegments.Pointer, bytesToWrite, IntPtr.Zero, nativeOverlapped) == 0) + if (Interop.Kernel32.WriteFileGather(handle, (long*)pinnedSegments.Pointer, bytesToWrite, IntPtr.Zero, nativeOverlapped) == 0) { // The operation failed, or it's pending. int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); @@ -507,8 +504,8 @@ private static unsafe ValueTask WriteFileGatherAsync(SafeFileHandle handle, private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) PrepareMemorySegments(IReadOnlyList> buffers) { // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " - Interop.Kernel32.FILE_SEGMENT_ELEMENT[] fileSegments = new Interop.Kernel32.FILE_SEGMENT_ELEMENT[buffers.Count + 1]; - fileSegments[buffers.Count].Buffer = IntPtr.Zero; + long[] fileSegments = new long[buffers.Count + 1]; + fileSegments[buffers.Count] = 0; MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; for (int i = 0; i < buffers.Count; i++) @@ -516,7 +513,7 @@ private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles Memory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); memoryHandles[i] = memoryHandle; - fileSegments[i] = new Interop.Kernel32.FILE_SEGMENT_ELEMENT { Buffer = new IntPtr(memoryHandle.Pointer) }; + fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); } return (fileSegments.AsMemory().Pin(), memoryHandles); @@ -525,8 +522,8 @@ private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) PrepareMemorySegments(IReadOnlyList> buffers) { // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " - Interop.Kernel32.FILE_SEGMENT_ELEMENT[] fileSegments = new Interop.Kernel32.FILE_SEGMENT_ELEMENT[buffers.Count + 1]; - fileSegments[buffers.Count].Buffer = IntPtr.Zero; + long[] fileSegments = new long[buffers.Count + 1]; + fileSegments[buffers.Count] = 0; MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; for (int i = 0; i < buffers.Count; i++) @@ -534,7 +531,7 @@ private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles ReadOnlyMemory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); memoryHandles[i] = memoryHandle; - fileSegments[i] = new Interop.Kernel32.FILE_SEGMENT_ELEMENT { Buffer = new IntPtr(memoryHandle.Pointer) }; + fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); } return (fileSegments.AsMemory().Pin(), memoryHandles); From 648ab9b648e8d84bea1f5258f5e997b278fba897 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 16:52:42 +0200 Subject: [PATCH 47/71] move the unpinning to finally block --- .../src/System/IO/RandomAccess.Unix.cs | 30 ++++++++++++------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 4cac1441b82cf..347a610271f7b 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -48,14 +48,19 @@ private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyL } long result; - fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + try { - result = Interop.Sys.PReadV(handle, pinnedVectors, buffers.Count, fileOffset); + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + { + result = Interop.Sys.PReadV(handle, pinnedVectors, buffers.Count, fileOffset); + } } - - foreach (MemoryHandle memoryHandle in handles) + finally { - memoryHandle.Dispose(); + foreach (MemoryHandle memoryHandle in handles) + { + memoryHandle.Dispose(); + } } return FileStreamHelpers.CheckFileCall(result, path: null); @@ -105,14 +110,19 @@ private static unsafe long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyL } long result; - fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + try { - result = Interop.Sys.PWriteV(handle, pinnedVectors, buffers.Count, fileOffset); + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) + { + result = Interop.Sys.PWriteV(handle, pinnedVectors, buffers.Count, fileOffset); + } } - - foreach (MemoryHandle memoryHandle in handles) + finally { - memoryHandle.Dispose(); + foreach (MemoryHandle memoryHandle in handles) + { + memoryHandle.Dispose(); + } } return FileStreamHelpers.CheckFileCall(result, path: null); From e075441e31cdc19a3c8910e25c15c0ece48b9ffc Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 16:57:26 +0200 Subject: [PATCH 48/71] update the comment --- src/libraries/Native/Unix/System.Native/pal_io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 31d497947e467..5ed8e449a6c05 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1506,7 +1506,7 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo count += current; // Incomplete pread operation may happen for two reasons: - // a) We have reached EOF (the next call to pread would return 0). + // a) We have reached EOF. // b) The operation was interrupted by a signal handler. // To mimic preadv, we stop on the first incomplete operation. if (current != vector.iov_len) @@ -1545,7 +1545,7 @@ int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorC count += current; - // Incomplete write operation may happen for few reasons: + // Incomplete pwrite operation may happen for few reasons: // a) There was not enough space available or the file is too large for given file system. // b) The operation was interrupted by a signal handler. // To mimic pwritev, we stop on the first incomplete operation. From b1d300626f9caa446d7a257f0b7fa5096a61b257 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 17:13:45 +0200 Subject: [PATCH 49/71] move the call to Dispose to the catch block --- .../Win32/SafeHandles/SafeFileHandle.Windows.cs | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 066c0b152ddb5..dab1e8ba39ff2 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -117,16 +117,11 @@ internal void InitThreadPoolBindingIfNeeded() } catch (ArgumentException ex) { + // We should close the handle so that the handle is not open until SafeFileHandle GC + Dispose(); + throw new IOException(SR.IO_BindHandleFailed, ex); } - finally - { - if (ThreadPoolBinding == null) - { - // We should close the handle so that the handle is not open until SafeFileHandle GC - Dispose(); - } - } } } From 98eaf249a17f15d56b093debfc38b9cf2c45a35b Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 17:42:10 +0200 Subject: [PATCH 50/71] InitThreadPoolBindingIfNeeded can dispose the handle only if we own it --- .../Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs | 7 +++++-- .../src/System/Runtime/InteropServices/SafeHandle.cs | 6 +++--- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index dab1e8ba39ff2..acb888372188e 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -117,8 +117,11 @@ internal void InitThreadPoolBindingIfNeeded() } catch (ArgumentException ex) { - // We should close the handle so that the handle is not open until SafeFileHandle GC - Dispose(); + if (ownsHandle) + { + // We should close the handle so that the handle is not open until SafeFileHandle GC + Dispose(); + } throw new IOException(SR.IO_BindHandleFailed, ex); } diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs index 6d1bbf831773a..8dfe4a57aa896 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs @@ -25,7 +25,7 @@ public abstract partial class SafeHandle : CriticalFinalizerObject, IDisposable /// Combined ref count and closed/disposed flags (so we can atomically modify them). private volatile int _state; /// Whether we can release this handle. - private readonly bool _ownsHandle; + protected readonly bool ownsHandle; /// Whether constructor completed. private volatile bool _fullyInitialized; @@ -54,7 +54,7 @@ protected SafeHandle(IntPtr invalidHandleValue, bool ownsHandle) { handle = invalidHandleValue; _state = StateBits.RefCountOne; // Ref count 1 and not closed or disposed. - _ownsHandle = ownsHandle; + this.ownsHandle = ownsHandle; if (!ownsHandle) { @@ -215,7 +215,7 @@ private void InternalRelease(bool disposeOrFinalizeOperation) // transitioning the handle to closed, however, since setting the closed // state will cause IsInvalid to always return true. performRelease = ((oldState & (StateBits.RefCount | StateBits.Closed)) == StateBits.RefCountOne) && - _ownsHandle && + ownsHandle && !IsInvalid; // Attempt the update to the new state, fail and retry if the initial From 76a432665e9f8f0a49a40b472a7b7973d6da0cb3 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 17:55:14 +0200 Subject: [PATCH 51/71] when 0 is allowed, the exception message should ask for non-negative number instead of positive number --- .../System.Private.CoreLib/src/System/IO/FileStream.cs | 6 +++--- .../src/System/IO/Strategies/FileStreamHelpers.cs | 4 ++-- .../System.Private.CoreLib/src/System/ThrowHelper.cs | 6 ++++++ 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs index 18de4e27955e5..d1161c7d71002 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs @@ -76,7 +76,7 @@ private static void ValidateHandle(SafeFileHandle handle, FileAccess access, int } else if (bufferSize < 0) { - throw new ArgumentOutOfRangeException(nameof(bufferSize), SR.ArgumentOutOfRange_NeedPosNum); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedNonNegNum(nameof(bufferSize)); } else if (handle.IsClosed) { @@ -199,7 +199,7 @@ public virtual void Lock(long position, long length) { if (position < 0 || length < 0) { - throw new ArgumentOutOfRangeException(position < 0 ? nameof(position) : nameof(length), SR.ArgumentOutOfRange_NeedNonNegNum); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedNonNegNum(position < 0 ? nameof(position) : nameof(length)); } else if (_strategy.IsClosed) { @@ -217,7 +217,7 @@ public virtual void Unlock(long position, long length) { if (position < 0 || length < 0) { - throw new ArgumentOutOfRangeException(position < 0 ? nameof(position) : nameof(length), SR.ArgumentOutOfRange_NeedNonNegNum); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedNonNegNum(position < 0 ? nameof(position) : nameof(length)); } else if (_strategy.IsClosed) { diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs index c2b2bbfeb158a..81e2042761b0a 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs @@ -86,11 +86,11 @@ internal static void ValidateArguments(string path, FileMode mode, FileAccess ac } else if (bufferSize < 0) { - ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(bufferSize)); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedNonNegNum(nameof(bufferSize)); } else if (preallocationSize < 0) { - ThrowHelper.ThrowArgumentOutOfRangeException_NeedPosNum(nameof(preallocationSize)); + ThrowHelper.ThrowArgumentOutOfRangeException_NeedNonNegNum(nameof(preallocationSize)); } // Write access validation diff --git a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs index ae242caef9cfe..117a0f05ec5a8 100644 --- a/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs +++ b/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @@ -492,6 +492,12 @@ internal static void ThrowArgumentOutOfRangeException_NeedPosNum(string? paramNa throw new ArgumentOutOfRangeException(paramName, SR.ArgumentOutOfRange_NeedPosNum); } + [DoesNotReturn] + internal static void ThrowArgumentOutOfRangeException_NeedNonNegNum(string paramName) + { + throw new ArgumentOutOfRangeException(paramName, SR.ArgumentOutOfRange_NeedNonNegNum); + } + private static Exception GetArraySegmentCtorValidationFailedException(Array? array, int offset, int count) { if (array == null) From dcd9e12a0304ee111c20a72eba9b757356517dce Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 18:19:30 +0200 Subject: [PATCH 52/71] fix the casting compilation error --- .../Native/Unix/System.Native/pal_io.c | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 5ed8e449a6c05..88456e3a0ae29 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1485,12 +1485,12 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo assert(vectors != NULL); assert(vectorCount >= 0); - ssize_t count = 0; + int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PREADV while ((count = preadv(fileDescriptor, vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else - ssize_t current; + int64_t current; for (int i = 0; i < vectorCount; i++) { struct iovec vector = vectors[i]; @@ -1508,16 +1508,16 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo // Incomplete pread operation may happen for two reasons: // a) We have reached EOF. // b) The operation was interrupted by a signal handler. - // To mimic preadv, we stop on the first incomplete operation. - if (current != vector.iov_len) + // To mimic preadv, we stop on the first incomplete operation. + if (current != (int64_t)vector.iov_len) { - return (int64_t)count; + return count; } } #endif assert(count >= -1); - return (int64_t)count; + return count; } int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset) @@ -1525,12 +1525,12 @@ int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorC assert(vectors != NULL); assert(vectorCount >= 0); - ssize_t count = 0; + int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PWRITEV while ((count = pwritev(fileDescriptor, vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else - ssize_t current; + int64_t current; for (int i = 0; i < vectorCount; i++) { struct iovec vector = vectors[i]; @@ -1548,14 +1548,14 @@ int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorC // Incomplete pwrite operation may happen for few reasons: // a) There was not enough space available or the file is too large for given file system. // b) The operation was interrupted by a signal handler. - // To mimic pwritev, we stop on the first incomplete operation. - if (current != vector.iov_len) + // To mimic pwritev, we stop on the first incomplete operation. + if (current != (int64_t)vector.iov_len) { - return (int64_t)count; + return count; } } #endif assert(count >= -1); - return (int64_t)count; + return count; } From 2b6a555fbcb643701d696f3efdb149293ac39de0 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 18:49:23 +0200 Subject: [PATCH 53/71] don't make ownsHandle a protected field as it would become a part of public contract --- .../Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs | 2 +- .../src/System/Runtime/InteropServices/SafeHandle.cs | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index acb888372188e..b5d310a77f533 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -117,7 +117,7 @@ internal void InitThreadPoolBindingIfNeeded() } catch (ArgumentException ex) { - if (ownsHandle) + if (OwnsHandle) { // We should close the handle so that the handle is not open until SafeFileHandle GC Dispose(); diff --git a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs index 8dfe4a57aa896..227dbf0641f5e 100644 --- a/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs +++ b/src/libraries/System.Private.CoreLib/src/System/Runtime/InteropServices/SafeHandle.cs @@ -25,7 +25,7 @@ public abstract partial class SafeHandle : CriticalFinalizerObject, IDisposable /// Combined ref count and closed/disposed flags (so we can atomically modify them). private volatile int _state; /// Whether we can release this handle. - protected readonly bool ownsHandle; + private readonly bool _ownsHandle; /// Whether constructor completed. private volatile bool _fullyInitialized; @@ -54,7 +54,7 @@ protected SafeHandle(IntPtr invalidHandleValue, bool ownsHandle) { handle = invalidHandleValue; _state = StateBits.RefCountOne; // Ref count 1 and not closed or disposed. - this.ownsHandle = ownsHandle; + _ownsHandle = ownsHandle; if (!ownsHandle) { @@ -74,6 +74,8 @@ protected SafeHandle(IntPtr invalidHandleValue, bool ownsHandle) } #endif + internal bool OwnsHandle => _ownsHandle; + protected internal void SetHandle(IntPtr handle) => this.handle = handle; public IntPtr DangerousGetHandle() => handle; @@ -215,7 +217,7 @@ private void InternalRelease(bool disposeOrFinalizeOperation) // transitioning the handle to closed, however, since setting the closed // state will cause IsInvalid to always return true. performRelease = ((oldState & (StateBits.RefCount | StateBits.Closed)) == StateBits.RefCountOne) && - ownsHandle && + _ownsHandle && !IsInvalid; // Attempt the update to the new state, fail and retry if the initial From 4a6bb621425e450a895fdbee3e7e1147b0e53f91 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 18:52:34 +0200 Subject: [PATCH 54/71] move IOVector from pal_networking.h to pal_io.h and use it in pal_io.c to hopefully fix the FreeBSD build issue --- .../Native/Unix/System.Native/pal_io.c | 20 +++++++++---------- .../Native/Unix/System.Native/pal_io.h | 12 +++++++++-- .../Unix/System.Native/pal_networking.c | 5 ++--- .../Unix/System.Native/pal_networking.h | 9 +-------- 4 files changed, 23 insertions(+), 23 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 88456e3a0ae29..7027d77e597ec 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1480,7 +1480,7 @@ int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferSize, int64 return (int32_t)count; } -int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset) +int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) { assert(vectors != NULL); assert(vectorCount >= 0); @@ -1488,13 +1488,13 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PREADV - while ((count = preadv(fileDescriptor, vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + while ((count = preadv(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; for (int i = 0; i < vectorCount; i++) { - struct iovec vector = vectors[i]; - while ((current = pread(fileDescriptor, vector.iov_base, vector.iov_len, (off_t)(fileOffset + count))) < 0 && errno == EINTR); + IOVector vector = vectors[i]; + while ((current = pread(fileDescriptor, vector.Base, vector.Count, (off_t)(fileOffset + count))) < 0 && errno == EINTR); if (current < 0) { @@ -1509,7 +1509,7 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo // a) We have reached EOF. // b) The operation was interrupted by a signal handler. // To mimic preadv, we stop on the first incomplete operation. - if (current != (int64_t)vector.iov_len) + if (current != (int64_t)vector.Count) { return count; } @@ -1520,7 +1520,7 @@ int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCo return count; } -int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset) +int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset) { assert(vectors != NULL); assert(vectorCount >= 0); @@ -1528,13 +1528,13 @@ int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorC int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); #if HAVE_PWRITEV - while ((count = pwritev(fileDescriptor, vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); + while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; for (int i = 0; i < vectorCount; i++) { - struct iovec vector = vectors[i]; - while ((current = pwrite(fileDescriptor, vector.iov_base, vector.iov_len, (off_t)(fileOffset + count))) < 0 && errno == EINTR); + IOVector vector = vectors[i]; + while ((current = pwrite(fileDescriptor, vector.Base, vector.Count, (off_t)(fileOffset + count))) < 0 && errno == EINTR); if (current < 0) { @@ -1549,7 +1549,7 @@ int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorC // a) There was not enough space available or the file is too large for given file system. // b) The operation was interrupted by a signal handler. // To mimic pwritev, we stop on the first incomplete operation. - if (current != (int64_t)vector.iov_len) + if (current != (int64_t)vector.Count) { return count; } diff --git a/src/libraries/Native/Unix/System.Native/pal_io.h b/src/libraries/Native/Unix/System.Native/pal_io.h index 52fb8522e2de5..1dc70387ad5ea 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.h +++ b/src/libraries/Native/Unix/System.Native/pal_io.h @@ -41,6 +41,14 @@ typedef struct // add more fields when needed. } ProcessStatus; +// NOTE: the layout of this type is intended to exactly match the layout of a `struct iovec`. There are +// assertions in pal_networking.c that validate this. +typedef struct +{ + uint8_t* Base; + uintptr_t Count; +} IOVector; + /* Provide consistent access to nanosecond fields, if they exist. */ /* Seconds are always available through st_atime, st_mtime, st_ctime. */ @@ -750,11 +758,11 @@ PALEXPORT int32_t SystemNative_PWrite(intptr_t fd, void* buffer, int32_t bufferS * * Returns the number of bytes read on success; otherwise, -1 is returned an errno is set. */ -PALEXPORT int64_t SystemNative_PReadV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset); +PALEXPORT int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset); /** * Writes the number of bytes specified in the buffers into the specified, opened file descriptor at specified offset. * * Returns the number of bytes written on success; otherwise, -1 is returned an errno is set. */ -PALEXPORT int64_t SystemNative_PWriteV(intptr_t fd, struct iovec* vectors, int32_t vectorCount, int64_t fileOffset); +PALEXPORT int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t fileOffset); diff --git a/src/libraries/Native/Unix/System.Native/pal_networking.c b/src/libraries/Native/Unix/System.Native/pal_networking.c index 11124f34c29bb..3236fe26acfeb 100644 --- a/src/libraries/Native/Unix/System.Native/pal_networking.c +++ b/src/libraries/Native/Unix/System.Native/pal_networking.c @@ -3,7 +3,6 @@ #include "pal_config.h" #include "pal_networking.h" -#include "pal_io.h" #include "pal_safecrt.h" #include "pal_utilities.h" #include @@ -3104,11 +3103,11 @@ int32_t SystemNative_Disconnect(intptr_t socket) addr.sa_family = AF_UNSPEC; err = connect(fd, &addr, sizeof(addr)); - if (err != 0) + if (err != 0) { // On some older kernels connect(AF_UNSPEC) may fail. Fall back to shutdown in these cases: err = shutdown(fd, SHUT_RDWR); - } + } #elif HAVE_DISCONNECTX // disconnectx causes a FIN close on OSX. It's the best we can do. err = disconnectx(fd, SAE_ASSOCID_ANY, SAE_CONNID_ANY); diff --git a/src/libraries/Native/Unix/System.Native/pal_networking.h b/src/libraries/Native/Unix/System.Native/pal_networking.h index cd2cf91e45eec..4bf9de658e26f 100644 --- a/src/libraries/Native/Unix/System.Native/pal_networking.h +++ b/src/libraries/Native/Unix/System.Native/pal_networking.h @@ -4,6 +4,7 @@ #pragma once #include "pal_compiler.h" +#include "pal_io.h" #include "pal_types.h" #include "pal_errno.h" #include @@ -275,14 +276,6 @@ typedef struct int32_t Seconds; // Number of seconds to linger for } LingerOption; -// NOTE: the layout of this type is intended to exactly match the layout of a `struct iovec`. There are -// assertions in pal_networking.c that validate this. -typedef struct -{ - uint8_t* Base; - uintptr_t Count; -} IOVector; - typedef struct { uint8_t* SocketAddress; From 5ec9e1fe14f400d41508ff495ff2634055da92da Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 19:20:33 +0200 Subject: [PATCH 55/71] don't use nullable field to store canSeek information. Store the file type instead (int) --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 43 +++++++++++-------- 1 file changed, 25 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 689dc4eac4cb8..44b0d72cf9d01 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -10,7 +10,7 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private bool? _canSeek; + private int _fileType = -1; public SafeFileHandle() : this(ownsHandle: true) { @@ -29,7 +29,7 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : this(ownsHand public bool IsAsync { get; private set; } - internal bool CanSeek => !IsClosed && (_canSeek ??= Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + internal bool CanSeek => !IsClosed && GetFileType() == Interop.Sys.FileTypes.S_IFREG; /// Opens the specified file with the requested flags and mode. /// The path to the file. @@ -66,27 +66,12 @@ private static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, int // Make sure it's not a directory; we do this after opening it once we have a file descriptor // to avoid race conditions. - Interop.Sys.FileStatus status; - if (Interop.Sys.FStat(handle, out status) != 0) - { - handle.Dispose(); - throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path); - } - if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR) + if (handle.GetFileType(path) == Interop.Sys.FileTypes.S_IFDIR) { handle.Dispose(); throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), path, isDirectory: true); } - if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFREG) - { - // we take advantage of the information provided by the fstat syscall - // and for regular files (most common case) - // avoid one extra sys call for determining whether file can be seeked - handle._canSeek = true; - Debug.Assert(Interop.Sys.LSeek(handle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); - } - return handle; } @@ -302,5 +287,27 @@ private void Init(string path, FileMode mode, FileAccess access, FileShare share } } } + + private int GetFileType(string? path = null) + { + if (_fileType == -1) + { + // Make sure it's not a directory; we do this after opening it once we have a file descriptor + // to avoid race conditions. + if (Interop.Sys.FStat(this, out Interop.Sys.FileStatus status) != 0) + { + if (OwnsHandle) + { + Dispose(); + } + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path); + } + + _fileType = status.Mode & Interop.Sys.FileTypes.S_IFMT; + Debug.Assert((Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0) == (_fileType == Interop.Sys.FileTypes.S_IFREG)); + } + + return _fileType; + } } } From 4d002b2dfa3a0fa137ecef298a2b6e6ac32f06a6 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Mon, 7 Jun 2021 20:20:23 +0200 Subject: [PATCH 56/71] update outdated debug assert --- .../src/System/IO/RandomAccess.Windows.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 42055daf26b16..5f742eddc6c71 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -467,7 +467,7 @@ private static unsafe ValueTask WriteFileGatherAsync(SafeFileHandle handle, try { NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(ReadOnlyMemory.Empty, fileOffset); - Debug.Assert(vts._memoryHandle.Pointer != null); + Debug.Assert(pinnedSegments.Pointer != null); // Queue an async WriteFile operation. if (Interop.Kernel32.WriteFileGather(handle, (long*)pinnedSegments.Pointer, bytesToWrite, IntPtr.Zero, nativeOverlapped) == 0) From d5c9229f535ecc6c05a3720f5a0fef7d54eaf088 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 8 Jun 2021 08:30:42 +0200 Subject: [PATCH 57/71] Revert "don't use nullable field to store canSeek information. Store the file type instead (int)" This reverts commit 5ec9e1fe14f400d41508ff495ff2634055da92da. --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 43 ++++++++----------- 1 file changed, 18 insertions(+), 25 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 44b0d72cf9d01..689dc4eac4cb8 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -10,7 +10,7 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private int _fileType = -1; + private bool? _canSeek; public SafeFileHandle() : this(ownsHandle: true) { @@ -29,7 +29,7 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : this(ownsHand public bool IsAsync { get; private set; } - internal bool CanSeek => !IsClosed && GetFileType() == Interop.Sys.FileTypes.S_IFREG; + internal bool CanSeek => !IsClosed && (_canSeek ??= Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); /// Opens the specified file with the requested flags and mode. /// The path to the file. @@ -66,12 +66,27 @@ private static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, int // Make sure it's not a directory; we do this after opening it once we have a file descriptor // to avoid race conditions. - if (handle.GetFileType(path) == Interop.Sys.FileTypes.S_IFDIR) + Interop.Sys.FileStatus status; + if (Interop.Sys.FStat(handle, out status) != 0) + { + handle.Dispose(); + throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path); + } + if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFDIR) { handle.Dispose(); throw Interop.GetExceptionForIoErrno(Interop.Error.EACCES.Info(), path, isDirectory: true); } + if ((status.Mode & Interop.Sys.FileTypes.S_IFMT) == Interop.Sys.FileTypes.S_IFREG) + { + // we take advantage of the information provided by the fstat syscall + // and for regular files (most common case) + // avoid one extra sys call for determining whether file can be seeked + handle._canSeek = true; + Debug.Assert(Interop.Sys.LSeek(handle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + } + return handle; } @@ -287,27 +302,5 @@ private void Init(string path, FileMode mode, FileAccess access, FileShare share } } } - - private int GetFileType(string? path = null) - { - if (_fileType == -1) - { - // Make sure it's not a directory; we do this after opening it once we have a file descriptor - // to avoid race conditions. - if (Interop.Sys.FStat(this, out Interop.Sys.FileStatus status) != 0) - { - if (OwnsHandle) - { - Dispose(); - } - throw Interop.GetExceptionForIoErrno(Interop.Sys.GetLastErrorInfo(), path); - } - - _fileType = status.Mode & Interop.Sys.FileTypes.S_IFMT; - Debug.Assert((Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0) == (_fileType == Interop.Sys.FileTypes.S_IFREG)); - } - - return _fileType; - } } } From 994b1123dbdc233a3af4ab965493f8f1d969637c Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 8 Jun 2021 08:45:54 +0200 Subject: [PATCH 58/71] CanSeek should be thread-safe --- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 24 ++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 689dc4eac4cb8..4029e330e935e 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -10,7 +10,8 @@ namespace Microsoft.Win32.SafeHandles { public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { - private bool? _canSeek; + // not using bool? as it's not thread safe + private NullableBool _canSeek = NullableBool.Undefined; public SafeFileHandle() : this(ownsHandle: true) { @@ -29,7 +30,7 @@ public SafeFileHandle(IntPtr preexistingHandle, bool ownsHandle) : this(ownsHand public bool IsAsync { get; private set; } - internal bool CanSeek => !IsClosed && (_canSeek ??= Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); + internal bool CanSeek => !IsClosed && GetCanSeek(); /// Opens the specified file with the requested flags and mode. /// The path to the file. @@ -83,7 +84,7 @@ private static SafeFileHandle Open(string path, Interop.Sys.OpenFlags flags, int // we take advantage of the information provided by the fstat syscall // and for regular files (most common case) // avoid one extra sys call for determining whether file can be seeked - handle._canSeek = true; + handle._canSeek = NullableBool.True; Debug.Assert(Interop.Sys.LSeek(handle, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0); } @@ -302,5 +303,22 @@ private void Init(string path, FileMode mode, FileAccess access, FileShare share } } } + + private bool GetCanSeek() + { + if (_canSeek == NullableBool.Undefined) + { + _canSeek = Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0 ? NullableBool.True : NullableBool.False; + } + + return _canSeek == NullableBool.True; + } + + private enum NullableBool + { + Undefined = 0, + False = -1, + True = 1 + } } } From 9405abeca96be47f92f281a820eac2fa75a573d6 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 8 Jun 2021 12:03:07 +0200 Subject: [PATCH 59/71] add more debug asserts to try to see why two CI legs have failed with mysterious "bad file descriptor" error --- .../src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs | 3 +++ .../System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index 4029e330e935e..f0b4ea17d8fb1 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -306,6 +306,9 @@ private void Init(string path, FileMode mode, FileAccess access, FileShare share private bool GetCanSeek() { + Debug.Assert(!IsClosed); + Debug.Assert(!IsInvalid); + if (_canSeek == NullableBool.Undefined) { _canSeek = Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0 ? NullableBool.True : NullableBool.False; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs index 56716deb0ccbc..5ee047869c9a6 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/Net5CompatFileStreamStrategy.Unix.cs @@ -610,7 +610,8 @@ public override long Seek(long offset, SeekOrigin origin) /// The new position in the stream. private long SeekCore(SafeFileHandle fileHandle, long offset, SeekOrigin origin, bool closeInvalidHandle = false) { - Debug.Assert(!fileHandle.IsClosed && fileHandle.CanSeek); + Debug.Assert(!fileHandle.IsInvalid); + Debug.Assert(fileHandle.CanSeek); Debug.Assert(origin >= SeekOrigin.Begin && origin <= SeekOrigin.End); long pos = FileStreamHelpers.CheckFileCall(Interop.Sys.LSeek(fileHandle, offset, (Interop.Sys.SeekWhence)(int)origin), _path); // SeekOrigin values are the same as Interop.libc.SeekWhence values From 752ff9dae78b609ba6e2eec1ec233dee95b87a75 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 8 Jun 2021 15:25:11 +0200 Subject: [PATCH 60/71] fix invalid link (it was pointing to a closed duplicate) --- src/libraries/System.IO.FileSystem/tests/File/AppendAsync.cs | 2 +- src/libraries/System.IO.FileSystem/tests/File/Create.cs | 2 +- .../System.IO.FileSystem/tests/File/EncryptDecrypt.cs | 2 +- .../System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs | 2 +- .../System.IO.FileSystem/tests/File/ReadWriteAllLinesAsync.cs | 2 +- .../System.IO.FileSystem/tests/File/ReadWriteAllTextAsync.cs | 4 ++-- .../System.IO.FileSystem/tests/FileStream/CopyToAsync.cs | 2 +- .../tests/FileStream/FileStreamConformanceTests.cs | 4 ++-- .../System.IO.FileSystem/tests/FileStream/IsAsync.cs | 2 +- .../System.IO.FileSystem/tests/FileStream/ReadAsync.cs | 4 ++-- .../System.IO.FileSystem/tests/FileStream/SafeFileHandle.cs | 2 +- .../System.IO.FileSystem/tests/FileStream/WriteAsync.cs | 4 ++-- .../tests/FileStream/ctor_sfh_fa_buffer_async.cs | 2 +- .../tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs | 2 +- 14 files changed, 18 insertions(+), 18 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/File/AppendAsync.cs b/src/libraries/System.IO.FileSystem/tests/File/AppendAsync.cs index 04d7ebec7ae4d..457366266d69c 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/AppendAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/AppendAsync.cs @@ -86,7 +86,7 @@ public override Task TaskAlreadyCanceledAsync() } } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class File_AppendAllLinesAsync_Encoded : File_AppendAllLinesAsync { protected override Task WriteAsync(string path, string[] content) => diff --git a/src/libraries/System.IO.FileSystem/tests/File/Create.cs b/src/libraries/System.IO.FileSystem/tests/File/Create.cs index c5ce80d742684..bf0417dbfe061 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/Create.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/Create.cs @@ -367,7 +367,7 @@ public void NegativeBuffer() Assert.Throws(() => Create(GetTestFilePath(), -100)); } } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class File_Create_str_i_fo : File_Create_str_i { public override FileStream Create(string path) diff --git a/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs b/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs index 624eb2aae7cd8..984afef972bc4 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/EncryptDecrypt.cs @@ -7,7 +7,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class EncryptDecrypt : FileSystemTest { [Fact] diff --git a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs index 471f950c14dc1..ded6187328376 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllBytesAsync.cs @@ -8,7 +8,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class File_ReadWriteAllBytesAsync : FileSystemTest { [Fact] diff --git a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllLinesAsync.cs b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllLinesAsync.cs index d717e2839956e..503e9b1f5b9cf 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllLinesAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllLinesAsync.cs @@ -10,7 +10,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class File_ReadWriteAllLines_EnumerableAsync : FileSystemTest { #region Utilities diff --git a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllTextAsync.cs b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllTextAsync.cs index ac77b2509bb1a..d45cd73ecad89 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllTextAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/ReadWriteAllTextAsync.cs @@ -9,7 +9,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class File_ReadWriteAllTextAsync : FileSystemTest { #region Utilities @@ -144,7 +144,7 @@ public virtual Task TaskAlreadyCanceledAsync() #endregion } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class File_ReadWriteAllText_EncodedAsync : File_ReadWriteAllTextAsync { protected override Task WriteAsync(string path, string content) => diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/CopyToAsync.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/CopyToAsync.cs index f059c49ab4ce0..3a48354964b8a 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/CopyToAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/CopyToAsync.cs @@ -8,7 +8,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_CopyToAsync : FileSystemTest { [Theory] diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs index 0fbc771104569..d1fd4fe764f4d 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/FileStreamConformanceTests.cs @@ -205,7 +205,7 @@ public class BufferedSyncFileStreamStandaloneConformanceTests : FileStreamStanda protected override int BufferSize => 10; } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] [SkipOnPlatform(TestPlatforms.Browser, "lots of operations aren't supported on browser")] // copied from StreamConformanceTests base class due to https://github.com/xunit/xunit/issues/2186 public class UnbufferedAsyncFileStreamStandaloneConformanceTests : FileStreamStandaloneConformanceTests { @@ -218,7 +218,7 @@ public class UnbufferedAsyncFileStreamStandaloneConformanceTests : FileStreamSta #endif } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] [SkipOnPlatform(TestPlatforms.Browser, "lots of operations aren't supported on browser")] // copied from StreamConformanceTests base class due to https://github.com/xunit/xunit/issues/2186 public class BufferedAsyncFileStreamStandaloneConformanceTests : FileStreamStandaloneConformanceTests { diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/IsAsync.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/IsAsync.cs index b0900760c2687..e187468035fe6 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/IsAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/IsAsync.cs @@ -8,7 +8,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_IsAsync : FileSystemTest { [Fact] diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ReadAsync.cs index ceea76e50dd9e..d1703df589545 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ReadAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ReadAsync.cs @@ -94,14 +94,14 @@ public async Task ReadAsyncCanceledFile() } } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_ReadAsync_AsyncReads : FileStream_AsyncReads { protected override Task ReadAsync(FileStream stream, byte[] buffer, int offset, int count, CancellationToken cancellationToken) => stream.ReadAsync(buffer, offset, count, cancellationToken); } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_BeginEndRead_AsyncReads : FileStream_AsyncReads { protected override Task ReadAsync(FileStream stream, byte[] buffer, int offset, int count, CancellationToken cancellationToken) => diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/SafeFileHandle.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/SafeFileHandle.cs index 40064cd42c359..0740eed02f0d8 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/SafeFileHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/SafeFileHandle.cs @@ -9,7 +9,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_SafeFileHandle : FileSystemTest { [Fact] diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/WriteAsync.cs index 9e4db63e6fbc3..7f7fd8ad240ba 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/WriteAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/WriteAsync.cs @@ -324,7 +324,7 @@ public async Task WriteAsyncMiniStress() } } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_WriteAsync_AsyncWrites : FileStream_AsyncWrites { protected override Task WriteAsync(FileStream stream, byte[] buffer, int offset, int count, CancellationToken cancellationToken) => @@ -371,7 +371,7 @@ public void CancelledTokenFastPath() } } - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_BeginEndWrite_AsyncWrites : FileStream_AsyncWrites { protected override Task WriteAsync(FileStream stream, byte[] buffer, int offset, int count, CancellationToken cancellationToken) => diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer_async.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer_async.cs index f05cbce26f5b9..b2a8ba19a1d1f 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer_async.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_sfh_fa_buffer_async.cs @@ -6,7 +6,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_ctor_sfh_fa_buffer_async : FileStream_ctor_sfh_fa_buffer { protected sealed override FileStream CreateFileStream(SafeFileHandle handle, FileAccess access, int bufferSize) diff --git a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs index a68e709ba04dd..f55b7bee6c189 100644 --- a/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs +++ b/src/libraries/System.IO.FileSystem/tests/FileStream/ctor_str_fm_fa_fs_buffer_fo.cs @@ -5,7 +5,7 @@ namespace System.IO.Tests { - [ActiveIssue("https://github.com/dotnet/runtime/issues/34583", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class FileStream_ctor_str_fm_fa_fs_buffer_fo : FileStream_ctor_str_fm_fa_fs_buffer { protected sealed override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize) From e609c067d5a269a0ce3066566d7a25f24911afc6 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 8 Jun 2021 15:27:00 +0200 Subject: [PATCH 61/71] async File IO is not supported on Windows for Mono --- src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs | 1 + .../tests/RandomAccess/NoBuffering.Windows.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs | 1 + 6 files changed, 6 insertions(+) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index 88040476c6f4e..98ae9782ce3ef 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -69,6 +69,7 @@ public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() } [Fact] + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public void ThrowsArgumentExceptionForAsyncFileHandle() { if (ShouldThrowForAsyncHandle) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs index 201fe85803de4..09b05e6fc0b22 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs @@ -7,6 +7,7 @@ namespace System.IO.Tests { + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class RandomAccess_NoBuffering : FileSystemTest { private const FileOptions NoBuffering = (FileOptions)0x20000000; diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs index 18c0d0e840dbc..2afa02cdc4dd8 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs @@ -9,6 +9,7 @@ namespace System.IO.Tests { + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class RandomAccess_ReadAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs index fe5388090d960..df9755bae5704 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -9,6 +9,7 @@ namespace System.IO.Tests { + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class RandomAccess_ReadScatterAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs index 61983a54651ba..4d5bbd169b785 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs @@ -8,6 +8,7 @@ namespace System.IO.Tests { + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class RandomAccess_WriteAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index 926b53fe24fdc..bdea17233a69a 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -8,6 +8,7 @@ namespace System.IO.Tests { + [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] public class RandomAccess_WriteGatherAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) From 5ae1dfbb2586839253002f28aec74ef82c4eaba8 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 8 Jun 2021 15:31:13 +0200 Subject: [PATCH 62/71] async file IO is not supported on browser --- src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs | 1 + .../tests/RandomAccess/NoBuffering.Windows.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs | 1 + .../System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs | 1 + 6 files changed, 6 insertions(+) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index 98ae9782ce3ef..9ad9e8029dc97 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -70,6 +70,7 @@ public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() [Fact] [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] public void ThrowsArgumentExceptionForAsyncFileHandle() { if (ShouldThrowForAsyncHandle) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs index 09b05e6fc0b22..31628018de5d1 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs @@ -8,6 +8,7 @@ namespace System.IO.Tests { [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] public class RandomAccess_NoBuffering : FileSystemTest { private const FileOptions NoBuffering = (FileOptions)0x20000000; diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs index 2afa02cdc4dd8..6480cf6d4e556 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs @@ -10,6 +10,7 @@ namespace System.IO.Tests { [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] public class RandomAccess_ReadAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs index df9755bae5704..e1d93575d5c2f 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -10,6 +10,7 @@ namespace System.IO.Tests { [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] public class RandomAccess_ReadScatterAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs index 4d5bbd169b785..ee1b58e1f2dbb 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs @@ -9,6 +9,7 @@ namespace System.IO.Tests { [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] public class RandomAccess_WriteAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index bdea17233a69a..b2e53ef0b81a6 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -9,6 +9,7 @@ namespace System.IO.Tests { [ActiveIssue("https://github.com/dotnet/runtime/issues/34582", TestPlatforms.Windows, TargetFrameworkMonikers.Netcoreapp, TestRuntimes.Mono)] + [SkipOnPlatform(TestPlatforms.Browser, "async file IO is not supported on browser")] public class RandomAccess_WriteGatherAsync : RandomAccess_Base> { protected override ValueTask MethodUnderTest(SafeFileHandle handle, byte[] bytes, long fileOffset) From d4e5b09d70303d48545804c638b2213d442f6cc8 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 9 Jun 2021 16:46:03 +0200 Subject: [PATCH 63/71] fix WASM build --- src/mono/mono/mini/wasm_m2n_invoke.g.h | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/mono/mono/mini/wasm_m2n_invoke.g.h b/src/mono/mono/mini/wasm_m2n_invoke.g.h index 715070c03fd2e..7f363878fd910 100644 --- a/src/mono/mono/mini/wasm_m2n_invoke.g.h +++ b/src/mono/mono/mini/wasm_m2n_invoke.g.h @@ -1625,6 +1625,26 @@ wasm_invoke_vlii (void *target_func, InterpMethodArguments *margs) } +static void +wasm_invoke_iiiil (void *target_func, InterpMethodArguments *margs) +{ + typedef int (*T)(int arg_0, int arg_1, int arg_2, gint64 arg_3); + T func = (T)target_func; + int res = func ((int)(gssize)margs->iargs [0], (int)(gssize)margs->iargs [1], (int)(gssize)margs->iargs [2], get_long_arg (margs, 3)); + *(int*)margs->retval = res; + +} + +static void +wasm_invoke_liiil (void *target_func, InterpMethodArguments *margs) +{ + typedef gint64 (*T)(int arg_0, int arg_1, int arg_2, gint64 arg_3); + T func = (T)target_func; + gint64 res = func ((int)(gssize)margs->iargs [0], (int)(gssize)margs->iargs [1], (int)(gssize)margs->iargs [2], get_long_arg (margs, 3)); + *(gint64*)margs->retval = res; + +} + static const char* interp_to_native_signatures [] = { "DD", "DDD", @@ -1718,6 +1738,7 @@ static const char* interp_to_native_signatures [] = { "IIIIIIIIIIII", "IIIIIIIIIIIII", "IIIIIIIIIIIIII", +"IIIIL", "IIIL", "IIILIIII", "IIILLI", @@ -1732,6 +1753,7 @@ static const char* interp_to_native_signatures [] = { "L", "LI", "LII", +"LIIIL", "LIL", "LILI", "LILII", @@ -1888,6 +1910,7 @@ wasm_invoke_iiiiiiiiiii, wasm_invoke_iiiiiiiiiiii, wasm_invoke_iiiiiiiiiiiii, wasm_invoke_iiiiiiiiiiiiii, +wasm_invoke_iiiil, wasm_invoke_iiil, wasm_invoke_iiiliiii, wasm_invoke_iiilli, @@ -1902,6 +1925,7 @@ wasm_invoke_ili, wasm_invoke_l, wasm_invoke_li, wasm_invoke_lii, +wasm_invoke_liiil, wasm_invoke_lil, wasm_invoke_lili, wasm_invoke_lilii, From abc4b14a91f372bd0d2a0254df571d8873513158 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 9 Jun 2021 16:46:44 +0200 Subject: [PATCH 64/71] don't use preadv and pwritev on WASM as WASM implementation has bugs --- src/libraries/Native/Unix/System.Native/pal_io.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/libraries/Native/Unix/System.Native/pal_io.c b/src/libraries/Native/Unix/System.Native/pal_io.c index 7027d77e597ec..d7eb6c4ab23ac 100644 --- a/src/libraries/Native/Unix/System.Native/pal_io.c +++ b/src/libraries/Native/Unix/System.Native/pal_io.c @@ -1487,7 +1487,7 @@ int64_t SystemNative_PReadV(intptr_t fd, IOVector* vectors, int32_t vectorCount, int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); -#if HAVE_PREADV +#if HAVE_PREADV && !defined(TARGET_WASM) // preadv is buggy on WASM while ((count = preadv(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; @@ -1527,7 +1527,7 @@ int64_t SystemNative_PWriteV(intptr_t fd, IOVector* vectors, int32_t vectorCount int64_t count = 0; int fileDescriptor = ToFileDescriptor(fd); -#if HAVE_PWRITEV +#if HAVE_PWRITEV && !defined(TARGET_WASM) // pwritev is buggy on WASM while ((count = pwritev(fileDescriptor, (struct iovec*)vectors, (int)vectorCount, (off_t)fileOffset)) < 0 && errno == EINTR); #else int64_t current; From fc76167a9d3285a89327056b6ceaf7a140eaa106 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Wed, 9 Jun 2021 18:16:59 +0200 Subject: [PATCH 65/71] fix a typo --- .../System.Private.CoreLib/src/System/IO/FileStream.cs | 2 +- .../src/System/IO/Strategies/FileStreamHelpers.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs index f2b86227c6269..9653bd057bac3 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/FileStream.cs @@ -197,7 +197,7 @@ public FileStream(string path, FileStreamOptions options) } } - FileStreamHelpers.SerializaitonGuard(options.Access); + FileStreamHelpers.SerializationGuard(options.Access); _strategy = FileStreamHelpers.ChooseStrategy( this, path, options.Mode, options.Access, options.Share, options.BufferSize, options.Options, options.PreallocationSize); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs index a2a51bf9bb0ba..3897de9fa3692 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/FileStreamHelpers.cs @@ -108,10 +108,10 @@ internal static void ValidateArguments(string path, FileMode mode, FileAccess ac throw new ArgumentException(SR.Argument_InvalidAppendMode, nameof(access)); } - SerializaitonGuard(access); + SerializationGuard(access); } - internal static void SerializaitonGuard(FileAccess access) + internal static void SerializationGuard(FileAccess access) { if ((access & FileAccess.Write) == FileAccess.Write) { From 1014688405612260c6d1e732e308e89d5845ce88 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Thu, 10 Jun 2021 13:51:35 +0200 Subject: [PATCH 66/71] add missing CreateFileW default flags --- .../src/Interop/Windows/NtDll/Interop.NtCreateFile.cs | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs index 1841ba558eb65..12eb824cb2f33 100644 --- a/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs +++ b/src/libraries/Common/src/Interop/Windows/NtDll/Interop.NtCreateFile.cs @@ -116,7 +116,7 @@ private static CreateDisposition GetCreateDisposition(FileMode mode) private static DesiredAccess GetDesiredAccess(FileAccess access, FileMode fileMode, FileOptions options) { - DesiredAccess result = 0; + DesiredAccess result = DesiredAccess.FILE_READ_ATTRIBUTES | DesiredAccess.SYNCHRONIZE; // taken from CreateFileW implementation if ((access & FileAccess.Read) != 0) { @@ -130,10 +130,6 @@ private static DesiredAccess GetDesiredAccess(FileAccess access, FileMode fileMo { result |= DesiredAccess.FILE_APPEND_DATA; } - if ((options & FileOptions.Asynchronous) == 0) - { - result |= DesiredAccess.SYNCHRONIZE; // required by FILE_SYNCHRONOUS_IO_NONALERT - } if ((options & FileOptions.DeleteOnClose) != 0) { result |= DesiredAccess.DELETE; // required by FILE_DELETE_ON_CLOSE @@ -186,7 +182,7 @@ private static CreateOptions GetCreateOptions(FileOptions options) } private static ObjectAttributes GetObjectAttributes(FileShare share) - => (share & FileShare.Inheritable) != 0 ? ObjectAttributes.OBJ_INHERIT : 0; + => ((share & FileShare.Inheritable) != 0 ? ObjectAttributes.OBJ_INHERIT : 0) | ObjectAttributes.OBJ_CASE_INSENSITIVE; /// /// File creation disposition when calling directly to NT APIs. From 05327b18be5b832089866d2326908192ae433c60 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 11 Jun 2021 15:17:30 +0200 Subject: [PATCH 67/71] reduce code duplication --- .../src/System/IO/RandomAccess.Windows.cs | 50 ++++--- .../AsyncWindowsFileStreamStrategy.cs | 125 ++++-------------- 2 files changed, 58 insertions(+), 117 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 5f742eddc6c71..aa46cd79817b1 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -146,7 +146,16 @@ internal static unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) + private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) + => Map(QueueAsyncReadFile(handle, buffer, fileOffset, cancellationToken)); + + private static ValueTask Map((SafeFileHandle.ValueTaskSource? vts, int errorCode) tuple) + => tuple.vts != null + ? new ValueTask(tuple.vts, tuple.vts.Version) + : tuple.errorCode == 0 ? ValueTask.FromResult(0) : ValueTask.FromException(Win32Marshal.GetExceptionForWin32Error(tuple.errorCode)); + + internal static unsafe (SafeFileHandle.ValueTaskSource? vts, int errorCode) QueueAsyncReadFile( + SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) { SafeFileHandle.ValueTaskSource vts = handle.GetValueTaskSource(); try @@ -174,12 +183,12 @@ private static unsafe ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Me // to do so looks like we are freeing a pending overlapped later). nativeOverlapped->InternalLow = IntPtr.Zero; vts.Dispose(); - return ValueTask.FromResult(0); + return (null, 0); default: // Error. Callback will not be called. vts.Dispose(); - return ValueTask.FromException(Win32Marshal.GetExceptionForWin32Error(errorCode)); + return (null, errorCode); } } } @@ -191,10 +200,14 @@ private static unsafe ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Me // Completion handled by callback. vts.FinishedScheduling(); - return new ValueTask(vts, vts.Version); + return (vts, -1); } - private static unsafe ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) + private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) + => Map(QueueAsyncWriteFile(handle, buffer, fileOffset, cancellationToken)); + + internal static unsafe (SafeFileHandle.ValueTaskSource? vts, int errorCode) QueueAsyncWriteFile( + SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) { SafeFileHandle.ValueTaskSource vts = handle.GetValueTaskSource(); try @@ -207,19 +220,20 @@ private static unsafe ValueTask WriteAtOffsetAsync(SafeFileHandle handle, R { // The operation failed, or it's pending. int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(handle); - if (errorCode == Interop.Errors.ERROR_IO_PENDING) - { - // Common case: IO was initiated, completion will be handled by callback. - // Register for cancellation now that the operation has been initiated. - vts.RegisterForCancellation(cancellationToken); - } - else + switch (errorCode) { - // Error. Callback will not be invoked. - vts.Dispose(); - return errorCode == Interop.Errors.ERROR_NO_DATA // EOF on a pipe. IO callback will not be called. - ? ValueTask.FromResult(0) - : ValueTask.FromException(SafeFileHandle.ValueTaskSource.GetIOError(errorCode, path: null)); + case Interop.Errors.ERROR_IO_PENDING: + // Common case: IO was initiated, completion will be handled by callback. + // Register for cancellation now that the operation has been initiated. + vts.RegisterForCancellation(cancellationToken); + break; + case Interop.Errors.ERROR_NO_DATA: // EOF on a pipe. IO callback will not be called. + vts.Dispose(); + return (null, 0); + default: + // Error. Callback will not be invoked. + vts.Dispose(); + return (null, errorCode); } } } @@ -231,7 +245,7 @@ private static unsafe ValueTask WriteAtOffsetAsync(SafeFileHandle handle, R // Completion handled by callback. vts.FinishedScheduling(); - return new ValueTask(vts, vts.Version); + return (vts, -1); } private static long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs index 96f078c414378..a194c4802d15c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/Strategies/AsyncWindowsFileStreamStrategy.cs @@ -43,68 +43,27 @@ private unsafe ValueTask ReadAsyncInternal(Memory destination, Cancel ThrowHelper.ThrowNotSupportedException_UnreadableStream(); } - SafeFileHandle.ValueTaskSource vts = _fileHandle.GetValueTaskSource(); - try + long positionBefore = _filePosition; + if (CanSeek) { - long positionBefore = _filePosition; - NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(destination, positionBefore); - Debug.Assert(vts._memoryHandle.Pointer != null); - - // Calculate position in the file we should be at after the read is done - if (CanSeek) + long len = Length; + if (positionBefore + destination.Length > len) { - long len = Length; - - if (positionBefore + destination.Length > len) - { - destination = positionBefore <= len ? - destination.Slice(0, (int)(len - positionBefore)) : - default; - } - - // When using overlapped IO, the OS is not supposed to - // touch the file pointer location at all. We will adjust it - // ourselves, but only in memory. This isn't threadsafe. - _filePosition += destination.Length; + destination = positionBefore <= len ? + destination.Slice(0, (int)(len - positionBefore)) : + default; } - // Queue an async ReadFile operation. - if (Interop.Kernel32.ReadFile(_fileHandle, (byte*)vts._memoryHandle.Pointer, destination.Length, IntPtr.Zero, nativeOverlapped) == 0) - { - // The operation failed, or it's pending. - int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(_fileHandle); - switch (errorCode) - { - case Interop.Errors.ERROR_IO_PENDING: - // Common case: IO was initiated, completion will be handled by callback. - // Register for cancellation now that the operation has been initiated. - vts.RegisterForCancellation(cancellationToken); - break; - - case Interop.Errors.ERROR_BROKEN_PIPE: - // EOF on a pipe. Callback will not be called. - // We clear the overlapped status bit for this special case (failure - // to do so looks like we are freeing a pending overlapped later). - nativeOverlapped->InternalLow = IntPtr.Zero; - vts.Dispose(); - return ValueTask.FromResult(0); - - default: - // Error. Callback will not be called. - vts.Dispose(); - return ValueTask.FromException(HandleIOError(positionBefore, errorCode)); - } - } - } - catch - { - vts.Dispose(); - throw; + // When using overlapped IO, the OS is not supposed to + // touch the file pointer location at all. We will adjust it + // ourselves, but only in memory. This isn't threadsafe. + _filePosition += destination.Length; } - // Completion handled by callback. - vts.FinishedScheduling(); - return new ValueTask(vts, vts.Version); + (SafeFileHandle.ValueTaskSource? vts, int errorCode) = RandomAccess.QueueAsyncReadFile(_fileHandle, destination, positionBefore, cancellationToken); + return vts != null + ? new ValueTask(vts, vts.Version) + : (errorCode == 0) ? ValueTask.FromResult(0) : ValueTask.FromException(HandleIOError(positionBefore, errorCode)); } public override void Write(byte[] buffer, int offset, int count) @@ -123,52 +82,20 @@ private unsafe ValueTask WriteAsyncInternal(ReadOnlyMemory source, Cancell ThrowHelper.ThrowNotSupportedException_UnwritableStream(); } - SafeFileHandle.ValueTaskSource vts = _fileHandle.GetValueTaskSource(); - try - { - long positionBefore = _filePosition; - NativeOverlapped* nativeOverlapped = vts.PrepareForOperation(source, positionBefore); - Debug.Assert(vts._memoryHandle.Pointer != null); - - if (CanSeek) - { - // When using overlapped IO, the OS is not supposed to - // touch the file pointer location at all. We will adjust it - // ourselves, but only in memory. This isn't threadsafe. - _filePosition += source.Length; - UpdateLengthOnChangePosition(); - } - - // Queue an async WriteFile operation. - if (Interop.Kernel32.WriteFile(_fileHandle, (byte*)vts._memoryHandle.Pointer, source.Length, IntPtr.Zero, nativeOverlapped) == 0) - { - // The operation failed, or it's pending. - int errorCode = FileStreamHelpers.GetLastWin32ErrorAndDisposeHandleIfInvalid(_fileHandle); - if (errorCode == Interop.Errors.ERROR_IO_PENDING) - { - // Common case: IO was initiated, completion will be handled by callback. - // Register for cancellation now that the operation has been initiated. - vts.RegisterForCancellation(cancellationToken); - } - else - { - // Error. Callback will not be invoked. - vts.Dispose(); - return errorCode == Interop.Errors.ERROR_NO_DATA ? // EOF on a pipe. IO callback will not be called. - ValueTask.CompletedTask : - ValueTask.FromException(HandleIOError(positionBefore, errorCode)); - } - } - } - catch + long positionBefore = _filePosition; + if (CanSeek) { - vts.Dispose(); - throw; + // When using overlapped IO, the OS is not supposed to + // touch the file pointer location at all. We will adjust it + // ourselves, but only in memory. This isn't threadsafe. + _filePosition += source.Length; + UpdateLengthOnChangePosition(); } - // Completion handled by callback. - vts.FinishedScheduling(); - return new ValueTask(vts, vts.Version); + (SafeFileHandle.ValueTaskSource? vts, int errorCode) = RandomAccess.QueueAsyncWriteFile(_fileHandle, source, positionBefore, cancellationToken); + return vts != null + ? new ValueTask(vts, vts.Version) + : (errorCode == 0) ? ValueTask.CompletedTask : ValueTask.FromException(HandleIOError(positionBefore, errorCode)); } private Exception HandleIOError(long positionBefore, int errorCode) From 05e65eb550e39cb1f06b6cdb6a7d1f1ca7c73a95 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 11 Jun 2021 20:39:17 +0200 Subject: [PATCH 68/71] Apply suggestions from code review Co-authored-by: Stephen Toub --- .../src/System/IO/File.netcoreapp.cs | 7 ++++--- .../src/System/IO/RandomAccess.Unix.cs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs index c3aa1072c79fc..ed41ab7e589e2 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs @@ -72,11 +72,12 @@ private static byte[] ReadAllBytesUnknownLength(FileStream fs) byte[] tmp = ArrayPool.Shared.Rent((int)newLength); buffer.CopyTo(tmp); - if (rentedArray != null) + byte[]? oldRentedArray = rentedArray; + buffer = rentedArray = tmp; + if (oldRentedArray != null) { - ArrayPool.Shared.Return(rentedArray); + ArrayPool.Shared.Return(oldRentedArray); } - buffer = rentedArray = tmp; } Debug.Assert(bytesRead < buffer.Length); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 347a610271f7b..4a327ec815c47 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -69,7 +69,7 @@ private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyL private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask(Task.Factory.StartNew(state => + return new ValueTask(Task.Factory.StartNew(static state => { var args = ((SafeFileHandle handle, Memory buffer, long fileOffset))state!; return ReadAtOffset(args.handle, args.buffer.Span, args.fileOffset); From c87354c999fa4400956ff2cb69e92e82ec41e931 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Fri, 11 Jun 2021 21:18:39 +0200 Subject: [PATCH 69/71] address code review feedback --- .../SectorAlignedMemory.Windows.cs | 5 -- .../SafeHandles/SafeFileHandle.Windows.cs | 2 +- .../src/System/IO/RandomAccess.Unix.cs | 38 +++++----- .../src/System/IO/RandomAccess.Windows.cs | 74 +++++++++---------- .../src/System/IO/RandomAccess.cs | 20 ++--- 5 files changed, 66 insertions(+), 73 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs index cede38d4b18ca..2d712be383bf0 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs @@ -22,11 +22,6 @@ private unsafe SectorAlignedMemory(void* memory, int length) this.length = length; } - ~SectorAlignedMemory() - { - Dispose(false); - } - public static unsafe SectorAlignedMemory Allocate(int length) { void* memory = VirtualAlloc( diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index b5d310a77f533..9ed60c374208e 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -66,7 +66,7 @@ private static IntPtr NtCreateFile(string fullPath, FileMode mode, FileAccess ac } else { - var vsb = new ValueStringBuilder(stackalloc char[1024]); + var vsb = new ValueStringBuilder(stackalloc char[256]); vsb.Append(MandatoryNtPrefix); if (fullPath.StartsWith(@"\\?\", StringComparison.Ordinal)) // NtCreateFile does not support "\\?\" prefix, only "\??\" diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 4a327ec815c47..217e12c085550 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -39,17 +39,17 @@ private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyL MemoryHandle[] handles = new MemoryHandle[buffers.Count]; Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count]; - for (int i = 0; i < buffers.Count; i++) - { - Memory buffer = buffers[i]; - MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; - handles[i] = memoryHandle; - } - long result; try { + for (int i = 0; i < buffers.Count; i++) + { + Memory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; + handles[i] = memoryHandle; + } + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) { result = Interop.Sys.PReadV(handle, pinnedVectors, buffers.Count, fileOffset); @@ -79,7 +79,7 @@ private static ValueTask ReadAtOffsetAsync(SafeFileHandle handle, Memory ReadScatterAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask(Task.Factory.StartNew(state => + return new ValueTask(Task.Factory.StartNew(static state => { var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state!; return ReadScatterAtOffset(args.handle, args.buffers, args.fileOffset); @@ -101,17 +101,17 @@ private static unsafe long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyL MemoryHandle[] handles = new MemoryHandle[buffers.Count]; Span vectors = buffers.Count <= IovStackThreshold ? stackalloc Interop.Sys.IOVector[IovStackThreshold] : new Interop.Sys.IOVector[buffers.Count ]; - for (int i = 0; i < buffers.Count; i++) - { - ReadOnlyMemory buffer = buffers[i]; - MemoryHandle memoryHandle = buffer.Pin(); - vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; - handles[i] = memoryHandle; - } - long result; try { + for (int i = 0; i < buffers.Count; i++) + { + ReadOnlyMemory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + vectors[i] = new Interop.Sys.IOVector { Base = (byte*)memoryHandle.Pointer, Count = (UIntPtr)buffer.Length }; + handles[i] = memoryHandle; + } + fixed (Interop.Sys.IOVector* pinnedVectors = &MemoryMarshal.GetReference(vectors)) { result = Interop.Sys.PWriteV(handle, pinnedVectors, buffers.Count, fileOffset); @@ -131,7 +131,7 @@ private static unsafe long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyL private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask(Task.Factory.StartNew(state => + return new ValueTask(Task.Factory.StartNew(static state => { var args = ((SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset))state!; return WriteAtOffset(args.handle, args.buffer.Span, args.fileOffset); @@ -141,7 +141,7 @@ private static ValueTask WriteAtOffsetAsync(SafeFileHandle handle, ReadOnly private static ValueTask WriteGatherAtOffsetAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken) { - return new ValueTask(Task.Factory.StartNew(state => + return new ValueTask(Task.Factory.StartNew(static state => { var args = ((SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset))state!; return WriteGatherAtOffset(args.handle, args.buffers, args.fileOffset); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index aa46cd79817b1..22fbef5ab06ad 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -326,10 +326,27 @@ private static async ValueTask ReadScatterAtOffsetSingleSyscallAsync(SafeF return await ReadAtOffsetAsync(handle, buffers[0], fileOffset, cancellationToken).ConfigureAwait(false); } - (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) = PrepareMemorySegments(buffers); + // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " + long[] fileSegments = new long[buffers.Count + 1]; + fileSegments[buffers.Count] = 0; + + MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; + MemoryHandle pinnedSegments = fileSegments.AsMemory().Pin(); try { + for (int i = 0; i < buffers.Count; i++) + { + Memory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + memoryHandles[i] = memoryHandle; + + unsafe // async method can't be unsafe + { + fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); + } + } + return await ReadFileScatterAsync(handle, pinnedSegments, totalBytes, fileOffset, cancellationToken).ConfigureAwait(false); } finally @@ -458,10 +475,27 @@ private static async ValueTask WriteGatherAtOffsetSingleSyscallAsync(SafeF return await WriteAtOffsetAsync(handle, buffers[0], fileOffset, cancellationToken).ConfigureAwait(false); } - (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) = PrepareMemorySegments(buffers); + // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " + long[] fileSegments = new long[buffers.Count + 1]; + fileSegments[buffers.Count] = 0; + + MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; + MemoryHandle pinnedSegments = fileSegments.AsMemory().Pin(); try { + for (int i = 0; i < buffers.Count; i++) + { + ReadOnlyMemory buffer = buffers[i]; + MemoryHandle memoryHandle = buffer.Pin(); + memoryHandles[i] = memoryHandle; + + unsafe // async method can't be unsafe + { + fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); + } + } + return await WriteFileGatherAsync(handle, pinnedSegments, totalBytes, fileOffset, cancellationToken).ConfigureAwait(false); } finally @@ -515,42 +549,6 @@ private static unsafe ValueTask WriteFileGatherAsync(SafeFileHandle handle, return new ValueTask(vts, vts.Version); } - private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) PrepareMemorySegments(IReadOnlyList> buffers) - { - // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " - long[] fileSegments = new long[buffers.Count + 1]; - fileSegments[buffers.Count] = 0; - - MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; - for (int i = 0; i < buffers.Count; i++) - { - Memory buffer = buffers[i]; - MemoryHandle memoryHandle = buffer.Pin(); - memoryHandles[i] = memoryHandle; - fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); - } - - return (fileSegments.AsMemory().Pin(), memoryHandles); - } - - private static unsafe (MemoryHandle pinnedSegments, MemoryHandle[] memoryHandles) PrepareMemorySegments(IReadOnlyList> buffers) - { - // "The array must contain enough elements to store nNumberOfBytesToWrite bytes of data, and one element for the terminating NULL. " - long[] fileSegments = new long[buffers.Count + 1]; - fileSegments[buffers.Count] = 0; - - MemoryHandle[] memoryHandles = new MemoryHandle[buffers.Count]; - for (int i = 0; i < buffers.Count; i++) - { - ReadOnlyMemory buffer = buffers[i]; - MemoryHandle memoryHandle = buffer.Pin(); - memoryHandles[i] = memoryHandle; - fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); - } - - return (fileSegments.AsMemory().Pin(), memoryHandles); - } - private static NativeOverlapped GetNativeOverlapped(long fileOffset) { NativeOverlapped nativeOverlapped = default; diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs index 943c17e8fb549..199d0d4ea91c5 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.cs @@ -92,13 +92,13 @@ public static long Read(SafeFileHandle handle, IReadOnlyList> buffe /// Position of the file is not advanced. public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffer, long fileOffset, CancellationToken cancellationToken = default) { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + if (cancellationToken.IsCancellationRequested) { return ValueTask.FromCanceled(cancellationToken); } - ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); - return ReadAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); } @@ -121,14 +121,14 @@ public static ValueTask ReadAsync(SafeFileHandle handle, Memory buffe /// Position of the file is not advanced. public static ValueTask ReadAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + ValidateBuffers(buffers); + if (cancellationToken.IsCancellationRequested) { return ValueTask.FromCanceled(cancellationToken); } - ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); - ValidateBuffers(buffers); - return ReadScatterAtOffsetAsync(handle, buffers, fileOffset, cancellationToken); } @@ -198,13 +198,13 @@ public static long Write(SafeFileHandle handle, IReadOnlyListPosition of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemory buffer, long fileOffset, CancellationToken cancellationToken = default) { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + if (cancellationToken.IsCancellationRequested) { return ValueTask.FromCanceled(cancellationToken); } - ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); - return WriteAtOffsetAsync(handle, buffer, fileOffset, cancellationToken); } @@ -227,14 +227,14 @@ public static ValueTask WriteAsync(SafeFileHandle handle, ReadOnlyMemoryPosition of the file is not advanced. public static ValueTask WriteAsync(SafeFileHandle handle, IReadOnlyList> buffers, long fileOffset, CancellationToken cancellationToken = default) { + ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); + ValidateBuffers(buffers); + if (cancellationToken.IsCancellationRequested) { return ValueTask.FromCanceled(cancellationToken); } - ValidateInput(handle, fileOffset, mustBeAsync: OperatingSystem.IsWindows()); - ValidateBuffers(buffers); - return WriteGatherAtOffsetAsync(handle, buffers, fileOffset, cancellationToken); } From f80bbb7df342a49a0c65b1fe1c4e30a770c08bf7 Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 15 Jun 2021 16:55:36 +0200 Subject: [PATCH 70/71] Apply suggestions from code review Co-authored-by: Stephen Toub --- .../tests/RandomAccess/Base.cs | 15 ++++-------- .../tests/RandomAccess/ReadScatter.cs | 3 +-- .../tests/RandomAccess/ReadScatterAsync.cs | 3 +-- .../SectorAlignedMemory.Windows.cs | 2 +- .../tests/RandomAccess/WriteGather.cs | 3 +-- .../tests/RandomAccess/WriteGatherAsync.cs | 3 +-- .../Win32/SafeHandles/SafeFileHandle.Unix.cs | 9 +++---- .../SafeHandles/SafeFileHandle.Windows.cs | 24 ++++++++++--------- .../src/System/IO/RandomAccess.Windows.cs | 16 +++++-------- .../SyncWindowsFileStreamStrategy.cs | 4 ++-- 10 files changed, 36 insertions(+), 46 deletions(-) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index 9ad9e8029dc97..a79db08625f16 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -21,8 +21,7 @@ public abstract class RandomAccess_Base : FileSystemTest [Fact] public void ThrowsArgumentNullExceptionForNullHandle() { - ArgumentNullException ex = Assert.Throws(() => MethodUnderTest(null, Array.Empty(), 0)); - Assert.Equal("handle", ex.ParamName); + AssertExtensions.Throws("handle", () => MethodUnderTest(null, Array.Empty(), 0)); } [Fact] @@ -30,8 +29,7 @@ public void ThrowsArgumentExceptionForInvalidHandle() { SafeFileHandle handle = new SafeFileHandle(new IntPtr(-1), ownsHandle: false); - ArgumentException ex = Assert.Throws(() => MethodUnderTest(handle, Array.Empty(), 0)); - Assert.Equal("handle", ex.ParamName); + AssertExtensions.Throws("handle", () => MethodUnderTest(handle, Array.Empty(), 0)); } [Fact] @@ -62,8 +60,7 @@ public void ThrowsArgumentOutOfRangeExceptionForNegativeFileOffset() FileOptions options = ShouldThrowForAsyncHandle ? FileOptions.None : FileOptions.Asynchronous; using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: options)) { - ArgumentOutOfRangeException ex = Assert.Throws(() => MethodUnderTest(handle, Array.Empty(), -1)); - Assert.Equal("fileOffset", ex.ParamName); + AssertExtensions.Throws("fileOffset", () => MethodUnderTest(handle, Array.Empty(), -1)); } } } @@ -77,8 +74,7 @@ public void ThrowsArgumentExceptionForAsyncFileHandle() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) { - ArgumentException ex = Assert.Throws(() => MethodUnderTest(handle, new byte[100], 0)); - Assert.Equal("handle", ex.ParamName); + AssertExtensions.Throws("handle", () => MethodUnderTest(handle, new byte[100], 0)); } } } @@ -90,8 +86,7 @@ public void ThrowsArgumentExceptionForSyncFileHandle() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.None)) { - ArgumentException ex = Assert.Throws(() => MethodUnderTest(handle, new byte[100], 0)); - Assert.Equal("handle", ex.ParamName); + AssertExtensions.Throws("handle", () => MethodUnderTest(handle, new byte[100], 0)); } } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs index a87af17357ee3..fb85ae3a60c7a 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs @@ -20,8 +20,7 @@ public void ThrowsArgumentNullExceptionForNullBuffers() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.Read(handle, buffers: null, 0)); - Assert.Equal("buffers", ex.ParamName); + AssertExtensions.Throws("buffers", () => RandomAccess.Read(handle, buffers: null, 0)); } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs index e1d93575d5c2f..419c7b3affccf 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -24,8 +24,7 @@ public void ThrowsArgumentNullExceptionForNullBuffers() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, options: FileOptions.Asynchronous)) { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.ReadAsync(handle, buffers: null, 0)); - Assert.Equal("buffers", ex.ParamName); + AssertExtensions.Throws("buffers", () => RandomAccess.ReadAsync(handle, buffers: null, 0)); } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs index 2d712be383bf0..78383748366ca 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs @@ -9,7 +9,7 @@ namespace System.IO.Tests { - internal class SectorAlignedMemory : MemoryManager + internal sealed class SectorAlignedMemory : MemoryManager { private bool disposed = false; private int refCount = 0; diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs index 911432f685571..565bd3b988d71 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs @@ -19,8 +19,7 @@ public void ThrowsArgumentNullExceptionForNullBuffers() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write)) { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.Write(handle, buffers: null, 0)); - Assert.Equal("buffers", ex.ParamName); + AssertExtensions.Throws("buffers", () => RandomAccess.Write(handle, buffers: null, 0)); } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index b2e53ef0b81a6..ce19dd110aa1d 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -23,8 +23,7 @@ public void ThrowsArgumentNullExceptionForNullBuffers() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous)) { - ArgumentNullException ex = Assert.Throws(() => RandomAccess.WriteAsync(handle, buffers: null, 0)); - Assert.Equal("buffers", ex.ParamName); + AssertExtensions.Throws("buffers", () => RandomAccess.WriteAsync(handle, buffers: null, 0)); } } diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs index f0b4ea17d8fb1..7a13a80610d04 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Unix.cs @@ -11,7 +11,7 @@ namespace Microsoft.Win32.SafeHandles public sealed class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { // not using bool? as it's not thread safe - private NullableBool _canSeek = NullableBool.Undefined; + private volatile NullableBool _canSeek = NullableBool.Undefined; public SafeFileHandle() : this(ownsHandle: true) { @@ -309,12 +309,13 @@ private bool GetCanSeek() Debug.Assert(!IsClosed); Debug.Assert(!IsInvalid); - if (_canSeek == NullableBool.Undefined) + NullableBool canSeek = _canSeek; + if (canSeek == NullableBool.Undefined) { - _canSeek = Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0 ? NullableBool.True : NullableBool.False; + _canSeek = canSeek = Interop.Sys.LSeek(this, 0, Interop.Sys.SeekWhence.SEEK_CUR) >= 0 ? NullableBool.True : NullableBool.False; } - return _canSeek == NullableBool.True; + return canSeek == NullableBool.True; } private enum NullableBool diff --git a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs index 9ed60c374208e..36710dcdbde5d 100644 --- a/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/Microsoft/Win32/SafeHandles/SafeFileHandle.Windows.cs @@ -12,8 +12,8 @@ namespace Microsoft.Win32.SafeHandles public sealed partial class SafeFileHandle : SafeHandleZeroOrMinusOneIsInvalid { internal const FileOptions NoBuffering = (FileOptions)0x20000000; - private FileOptions _fileOptions = (FileOptions)(-1); - private int _fileType = -1; + private volatile FileOptions _fileOptions = (FileOptions)(-1); + private volatile int _fileType = -1; public SafeFileHandle() : base(true) { @@ -130,9 +130,10 @@ internal void InitThreadPoolBindingIfNeeded() internal unsafe FileOptions GetFileOptions() { - if (_fileOptions != (FileOptions)(-1)) + FileOptions fileOptions = _fileOptions; + if (fileOptions != (FileOptions)(-1)) { - return _fileOptions; + return fileOptions; } Interop.NtDll.CreateOptions options; @@ -181,17 +182,18 @@ internal unsafe FileOptions GetFileOptions() internal int GetFileType() { - if (_fileType == -1) + int fileType = _fileType; + if (fileType == -1) { - _fileType = Interop.Kernel32.GetFileType(this); + _fileType = fileType = Interop.Kernel32.GetFileType(this); - Debug.Assert(_fileType == Interop.Kernel32.FileTypes.FILE_TYPE_DISK - || _fileType == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE - || _fileType == Interop.Kernel32.FileTypes.FILE_TYPE_CHAR, - "Unknown file type!"); + Debug.Assert(fileType == Interop.Kernel32.FileTypes.FILE_TYPE_DISK + || fileType == Interop.Kernel32.FileTypes.FILE_TYPE_PIPE + || fileType == Interop.Kernel32.FileTypes.FILE_TYPE_CHAR, + $"Unknown file type: {fileType}"); } - return _fileType; + return fileType; } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 22fbef5ab06ad..2ab60472ebd52 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -90,11 +90,9 @@ internal static unsafe int ReadFileNative(SafeFileHandle handle, Span byte fixed (byte* p = &MemoryMarshal.GetReference(bytes)) { - r = overlapped != null ? - (syncUsingOverlapped - ? Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, overlapped) - : Interop.Kernel32.ReadFile(handle, p, bytes.Length, IntPtr.Zero, overlapped)) - : Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, IntPtr.Zero); + r = overlapped == null || syncUsingOverlapped ? + Interop.Kernel32.ReadFile(handle, p, bytes.Length, out numBytesRead, overlapped) : + Interop.Kernel32.ReadFile(handle, p, bytes.Length, IntPtr.Zero, overlapped); } if (r == 0) @@ -127,11 +125,9 @@ internal static unsafe int WriteFileNative(SafeFileHandle handle, ReadOnlySpan destination) Debug.Assert(!_fileHandle.IsClosed, "!_handle.IsClosed"); int r = RandomAccess.ReadAtOffset(_fileHandle, destination, _filePosition, _path); - Debug.Assert(r >= 0, "RandomAccess.ReadAtOffsete is likely broken."); + Debug.Assert(r >= 0, $"RandomAccess.ReadAtOffset returned {r}."); _filePosition += r; return r; @@ -107,7 +107,7 @@ private unsafe void WriteSpan(ReadOnlySpan source) Debug.Assert(!_fileHandle.IsClosed, "!_handle.IsClosed"); int r = RandomAccess.WriteAtOffset(_fileHandle, source, _filePosition, _path); - Debug.Assert(r >= 0, "RandomAccess.WriteAtOffset is likely broken."); + Debug.Assert(r >= 0, $"RandomAccess.WriteAtOffset returned {r}."); _filePosition += r; UpdateLengthOnChangePosition(); From 07051fabeeae10e5216dbddd2d152f902ebd105d Mon Sep 17 00:00:00 2001 From: Adam Sitnik Date: Tue, 15 Jun 2021 18:09:19 +0200 Subject: [PATCH 71/71] address code review feedback --- .../Unix/System.Native/Interop.PRead.cs | 2 +- .../Kernel32/Interop.FileScatterGather.cs | 20 +++---- .../ref/System.IO.FileSystem.Forwards.cs | 1 - .../tests/File/OpenHandle.cs | 21 +++---- .../tests/RandomAccess/Base.cs | 6 +- .../tests/RandomAccess/NoBuffering.Windows.cs | 29 ++++----- .../tests/RandomAccess/Read.cs | 27 ++++++--- .../tests/RandomAccess/ReadAsync.cs | 37 ++++++++---- .../tests/RandomAccess/ReadScatter.cs | 41 +++++++++++-- .../tests/RandomAccess/ReadScatterAsync.cs | 50 +++++++++++++--- .../SectorAlignedMemory.Windows.cs | 42 +++++++------ .../tests/RandomAccess/Write.cs | 27 ++++++--- .../tests/RandomAccess/WriteAsync.cs | 37 ++++++++---- .../tests/RandomAccess/WriteGather.cs | 50 ++++++++++++++-- .../tests/RandomAccess/WriteGatherAsync.cs | 59 ++++++++++++++++--- .../src/System/IO/File.netcoreapp.cs | 16 ++--- .../src/System/IO/RandomAccess.Unix.cs | 6 +- .../src/System/IO/RandomAccess.Windows.cs | 4 +- 18 files changed, 334 insertions(+), 141 deletions(-) diff --git a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs index 7b95ded24a54c..664da015febb2 100644 --- a/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs +++ b/src/libraries/Common/src/Interop/Unix/System.Native/Interop.PRead.cs @@ -8,6 +8,6 @@ internal static partial class Interop internal static partial class Sys { [DllImport(Libraries.SystemNative, EntryPoint = "SystemNative_PRead", SetLastError = true)] - internal static extern unsafe int PRead(SafeHandle fd, byte* buffer, int count, long fileOffset); + internal static extern unsafe int PRead(SafeHandle fd, byte* buffer, int bufferSize, long fileOffset); } } diff --git a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs index f573232523d08..8e61970c3c242 100644 --- a/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs +++ b/src/libraries/Common/src/Interop/Windows/Kernel32/Interop.FileScatterGather.cs @@ -11,18 +11,18 @@ internal static partial class Kernel32 { [DllImport(Libraries.Kernel32, SetLastError = true)] internal static extern unsafe int ReadFileScatter( - SafeHandle handle, - long* segments, - int numBytesToRead, - IntPtr reserved_mustBeZero, - NativeOverlapped* overlapped); + SafeHandle hFile, + long* aSegmentArray, + int nNumberOfBytesToRead, + IntPtr lpReserved, + NativeOverlapped* lpOverlapped); [DllImport(Libraries.Kernel32, SetLastError = true)] internal static extern unsafe int WriteFileGather( - SafeHandle handle, - long* segments, - int numBytesToWrite, - IntPtr reserved_mustBeZero, - NativeOverlapped* overlapped); + SafeHandle hFile, + long* aSegmentArray, + int nNumberOfBytesToWrite, + IntPtr lpReserved, + NativeOverlapped* lpOverlapped); } } diff --git a/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs b/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs index 3f6524af8335b..f2ea51b0ddb0d 100644 --- a/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs +++ b/src/libraries/System.IO.FileSystem/ref/System.IO.FileSystem.Forwards.cs @@ -16,7 +16,6 @@ [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.MatchCasing))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.MatchType))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.SearchOption))] -[assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.RandomAccess))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.Enumeration.FileSystemEntry))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.Enumeration.FileSystemEnumerable<>))] [assembly: System.Runtime.CompilerServices.TypeForwardedTo(typeof(System.IO.Enumeration.FileSystemEnumerator<>))] diff --git a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs index 845ba871bfcee..a283b01a0b7e9 100644 --- a/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs +++ b/src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs @@ -30,10 +30,10 @@ public override void NegativePreallocationSizeThrows() () => File.OpenHandle("validPath", FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, preallocationSize: -1)); } + [ActiveIssue("https://github.com/dotnet/runtime/issues/53432")] [Theory, MemberData(nameof(StreamSpecifiers))] public override void FileModeAppendExisting(string streamSpecifier) { - // currently not enabled due to https://github.com/dotnet/runtime/issues/53432 _ = streamSpecifier; // to keep the xUnit analyser happy } @@ -56,20 +56,15 @@ public void SafeFileHandle_IsAsync_ReturnsCorrectInformation(FileOptions options } } + // Unix doesn't directly support DeleteOnClose + // For FileStream created out of path, we mimic it by closing the handle first + // and then unlinking the path + // Since SafeFileHandle does not always have the path and we can't find path for given file descriptor on Unix + // this test runs only on Windows + [PlatformSpecific(TestPlatforms.Windows)] [Theory] [InlineData(FileOptions.DeleteOnClose)] [InlineData(FileOptions.DeleteOnClose | FileOptions.Asynchronous)] - public override void DeleteOnClose_FileDeletedAfterClose(FileOptions options) - { - // Unix doesn't directly support DeleteOnClose - // For FileStream created out of path, we mimic it by closing the handle first - // and then unlinking the path - // Since SafeFileHandle does not always have the path and we can't find path for given file descriptor on Unix - // this test runs only on Windows - if (OperatingSystem.IsWindows()) // async file handles are a Windows concept - { - base.DeleteOnClose_FileDeletedAfterClose(options); - } - } + public override void DeleteOnClose_FileDeletedAfterClose(FileOptions options) => base.DeleteOnClose_FileDeletedAfterClose(options); } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs index a79db08625f16..8d876f30174cd 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Base.cs @@ -91,13 +91,11 @@ public void ThrowsArgumentExceptionForSyncFileHandle() } } - protected static CancellationToken GetCancelledToken() + protected static CancellationTokenSource GetCancelledTokenSource() { CancellationTokenSource source = new CancellationTokenSource(); - CancellationToken token = source.Token; source.Cancel(); - - return token; + return source; } protected SafeFileHandle GetHandleToExistingFile(FileAccess access) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs index 31628018de5d1..ea9982600945e 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/NoBuffering.Windows.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Security.Cryptography; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; using Xunit; @@ -18,8 +19,7 @@ public async Task ReadAsyncUsingSingleBuffer() { const int fileSize = 1_000_000; // 1 MB string filePath = GetTestFilePath(); - byte[] expected = new byte[fileSize]; - new Random().NextBytes(expected); + byte[] expected = RandomNumberGenerator.GetBytes(fileSize); File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous | NoBuffering)) @@ -28,14 +28,6 @@ public async Task ReadAsyncUsingSingleBuffer() int current = 0; int total = 0; - do - { - current = await RandomAccess.ReadAsync(handle, buffer.Memory, fileOffset: total); - - Assert.True(expected.AsSpan(total, current).SequenceEqual(buffer.GetSpan().Slice(0, current))); - - total += current; - } // From https://docs.microsoft.com/en-us/windows/win32/fileio/file-buffering: // "File access sizes, including the optional file offset in the OVERLAPPED structure, // if specified, must be for a number of bytes that is an integer multiple of the volume sector size." @@ -45,6 +37,14 @@ public async Task ReadAsyncUsingSingleBuffer() // the read from offset=4097 THROWS (Invalid argument, offset is not a multiple of sector size!) // That is why we stop at the first incomplete read (the next one would throw). // It's possible to get 0 if we are lucky and file size is a multiple of physical sector size. + do + { + current = await RandomAccess.ReadAsync(handle, buffer.Memory, fileOffset: total); + + Assert.True(expected.AsSpan(total, current).SequenceEqual(buffer.GetSpan().Slice(0, current))); + + total += current; + } while (current == buffer.Memory.Length); Assert.Equal(fileSize, total); @@ -56,8 +56,7 @@ public async Task ReadAsyncUsingMultipleBuffers() { const int fileSize = 1_000_000; // 1 MB string filePath = GetTestFilePath(); - byte[] expected = new byte[fileSize]; - new Random().NextBytes(expected); + byte[] expected = RandomNumberGenerator.GetBytes(fileSize); File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous | NoBuffering)) @@ -96,8 +95,7 @@ public async Task WriteAsyncUsingSingleBuffer() string filePath = GetTestFilePath(); int bufferSize = Environment.SystemPageSize; int fileSize = bufferSize * 10; - byte[] content = new byte[fileSize]; - new Random().NextBytes(content); + byte[] content = RandomNumberGenerator.GetBytes(fileSize); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous | NoBuffering)) using (SectorAlignedMemory buffer = SectorAlignedMemory.Allocate(bufferSize)) @@ -125,8 +123,7 @@ public async Task WriteAsyncUsingMultipleBuffers() string filePath = GetTestFilePath(); int bufferSize = Environment.SystemPageSize; int fileSize = bufferSize * 10; - byte[] content = new byte[fileSize]; - new Random().NextBytes(content); + byte[] content = RandomNumberGenerator.GetBytes(fileSize); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous | NoBuffering)) using (SectorAlignedMemory buffer_1 = SectorAlignedMemory.Allocate(bufferSize)) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs index 00e8953645038..81021d04aee16 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Read.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Linq; +using System.Security.Cryptography; using Microsoft.Win32.SafeHandles; using Xunit; @@ -25,12 +26,23 @@ public void ThrowsOnWriteAccess() } [Fact] - public void HappyPath() + public void ReadToAnEmptyBufferReturnsZero() + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[1]); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) + { + Assert.Equal(0, RandomAccess.Read(handle, Array.Empty(), fileOffset: 0)); + } + } + + [Fact] + public void ReadsBytesFromGivenFileAtGivenOffset() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] expected = new byte[fileSize]; - new Random().NextBytes(expected); + byte[] expected = RandomNumberGenerator.GetBytes(fileSize); File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) @@ -41,10 +53,11 @@ public void HappyPath() do { - current = RandomAccess.Read( - handle, - actual.AsSpan(total, Math.Min(actual.Length - total, fileSize / 4)), - fileOffset: total); + Span buffer = actual.AsSpan(total, Math.Min(actual.Length - total, fileSize / 4)); + + current = RandomAccess.Read(handle, buffer, fileOffset: total); + + Assert.InRange(current, 0, buffer.Length); total += current; } while (current != 0); diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs index 6480cf6d4e556..c18da9ecd5059 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadAsync.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Linq; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -20,13 +21,17 @@ protected override bool ShouldThrowForSyncHandle => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle [Fact] - public Task TaskAlreadyCanceledAsync() + public async Task TaskAlreadyCanceledAsync() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) { - CancellationToken token = GetCancelledToken(); + CancellationTokenSource cts = GetCancelledTokenSource(); + CancellationToken token = cts.Token; + Assert.True(RandomAccess.ReadAsync(handle, new byte[1], 0, token).IsCanceled); - return Assert.ThrowsAsync(async () => await RandomAccess.ReadAsync(handle, new byte[1], 0, token)); + + TaskCanceledException ex = await Assert.ThrowsAsync(() => RandomAccess.ReadAsync(handle, new byte[1], 0, token).AsTask()); + Assert.Equal(token, ex.CancellationToken); } } @@ -40,12 +45,23 @@ public async Task ThrowsOnWriteAccess() } [Fact] - public async Task HappyPath() + public async Task ReadToAnEmptyBufferReturnsZeroAsync() + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[1]); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) + { + Assert.Equal(0, await RandomAccess.ReadAsync(handle, Array.Empty(), fileOffset: 0)); + } + } + + [Fact] + public async Task ReadsBytesFromGivenFileAtGivenOffsetAsync() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] expected = new byte[fileSize]; - new Random().NextBytes(expected); + byte[] expected = RandomNumberGenerator.GetBytes(fileSize); File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) @@ -56,10 +72,11 @@ public async Task HappyPath() do { - current = await RandomAccess.ReadAsync( - handle, - actual.AsMemory(total, Math.Min(actual.Length - total, fileSize / 4)), - fileOffset: total); + Memory buffer = actual.AsMemory(total, Math.Min(actual.Length - total, fileSize / 4)); + + current = await RandomAccess.ReadAsync(handle, buffer, fileOffset: total); + + Assert.InRange(current, 0, buffer.Length); total += current; } while (current != 0); diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs index fb85ae3a60c7a..68058b6242fc7 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatter.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Linq; +using System.Security.Cryptography; using Microsoft.Win32.SafeHandles; using Xunit; @@ -34,12 +35,23 @@ public void ThrowsOnWriteAccess() } [Fact] - public void HappyPath() + public void ReadToAnEmptyBufferReturnsZero() + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[1]); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) + { + Assert.Equal(0, RandomAccess.Read(handle, new Memory[] { Array.Empty() }, fileOffset: 0)); + } + } + + [Fact] + public void ReadsBytesFromGivenFileAtGivenOffset() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] expected = new byte[fileSize]; - new Random().NextBytes(expected); + byte[] expected = RandomNumberGenerator.GetBytes(fileSize); File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) @@ -51,16 +63,21 @@ public void HappyPath() do { int firstBufferLength = (int)Math.Min(actual.Length - total, fileSize / 4); + Memory buffer_1 = actual.AsMemory((int)total, firstBufferLength); + Memory buffer_2 = actual.AsMemory((int)total + firstBufferLength); current = RandomAccess.Read( handle, new Memory[] { - actual.AsMemory((int)total, firstBufferLength), - actual.AsMemory((int)total + firstBufferLength) + buffer_1, + Array.Empty(), + buffer_2 }, fileOffset: total); + Assert.InRange(current, 0, buffer_1.Length + buffer_2.Length); + total += current; } while (current != 0); @@ -68,5 +85,19 @@ public void HappyPath() Assert.Equal(expected, actual.Take((int)total).ToArray()); } } + + [Fact] + public void ReadToTheSameBufferOverwritesContent() + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[3] { 1, 2, 3 }); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open)) + { + byte[] buffer = new byte[1]; + Assert.Equal(buffer.Length + buffer.Length, RandomAccess.Read(handle, Enumerable.Repeat(buffer.AsMemory(), 2).ToList(), fileOffset: 0)); + Assert.Equal(2, buffer[0]); + } + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs index 419c7b3affccf..68d631a6dc390 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/ReadScatterAsync.cs @@ -2,6 +2,7 @@ // The .NET Foundation licenses this file to you under the MIT license. using System.Linq; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -29,13 +30,17 @@ public void ThrowsArgumentNullExceptionForNullBuffers() } [Fact] - public Task TaskAlreadyCanceledAsync() + public async Task TaskAlreadyCanceledAsync() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) { - CancellationToken token = GetCancelledToken(); + CancellationTokenSource cts = GetCancelledTokenSource(); + CancellationToken token = cts.Token; + Assert.True(RandomAccess.ReadAsync(handle, new Memory[] { new byte[1] }, 0, token).IsCanceled); - return Assert.ThrowsAsync(async () => await RandomAccess.ReadAsync(handle, new Memory[] { new byte[1] }, 0, token)); + + TaskCanceledException ex = await Assert.ThrowsAsync(() => RandomAccess.ReadAsync(handle, new Memory[] { new byte[1] }, 0, token).AsTask()); + Assert.Equal(token, ex.CancellationToken); } } @@ -49,12 +54,23 @@ public async Task ThrowsOnWriteAccess() } [Fact] - public async Task HappyPath() + public async Task ReadToAnEmptyBufferReturnsZeroAsync() + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[1]); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) + { + Assert.Equal(0, await RandomAccess.ReadAsync(handle, new Memory[] { Array.Empty() }, fileOffset: 0)); + } + } + + [Fact] + public async Task ReadsBytesFromGivenFileAtGivenOffsetAsync() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] expected = new byte[fileSize]; - new Random().NextBytes(expected); + byte[] expected = RandomNumberGenerator.GetBytes(fileSize); File.WriteAllBytes(filePath, expected); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) @@ -66,16 +82,20 @@ public async Task HappyPath() do { int firstBufferLength = (int)Math.Min(actual.Length - total, fileSize / 4); + Memory buffer_1 = actual.AsMemory((int)total, firstBufferLength); + Memory buffer_2 = actual.AsMemory((int)total + firstBufferLength); current = await RandomAccess.ReadAsync( handle, new Memory[] { - actual.AsMemory((int)total, firstBufferLength), - actual.AsMemory((int)total + firstBufferLength) + buffer_1, + buffer_2 }, fileOffset: total); + Assert.InRange(current, 0, buffer_1.Length + buffer_2.Length); + total += current; } while (current != 0); @@ -83,5 +103,19 @@ public async Task HappyPath() Assert.Equal(expected, actual.Take((int)total).ToArray()); } } + + [Fact] + public async Task ReadToTheSameBufferOverwritesContent() + { + string filePath = GetTestFilePath(); + File.WriteAllBytes(filePath, new byte[3] { 1, 2, 3 }); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Open, options: FileOptions.Asynchronous)) + { + byte[] buffer = new byte[1]; + Assert.Equal(buffer.Length + buffer.Length, await RandomAccess.ReadAsync(handle, Enumerable.Repeat(buffer.AsMemory(), 2).ToList(), fileOffset: 0)); + Assert.Equal(2, buffer[0]); + } + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs index 78383748366ca..593cc6e50246d 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/SectorAlignedMemory.Windows.cs @@ -11,15 +11,15 @@ namespace System.IO.Tests { internal sealed class SectorAlignedMemory : MemoryManager { - private bool disposed = false; - private int refCount = 0; - private IntPtr memory; - private int length; + private bool _disposed; + private int _refCount; + private IntPtr _memory; + private int _length; private unsafe SectorAlignedMemory(void* memory, int length) { - this.memory = (IntPtr)memory; - this.length = length; + _memory = (IntPtr)memory; + _length = length; } public static unsafe SectorAlignedMemory Allocate(int length) @@ -33,26 +33,24 @@ public static unsafe SectorAlignedMemory Allocate(int length) return new SectorAlignedMemory(memory, length); } - public bool IsDisposed => disposed; + public bool IsDisposed => _disposed; - public unsafe override Span GetSpan() => new Span((void*)memory, length); - - protected bool IsRetained => refCount > 0; + public unsafe override Span GetSpan() => new Span((void*)_memory, _length); public override MemoryHandle Pin(int elementIndex = 0) { unsafe { Retain(); - if ((uint)elementIndex > length) throw new ArgumentOutOfRangeException(nameof(elementIndex)); - void* pointer = Unsafe.Add((void*)memory, elementIndex); + if ((uint)elementIndex > _length) throw new ArgumentOutOfRangeException(nameof(elementIndex)); + void* pointer = Unsafe.Add((void*)_memory, elementIndex); return new MemoryHandle(pointer, default, this); } } - public bool Release() + private bool Release() { - int newRefCount = Interlocked.Decrement(ref refCount); + int newRefCount = Interlocked.Decrement(ref _refCount); if (newRefCount < 0) { @@ -62,31 +60,31 @@ public bool Release() return newRefCount != 0; } - public void Retain() + private void Retain() { - if (disposed) + if (_disposed) { throw new ObjectDisposedException(nameof(SectorAlignedMemory)); } - Interlocked.Increment(ref refCount); + Interlocked.Increment(ref _refCount); } protected override unsafe void Dispose(bool disposing) { - if (disposed) + if (_disposed) { return; } VirtualAlloc( - memory.ToPointer(), - new UIntPtr((uint)(Marshal.SizeOf() * length)), + _memory.ToPointer(), + new UIntPtr((uint)(Marshal.SizeOf() * _length)), MemOptions.MEM_FREE, PageOptions.PAGE_READWRITE); - memory = IntPtr.Zero; + _memory = IntPtr.Zero; - disposed = true; + _disposed = true; } protected override bool TryGetArray(out ArraySegment arraySegment) diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs index c959973aafd8d..abcac008dc6dd 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/Write.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Security.Cryptography; using Microsoft.Win32.SafeHandles; using Xunit; @@ -24,23 +25,35 @@ public void ThrowsOnReadAccess() } [Fact] - public void HappyPath() + public void WriteUsingEmptyBufferReturnsZero() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write)) + { + Assert.Equal(0, RandomAccess.Write(handle, Array.Empty(), fileOffset: 0)); + } + } + + [Fact] + public void WritesBytesFromGivenBufferToGivenFileAtGivenOffset() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] content = new byte[fileSize]; - new Random().NextBytes(content); + byte[] content = RandomNumberGenerator.GetBytes(fileSize); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) { int total = 0; + int current = 0; while (total != fileSize) { - total += RandomAccess.Write( - handle, - content.AsSpan(total, Math.Min(content.Length - total, fileSize / 4)), - fileOffset: total); + Span buffer = content.AsSpan(total, Math.Min(content.Length - total, fileSize / 4)); + + current = RandomAccess.Write(handle, buffer, fileOffset: total); + + Assert.InRange(current, 0, buffer.Length); + + total += current; } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs index ee1b58e1f2dbb..074f5baac5944 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteAsync.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -19,13 +20,17 @@ protected override bool ShouldThrowForSyncHandle => OperatingSystem.IsWindows(); // on Windows we can NOT perform async IO using sync handle [Fact] - public Task TaskAlreadyCanceledAsync() + public async Task TaskAlreadyCanceledAsync() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) { - CancellationToken token = GetCancelledToken(); + CancellationTokenSource cts = GetCancelledTokenSource(); + CancellationToken token = cts.Token; + Assert.True(RandomAccess.WriteAsync(handle, new byte[1], 0, token).IsCanceled); - return Assert.ThrowsAsync(async () => await RandomAccess.WriteAsync(handle, new byte[1], 0, token)); + + TaskCanceledException ex = await Assert.ThrowsAsync(() => RandomAccess.WriteAsync(handle, new byte[1], 0, token).AsTask()); + Assert.Equal(token, ex.CancellationToken); } } @@ -39,23 +44,35 @@ public async Task ThrowsOnReadAccess() } [Fact] - public async Task HappyPath() + public async Task WriteUsingEmptyBufferReturnsZeroAsync() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write, options: FileOptions.Asynchronous)) + { + Assert.Equal(0, await RandomAccess.WriteAsync(handle, Array.Empty(), fileOffset: 0)); + } + } + + [Fact] + public async Task WritesBytesFromGivenBufferToGivenFileAtGivenOffsetAsync() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] content = new byte[fileSize]; - new Random().NextBytes(content); + byte[] content = RandomNumberGenerator.GetBytes(fileSize); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous)) { int total = 0; + int current = 0; while (total != fileSize) { - total += await RandomAccess.WriteAsync( - handle, - content.AsMemory(total, Math.Min(content.Length - total, fileSize / 4)), - fileOffset: total); + Memory buffer = content.AsMemory(total, Math.Min(content.Length - total, fileSize / 4)); + + current = await RandomAccess.WriteAsync(handle, buffer, fileOffset: total); + + Assert.InRange(current, 0, buffer.Length); + + total += current; } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs index 565bd3b988d71..a9483d6c0eae2 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGather.cs @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; using Microsoft.Win32.SafeHandles; using Xunit; @@ -33,33 +36,68 @@ public void ThrowsOnReadAccess() } [Fact] - public void HappyPath() + public void WriteUsingEmptyBufferReturnsZero() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write)) + { + Assert.Equal(0, RandomAccess.Write(handle, new ReadOnlyMemory[] { Array.Empty() }, fileOffset: 0)); + } + } + + [Fact] + public void WritesBytesFromGivenBuffersToGivenFileAtGivenOffset() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] content = new byte[fileSize]; - new Random().NextBytes(content); + byte[] content = RandomNumberGenerator.GetBytes(fileSize); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None)) { long total = 0; + long current = 0; while (total != fileSize) { int firstBufferLength = (int)Math.Min(content.Length - total, fileSize / 4); + Memory buffer_1 = content.AsMemory((int)total, firstBufferLength); + Memory buffer_2 = content.AsMemory((int)total + firstBufferLength); - total += RandomAccess.Write( + current = RandomAccess.Write( handle, new ReadOnlyMemory[] { - content.AsMemory((int)total, firstBufferLength), - content.AsMemory((int)total + firstBufferLength) + buffer_1, + Array.Empty(), + buffer_2 }, fileOffset: total); + + Assert.InRange(current, 0, buffer_1.Length + buffer_2.Length); + + total += current; } } Assert.Equal(content, File.ReadAllBytes(filePath)); } + + [Fact] + public void DuplicatedBufferDuplicatesContent() + { + const byte value = 1; + const int repeatCount = 2; + string filePath = GetTestFilePath(); + ReadOnlyMemory buffer = new byte[1] { value }; + List> buffers = Enumerable.Repeat(buffer, repeatCount).ToList(); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Create, FileAccess.Write)) + { + Assert.Equal(repeatCount, RandomAccess.Write(handle, buffers, fileOffset: 0)); + } + + byte[] actualContent = File.ReadAllBytes(filePath); + Assert.Equal(repeatCount, actualContent.Length); + Assert.All(actualContent, actual => Assert.Equal(value, actual)); + } } } diff --git a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs index ce19dd110aa1d..f8369eb13c782 100644 --- a/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs +++ b/src/libraries/System.IO.FileSystem/tests/RandomAccess/WriteGatherAsync.cs @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography; using System.Threading; using System.Threading.Tasks; using Microsoft.Win32.SafeHandles; @@ -28,13 +31,17 @@ public void ThrowsArgumentNullExceptionForNullBuffers() } [Fact] - public Task TaskAlreadyCanceledAsync() + public async Task TaskAlreadyCanceledAsync() { using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.CreateNew, FileAccess.ReadWrite, options: FileOptions.Asynchronous)) { - CancellationToken token = GetCancelledToken(); + CancellationTokenSource cts = GetCancelledTokenSource(); + CancellationToken token = cts.Token; + Assert.True(RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { new byte[1] }, 0, token).IsCanceled); - return Assert.ThrowsAsync(async () => await RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { new byte[1] }, 0, token)); + + TaskCanceledException ex = await Assert.ThrowsAsync(() => RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { new byte[1] }, 0, token).AsTask()); + Assert.Equal(token, ex.CancellationToken); } } @@ -48,33 +55,67 @@ public async Task ThrowsOnReadAccess() } [Fact] - public async Task HappyPath() + public async Task WriteUsingEmptyBufferReturnsZeroAsync() + { + using (SafeFileHandle handle = File.OpenHandle(GetTestFilePath(), FileMode.Create, FileAccess.Write, options: FileOptions.Asynchronous)) + { + Assert.Equal(0, await RandomAccess.WriteAsync(handle, new ReadOnlyMemory[] { Array.Empty() }, fileOffset: 0)); + } + } + + [Fact] + public async Task WritesBytesFromGivenBufferToGivenFileAtGivenOffsetAsync() { const int fileSize = 4_001; string filePath = GetTestFilePath(); - byte[] content = new byte[fileSize]; - new Random().NextBytes(content); + byte[] content = RandomNumberGenerator.GetBytes(fileSize); using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.Asynchronous)) { long total = 0; + long current = 0; while (total != fileSize) { int firstBufferLength = (int)Math.Min(content.Length - total, fileSize / 4); + Memory buffer_1 = content.AsMemory((int)total, firstBufferLength); + Memory buffer_2 = content.AsMemory((int)total + firstBufferLength); - total += await RandomAccess.WriteAsync( + current = await RandomAccess.WriteAsync( handle, new ReadOnlyMemory[] { - content.AsMemory((int)total, firstBufferLength), - content.AsMemory((int)total + firstBufferLength) + buffer_1, + buffer_2 }, fileOffset: total); + + Assert.InRange(current, 0, buffer_1.Length + buffer_2.Length); + + total += current; } } Assert.Equal(content, File.ReadAllBytes(filePath)); } + + [Fact] + public async Task DuplicatedBufferDuplicatesContentAsync() + { + const byte value = 1; + const int repeatCount = 2; + string filePath = GetTestFilePath(); + ReadOnlyMemory buffer = new byte[1] { value }; + List> buffers = Enumerable.Repeat(buffer, repeatCount).ToList(); + + using (SafeFileHandle handle = File.OpenHandle(filePath, FileMode.Create, FileAccess.Write, options: FileOptions.Asynchronous)) + { + Assert.Equal(repeatCount, await RandomAccess.WriteAsync(handle, buffers, fileOffset: 0)); + } + + byte[] actualContent = File.ReadAllBytes(filePath); + Assert.Equal(repeatCount, actualContent.Length); + Assert.All(actualContent, actual => Assert.Equal(value, actual)); + } } } diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs index ed41ab7e589e2..b5502ff9926cb 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/File.netcoreapp.cs @@ -10,18 +10,18 @@ namespace System.IO public static partial class File { /// - /// Initializes a new instance of the class with the specified path, creation mode, read/write and sharing permission, the access other FileStreams can have to the same file, the buffer size, additional file options and the allocation size. + /// Initializes a new instance of the class with the specified path, creation mode, read/write and sharing permission, the access other FileStreams can have to the same file, the buffer size, additional file options and the allocation size. /// - /// for information about exceptions. + /// for information about exceptions. public static FileStream Open(string path, FileStreamOptions options) => new FileStream(path, options); /// /// Initializes a new instance of the class with the specified path, creation mode, read/write and sharing permission, the access other SafeFileHandles can have to the same file, additional file options and the allocation size. /// /// A relative or absolute path for the file that the current instance will encapsulate. - /// One of the enumeration values that determines how to open or create the file. The default value is - /// A bitwise combination of the enumeration values that determines how the file can be accessed. The default value is - /// A bitwise combination of the enumeration values that determines how the file will be shared by processes. The default value is . + /// One of the enumeration values that determines how to open or create the file. The default value is + /// A bitwise combination of the enumeration values that determines how the file can be accessed. The default value is + /// A bitwise combination of the enumeration values that determines how the file will be shared by processes. The default value is . /// The initial allocation size in bytes for the file. A positive value is effective only when a regular file is being created, overwritten, or replaced. /// Negative values are not allowed. In other cases (including the default 0 value), it's ignored. /// An object that describes optional parameters to use. @@ -33,15 +33,15 @@ public static partial class File /// is negative. /// -or- /// , , or contain an invalid value. - /// The file cannot be found, such as when is or , and the file specified by does not exist. The file must already exist in these modes. - /// An I/O error, such as specifying when the file specified by already exists, occurred. + /// The file cannot be found, such as when is or , and the file specified by does not exist. The file must already exist in these modes. + /// An I/O error, such as specifying when the file specified by already exists, occurred. /// -or- /// The disk was full (when was provided and was pointing to a regular file). /// -or- /// The file was too large (when was provided and was pointing to a regular file). /// The caller does not have the required permission. /// The specified path is invalid, such as being on an unmapped drive. - /// The requested is not permitted by the operating system for the specified , such as when is or and the file or directory is set for read-only access. + /// The requested is not permitted by the operating system for the specified , such as when is or and the file or directory is set for read-only access. /// -or- /// is specified for , but file encryption is not supported on the current platform. /// The specified path, file name, or both exceed the system-defined maximum length. diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs index 217e12c085550..22a3934fe4089 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Unix.cs @@ -42,7 +42,8 @@ private static unsafe long ReadScatterAtOffset(SafeFileHandle handle, IReadOnlyL long result; try { - for (int i = 0; i < buffers.Count; i++) + int buffersCount = buffers.Count; + for (int i = 0; i < buffersCount; i++) { Memory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); @@ -104,7 +105,8 @@ private static unsafe long WriteGatherAtOffset(SafeFileHandle handle, IReadOnlyL long result; try { - for (int i = 0; i < buffers.Count; i++) + int buffersCount = buffers.Count; + for (int i = 0; i < buffersCount; i++) { ReadOnlyMemory buffer = buffers[i]; MemoryHandle memoryHandle = buffer.Pin(); diff --git a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs index 2ab60472ebd52..5d198ccb0785c 100644 --- a/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs +++ b/src/libraries/System.Private.CoreLib/src/System/IO/RandomAccess.Windows.cs @@ -337,7 +337,7 @@ private static async ValueTask ReadScatterAtOffsetSingleSyscallAsync(SafeF MemoryHandle memoryHandle = buffer.Pin(); memoryHandles[i] = memoryHandle; - unsafe // async method can't be unsafe + unsafe // awaits can't be in an unsafe context { fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); } @@ -486,7 +486,7 @@ private static async ValueTask WriteGatherAtOffsetSingleSyscallAsync(SafeF MemoryHandle memoryHandle = buffer.Pin(); memoryHandles[i] = memoryHandle; - unsafe // async method can't be unsafe + unsafe // awaits can't be in an unsafe context { fileSegments[i] = new IntPtr(memoryHandle.Pointer).ToInt64(); }