Skip to content

Optimize Directory.GetFiles by passing safe patterns to NtQueryDirectoryFile #122947

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephentoub merged 11 commits into from
Jan 26, 2026
+1,130 −21
Merged

Optimize Directory.GetFiles by passing safe patterns to NtQueryDirectoryFile #122947

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
b20b3f2
Initial plan
Copilot Jan 6, 2026
e5120c8
Add OS-level pattern filtering for Directory.GetFiles optimization
stephentoub Jan 7, 2026
358be40
Add more tests
stephentoub Jan 22, 2026
150d2f1
Add more tests
stephentoub Jan 23, 2026
142a3fb
Address code review feedback and fix Unix test failures
Copilot Jan 23, 2026
b47ceeb
Make tests work on all OSes using PathInternal.IsCaseSensitive
Copilot Jan 23, 2026
37a40b2
Remove redundant IsCaseSensitive conditionals with identical branches
Copilot Jan 23, 2026
f44085f
Add .a and .ab files to LeadingDotPatterns test for DOS_QM collapsing
Copilot Jan 24, 2026
a318419
Fix Unix test failures: set AttributesToSkip=0 for dotfile tests
Copilot Jan 24, 2026
9f5d6ef
Fix Windows test failure: include .txt in Win32 DOS_QM ?.txt pattern …
Copilot Jan 25, 2026
6652212
Merge branch 'main' into copilot/optimize-directory-getfiles-patterns
stephentoub Jan 25, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions src/libraries/System.Private.CoreLib/src/System/IO/Enumeration/FileSystemEnumerable.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,20 +17,22 @@ public class FileSystemEnumerable<TResult> : IEnumerable<TResult>
private readonly FindTransform _transform;
private readonly EnumerationOptions _options;
private readonly string _directory;
private readonly string? _expression;

public FileSystemEnumerable(string directory, FindTransform transform, EnumerationOptions? options = null)
: this(directory, transform, options, isNormalized: false)
{
}

internal FileSystemEnumerable(string directory, FindTransform transform, EnumerationOptions? options, bool isNormalized)
internal FileSystemEnumerable(string directory, FindTransform transform, EnumerationOptions? options, bool isNormalized, string? expression = null)
{
ArgumentNullException.ThrowIfNull(directory);
ArgumentNullException.ThrowIfNull(transform);

_directory = directory;
_transform = transform;
_options = options ?? EnumerationOptions.Default;
_expression = expression;

// We need to create the enumerator up front to ensure that we throw I/O exceptions for
// the root directory on creation of the enumerable.
Expand Down Expand Up @@ -62,7 +64,7 @@ private sealed class DelegateEnumerator : FileSystemEnumerator<TResult>
private readonly FileSystemEnumerable<TResult> _enumerable;

public DelegateEnumerator(FileSystemEnumerable<TResult> enumerable, bool isNormalized)
: base(enumerable._directory, isNormalized, enumerable._options)
: base(enumerable._directory, isNormalized, enumerable._options, enumerable._expression)
{
_enumerable = enumerable;
}
Expand Down
21 changes: 15 additions & 6 deletions ...libraries/System.Private.CoreLib/src/System/IO/Enumeration/FileSystemEnumerableFactory.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -118,7 +118,9 @@ internal static IEnumerable<string> UserFiles(string directory,
return new FileSystemEnumerable<string>(
directory,
(ref FileSystemEntry entry) => entry.ToSpecifiedFullPath(),
options)
options,
isNormalized: false,
expression)
{
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
!entry.IsDirectory && MatchesPattern(expression, entry.FileName, options)
Expand All @@ -132,7 +134,9 @@ internal static IEnumerable<string> UserDirectories(string directory,
return new FileSystemEnumerable<string>(
directory,
(ref FileSystemEntry entry) => entry.ToSpecifiedFullPath(),
options)
options,
isNormalized: false,
expression)
{
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
entry.IsDirectory && MatchesPattern(expression, entry.FileName, options)
Expand All @@ -146,7 +150,9 @@ internal static IEnumerable<string> UserEntries(string directory,
return new FileSystemEnumerable<string>(
directory,
(ref FileSystemEntry entry) => entry.ToSpecifiedFullPath(),
options)
options,
isNormalized: false,
expression)
{
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
MatchesPattern(expression, entry.FileName, options)
Expand All @@ -163,7 +169,8 @@ internal static IEnumerable<FileInfo> FileInfos(
directory,
(ref FileSystemEntry entry) => (FileInfo)entry.ToFileSystemInfo(),
options,
isNormalized)
isNormalized,
expression)
{
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
!entry.IsDirectory && MatchesPattern(expression, entry.FileName, options)
Expand All @@ -180,7 +187,8 @@ internal static IEnumerable<DirectoryInfo> DirectoryInfos(
directory,
(ref FileSystemEntry entry) => (DirectoryInfo)entry.ToFileSystemInfo(),
options,
isNormalized)
isNormalized,
expression)
{
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
entry.IsDirectory && MatchesPattern(expression, entry.FileName, options)
Expand All @@ -197,7 +205,8 @@ internal static IEnumerable<FileSystemInfo> FileSystemInfos(
directory,
(ref FileSystemEntry entry) => entry.ToFileSystemInfo(),
options,
isNormalized)
isNormalized,
expression)
{
ShouldIncludePredicate = (ref FileSystemEntry entry) =>
MatchesPattern(expression, entry.FileName, options)
Expand Down
13 changes: 13 additions & 0 deletions src/libraries/System.Private.CoreLib/src/System/IO/Enumeration/FileSystemEnumerator.Unix.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,19 @@ public abstract unsafe partial class FileSystemEnumerator<TResult> : CriticalFin
// Used for creating full paths
private char[]? _pathBuffer;

/// <summary>
/// Encapsulates a find operation.
/// </summary>
/// <param name="directory">The directory to search in.</param>
/// <param name="isNormalized">Whether the directory path is already normalized or not.</param>
/// <param name="options">Enumeration options to use.</param>
/// <param name="expression">The search expression to potentially use for OS-level filtering (Windows only).</param>
internal FileSystemEnumerator(string directory, bool isNormalized, EnumerationOptions? options, string? expression) :
this(directory, isNormalized, options)
{
_ = expression; // unused
}

private void Init()
{
// We need to initialize the directory handle up front to ensure
Expand Down
89 changes: 77 additions & 12 deletions ...ibraries/System.Private.CoreLib/src/System/IO/Enumeration/FileSystemEnumerator.Windows.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using System.Buffers;
using System.Collections.Generic;
using System.Diagnostics;
using System.Diagnostics.CodeAnalysis;
using System.IO;
using System.Runtime.CompilerServices;
using System.Runtime.ConstrainedExecution;
Expand All @@ -25,6 +26,7 @@ public abstract unsafe partial class FileSystemEnumerator<TResult> : CriticalFin
private readonly string _originalRootDirectory;
private readonly string _rootDirectory;
private readonly EnumerationOptions _options;
private readonly string? _expression;

private readonly object _lock = new object();

Expand All @@ -37,6 +39,20 @@ public abstract unsafe partial class FileSystemEnumerator<TResult> : CriticalFin
private string? _currentPath;
private bool _lastEntryFound;
private Queue<(IntPtr Handle, string Path, int RemainingDepth)>? _pending;
private bool _isFirstGetData = true;

/// <summary>
/// Encapsulates a find operation.
/// </summary>
/// <param name="directory">The directory to search in.</param>
/// <param name="isNormalized">Whether the directory path is already normalized or not.</param>
/// <param name="options">Enumeration options to use.</param>
/// <param name="expression">The search expression to potentially use for OS-level filtering (Windows only).</param>
internal FileSystemEnumerator(string directory, bool isNormalized, EnumerationOptions? options, string? expression) :
this(directory, isNormalized, options)
{
_expression = expression;
}

private void Init()
{
Expand Down Expand Up @@ -83,18 +99,39 @@ private bool GetData()
Debug.Assert(_directoryHandle != (IntPtr)(-1) && _directoryHandle != IntPtr.Zero && !_lastEntryFound);

Interop.NtDll.IO_STATUS_BLOCK statusBlock;
int status = Interop.NtDll.NtQueryDirectoryFile(
FileHandle: _directoryHandle,
Event: IntPtr.Zero,
ApcRoutine: IntPtr.Zero,
ApcContext: IntPtr.Zero,
IoStatusBlock: &statusBlock,
FileInformation: _buffer,
Length: (uint)_bufferLength,
FileInformationClass: Interop.NtDll.FILE_INFORMATION_CLASS.FileFullDirectoryInformation,
ReturnSingleEntry: Interop.BOOLEAN.FALSE,
FileName: null,
RestartScan: Interop.BOOLEAN.FALSE);
int status;
fixed (char* pBuffer = _expression)
{
Interop.UNICODE_STRING* fileNamePtr = null;
Interop.UNICODE_STRING fileNameStruct = default;

// On the first call for each directory, check if we can use the expression as an OS-level filter hint.
// All results will still be validated in managed code, as the OS filtering may be looser than what .NET expects.
// We cannot use OS filtering when recursing, as the filter would exclude subdirectories that don't match the pattern.
if (_isFirstGetData)
{
_isFirstGetData = false;
if (!_options.RecurseSubdirectories && IsSafePatternForOSFilter(_expression))
{
fileNameStruct.Length = fileNameStruct.MaximumLength = (ushort)(_expression.Length * sizeof(char));
fileNameStruct.Buffer = (IntPtr)pBuffer;
fileNamePtr = &fileNameStruct;
}
}

status = Interop.NtDll.NtQueryDirectoryFile(
FileHandle: _directoryHandle,
Event: IntPtr.Zero,
ApcRoutine: IntPtr.Zero,
ApcContext: IntPtr.Zero,
IoStatusBlock: &statusBlock,
FileInformation: _buffer,
Length: (uint)_bufferLength,
FileInformationClass: Interop.NtDll.FILE_INFORMATION_CLASS.FileFullDirectoryInformation,
ReturnSingleEntry: Interop.BOOLEAN.FALSE,
FileName: fileNamePtr,
RestartScan: Interop.BOOLEAN.FALSE);
}

switch ((uint)status)
{
Expand Down Expand Up @@ -287,6 +324,7 @@ private bool DequeueNextDirectory()
return false;

(_directoryHandle, _currentPath, _remainingRecursionDepth) = _pending.Dequeue();
_isFirstGetData = true;
return true;
}

Expand Down Expand Up @@ -319,5 +357,32 @@ private void InternalDispose(bool disposing)

Dispose(disposing);
}

/// <summary>
/// Determines if the given expression is safe to pass to NtQueryDirectoryFile as a hint.
/// </summary>
/// <remarks>
/// This is a conservative check: false negatives are ok, but false positives are not.
/// Worst case is this returns false when it could have returned true and we miss an optimization opportunity.
/// Safe patterns are those where the OS filter will return a superset of what .NET expects,
/// allowing the managed MatchesPattern filter to ensure correctness.
/// Patterns combining * with literal characters (e.g., *.txt) are safe.
/// Patterns with ? are unsafe due to DOS_QM behavioral differences.
/// Pattern *.* is unsafe because .NET treats it as *, but OS requires a . in the name.
/// Patterns ending with . have special behaviors.
/// Patterns with invalid filename characters are unsafe as NtQueryDirectoryFile will reject them.
/// </remarks>
private static bool IsSafePatternForOSFilter([NotNullWhen(true)] string? expression) =>
!string.IsNullOrEmpty(expression) &&
expression is not "*" and not "*.*" &&
expression.Length <= 255 && // Max filename length
!expression.EndsWith('.') &&
!expression.ContainsAny(s_unsafeForFilter);

// Path.GetInvalidFileNameChars() minus '*' which is allowed in search patterns
private static readonly SearchValues<char> s_unsafeForFilter = SearchValues.Create(
"\u0000\u0001\u0002\u0003\u0004\u0005\u0006\u0007\u0008\u0009\u000a\u000b\u000c\u000d\u000e\u000f" +
"\u0010\u0011\u0012\u0013\u0014\u0015\u0016\u0017\u0018\u0019\u001a\u001b\u001c\u001d\u001e\u001f" +
"\"<>|:/\\?");
}
}
Loading
Loading