Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

File preallocationSize: align Windows and Unix behavior. #59338

Merged
merged 18 commits into from
Sep 23, 2021
Merged
Show file tree
Hide file tree
Changes from 7 commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/libraries/Native/Unix/System.Native/pal_io.c
Original file line number Diff line number Diff line change
Expand Up @@ -1019,7 +1019,7 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length)
#elif defined(F_PREALLOCATE) // macOS
fstore_t fstore;
fstore.fst_flags = F_ALLOCATEALL; // Allocate all requested space or no space at all.
fstore.fst_posmode = 0;
fstore.fst_posmode = F_PEOFPOSMODE; // allocate from the physical end of file.
tmds marked this conversation as resolved.
Show resolved Hide resolved
fstore.fst_offset = (off_t)offset;
fstore.fst_length = (off_t)length;
fstore.fst_bytesalloc = 0; // output size, can be > length
Expand All @@ -1032,6 +1032,8 @@ int32_t SystemNative_FAllocate(intptr_t fd, int64_t offset, int64_t length)
errno = EOPNOTSUPP;
#endif

assert(result == 0 || errno != EINVAL);

return result;
}

Expand Down
15 changes: 14 additions & 1 deletion src/libraries/System.IO.FileSystem/tests/File/Open.cs
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ protected override FileStream CreateFileStream(string path, FileMode mode, FileA
}
}

public class File_Open_str_options_as : FileStream_ctor_options_as
public class File_Open_str_options : FileStream_ctor_options
{
protected override FileStream CreateFileStream(string path, FileMode mode)
{
Expand Down Expand Up @@ -73,6 +73,19 @@ protected override FileStream CreateFileStream(string path, FileMode mode, FileA
BufferSize = bufferSize
tmds marked this conversation as resolved.
Show resolved Hide resolved
});
}

protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize)
{
return File.Open(path,
new FileStreamOptions {
Mode = mode,
Access = access,
Share = share,
Options = options,
BufferSize = bufferSize,
PreallocationSize = preallocationSize
});
}
}

public class File_OpenSpecial : FileStream_ctor_str_fm_fa_fs
Expand Down
10 changes: 3 additions & 7 deletions src/libraries/System.IO.FileSystem/tests/File/OpenHandle.cs
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
namespace System.IO.Tests
{
// to avoid a lot of code duplication, we reuse FileStream tests
public class File_OpenHandle : FileStream_ctor_options_as
public class File_OpenHandle : FileStream_ctor_options
{
protected override string GetExpectedParamName(string paramName) => paramName;

Expand All @@ -23,12 +23,8 @@ protected override FileStream CreateFileStream(string path, FileMode mode, FileA
protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options)
=> new FileStream(File.OpenHandle(path, mode, access, share, options), access, bufferSize, (options & FileOptions.Asynchronous) != 0);

[Fact]
public override void NegativePreallocationSizeThrows()
{
ArgumentOutOfRangeException ex = Assert.Throws<ArgumentOutOfRangeException>(
() => File.OpenHandle("validPath", FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, preallocationSize: -1));
}
protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize)
=> new FileStream(File.OpenHandle(path, mode, access, share, options, preallocationSize), access, bufferSize, (options & FileOptions.Asynchronous) != 0);

[ActiveIssue("https://github.com/dotnet/runtime/issues/53432")]
[Theory, MemberData(nameof(StreamSpecifiers))]
Expand Down
15 changes: 14 additions & 1 deletion src/libraries/System.IO.FileSystem/tests/FileInfo/Open.cs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ protected override FileStream CreateFileStream(string path, FileMode mode, FileA
}
}

public class FileInfo_Open_options_as : FileStream_ctor_options_as
public class FileInfo_Open_options : FileStream_ctor_options
{
protected override FileStream CreateFileStream(string path, FileMode mode)
{
Expand Down Expand Up @@ -105,6 +105,19 @@ protected override FileStream CreateFileStream(string path, FileMode mode, FileA
BufferSize = bufferSize
});
}

protected override FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize)
{
return new FileInfo(path).Open(
new FileStreamOptions {
Mode = mode,
Access = access,
Share = share,
Options = options,
BufferSize = bufferSize,
PreallocationSize = preallocationSize
});
}
}

public class FileInfo_OpenSpecial : FileStream_ctor_str_fm_fa_fs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

namespace System.IO.Tests
{
public partial class FileStream_ctor_options_as
public partial class FileStream_ctor_options
{
private static long GetAllocatedSize(FileStream fileStream)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,15 +6,18 @@

namespace System.IO.Tests
{
public partial class FileStream_ctor_options_as
public partial class FileStream_ctor_options
{
private static long GetAllocatedSize(FileStream fileStream)
{
bool isOSX = RuntimeInformation.IsOSPlatform(OSPlatform.OSX);
// Call 'stat' to get the number of blocks, and size of blocks.
using var px = Process.Start(new ProcessStartInfo
{
FileName = "stat",
ArgumentList = { "-c", "%b %B", fileStream.Name },
ArgumentList = { isOSX ? "-f" : "-c",
isOSX ? "%b %k" : "%b %B",
fileStream.Name },
RedirectStandardOutput = true
});
string stdout = px.StandardOutput.ReadToEnd();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@

namespace System.IO.Tests
{
public partial class FileStream_ctor_options_as
public partial class FileStream_ctor_options
{
private unsafe long GetAllocatedSize(FileStream fileStream)
{
Expand All @@ -30,10 +30,10 @@ public void ExtendedPathsAreSupported(string prefix)

string filePath = prefix + Path.GetFullPath(GetPathToNonExistingFile());

using (var fs = new FileStream(filePath, GetOptions(FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, preallocationSize)))
using (var fs = CreateFileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 4096, FileOptions.None, preallocationSize))
{
Assert.Equal(0, fs.Length);
Assert.Equal(preallocationSize, GetAllocatedSize(fs));
Assert.True(GetAllocatedSize(fs) >= preallocationSize);
}
}

Expand All @@ -48,7 +48,7 @@ public void WhenFileIsTooLargeTheErrorMessageContainsAllDetails(FileMode mode)
string filePath = GetPathToNonExistingFile();
Assert.StartsWith(Path.GetTempPath(), filePath); // this is what IsFat32 method relies on

IOException ex = Assert.Throws<IOException>(() => new FileStream(filePath, GetOptions(mode, FileAccess.Write, FileShare.None, FileOptions.None, tooMuch)));
IOException ex = Assert.Throws<IOException>(() => CreateFileStream(filePath, mode, FileAccess.Write, FileShare.None, bufferSize: 4096, FileOptions.None, tooMuch));
Assert.Contains(filePath, ex.Message);
Assert.Contains(tooMuch.ToString(), ex.Message);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@

namespace System.IO.Tests
{
public abstract class FileStream_ctor_options_as_base : FileStream_ctor_str_fm_fa_fs_buffer_fo
// Don't run in parallel as the WhenDiskIsFullTheErrorMessageContainsAllDetails test
// consumes entire available free space on the disk (only on Linux, this is how posix_fallocate works)
// and if we try to run other disk-writing test in the meantime we are going to get "No space left on device" exception.
[Collection("NoParallelTests")]
public partial class FileStream_ctor_options : FileStream_ctor_str_fm_fa_fs_buffer_fo
{
protected override string GetExpectedParamName(string paramName) => "value";

Expand Down Expand Up @@ -38,70 +42,97 @@ protected override FileStream CreateFileStream(string path, FileMode mode, FileA
Options = options
});

protected FileStreamOptions GetOptions(FileMode mode, FileAccess access, FileShare share, FileOptions options, long preallocationSize)
=> new FileStreamOptions
{
Mode = mode,
Access = access,
Share = share,
Options = options,
PreallocationSize = preallocationSize
};
}

[CollectionDefinition("NoParallelTests", DisableParallelization = true)]
public partial class NoParallelTests { }
protected virtual FileStream CreateFileStream(string path, FileMode mode, FileAccess access, FileShare share, int bufferSize, FileOptions options, long preallocationSize)
=> new FileStream(path,
new FileStreamOptions
{
Mode = mode,
Access = access,
Share = share,
BufferSize = bufferSize,
Options = options,
PreallocationSize = preallocationSize
});

// Don't run in parallel as the WhenDiskIsFullTheErrorMessageContainsAllDetails test
// consumes entire available free space on the disk (only on Linux, this is how posix_fallocate works)
// and if we try to run other disk-writing test in the meantime we are going to get "No space left on device" exception.
[Collection("NoParallelTests")]
public partial class FileStream_ctor_options_as : FileStream_ctor_options_as_base
{
[Fact]
public virtual void NegativePreallocationSizeThrows()
{
string filePath = GetPathToNonExistingFile();
string filePath = GetTestFilePath();
tmds marked this conversation as resolved.
Show resolved Hide resolved
ArgumentOutOfRangeException ex = Assert.Throws<ArgumentOutOfRangeException>(
() => new FileStream(filePath, GetOptions(FileMode.CreateNew, FileAccess.Write, FileShare.None, FileOptions.None, -1)));
() => CreateFileStream(filePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, bufferSize: 1, FileOptions.None, preallocationSize: -1));
}

[Theory]
[InlineData(FileMode.Append)]
[InlineData(FileMode.Open)]
[InlineData(FileMode.OpenOrCreate)]
public void PreallocationSizeThrowsForFileModeOpenAndAppend(FileMode mode)
[InlineData(FileMode.Truncate)]
public void PreallocationSizeThrowsForFileModesThatOpenExistingFiles(FileMode mode)
{
const int initialSize = 10;
string filePath = GetPathToNonExistingFile();
File.WriteAllBytes(filePath, new byte[initialSize]);

Assert.Throws<ArgumentException>(
() => new FileStream(filePath, GetOptions(mode, FileAccess.Write, FileShare.None, FileOptions.None, preallocationSize: 20)));
() => CreateFileStream(filePath, mode, FileAccess.Write, FileShare.None, bufferSize: 1, FileOptions.None, preallocationSize: 20));
}
tmds marked this conversation as resolved.
Show resolved Hide resolved

[Theory]
[InlineData(FileMode.Create)]
[InlineData(FileMode.CreateNew)]
[InlineData(FileMode.Truncate)]
public void PreallocationSize(FileMode mode)
public void PreallocationSizeThrowsForReadOnlyAccess(FileMode mode)
{
const long preallocationSize = 123;
Assert.Throws<ArgumentException>(
() => CreateFileStream(GetPathToNonExistingFile(), mode, FileAccess.Read, FileShare.None, bufferSize: 1, FileOptions.None, preallocationSize: 20));
}

string filePath = GetPathToNonExistingFile();
if (mode == FileMode.Truncate)
[Theory]
[InlineData(FileMode.Create, false)]
[InlineData(FileMode.Create, true)]
[InlineData(FileMode.CreateNew, false)]
[InlineData(FileMode.Append, false)]
[InlineData(FileMode.Append, true)]
[InlineData(FileMode.Open, true)]
[InlineData(FileMode.OpenOrCreate, true)]
[InlineData(FileMode.OpenOrCreate, false)]
[InlineData(FileMode.Truncate, true)]
public void ZeroPreallocationSizeDoesNotAllocate(FileMode mode, bool createFile)
{
string filename = GetPathToNonExistingFile();

if (createFile)
{
File.WriteAllText(filename, "");
}

using (var fs = CreateFileStream(filename, mode, FileAccess.Write, FileShare.None, bufferSize: 1, FileOptions.None, preallocationSize: 0))
tmds marked this conversation as resolved.
Show resolved Hide resolved
{
const int initialSize = 10;
File.WriteAllBytes(filePath, new byte[initialSize]);
Assert.Equal(0, fs.Length);
Assert.Equal(0, GetAllocatedSize(fs));
Assert.Equal(0, fs.Position);
}
}

[Theory]
[InlineData(FileAccess.Write, FileMode.Create)]
[InlineData(FileAccess.Write, FileMode.CreateNew)]
[InlineData(FileAccess.ReadWrite, FileMode.Create)]
[InlineData(FileAccess.ReadWrite, FileMode.CreateNew)]
public void PreallocationSize(FileAccess access, FileMode mode)
{
const long preallocationSize = 123;

using (var fs = new FileStream(filePath, GetOptions(mode, FileAccess.ReadWrite, FileShare.None, FileOptions.None, preallocationSize)))
using (var fs = CreateFileStream(GetPathToNonExistingFile(), mode, access, FileShare.None, bufferSize: 1, FileOptions.None, preallocationSize))
{
Assert.Equal(0, fs.Length);
if (SupportsPreallocation)
{
Assert.True(GetAllocatedSize(fs) >= preallocationSize);
tmds marked this conversation as resolved.
Show resolved Hide resolved
}
else
{
Assert.Equal(0, GetAllocatedSize(fs));
}
Assert.Equal(0, fs.Position);
}
}
Expand All @@ -114,14 +145,13 @@ public void PreallocationSize(FileMode mode)
[Theory]
[InlineData(FileMode.Create)]
[InlineData(FileMode.CreateNew)]
[InlineData(FileMode.Truncate)]
public void WhenDiskIsFullTheErrorMessageContainsAllDetails(FileMode mode)
{
const long tooMuch = 1024L * 1024L * 1024L * 1024L; // 1 TB

string filePath = GetPathToNonExistingFile();

IOException ex = Assert.Throws<IOException>(() => new FileStream(filePath, GetOptions(mode, FileAccess.Write, FileShare.None, FileOptions.None, tooMuch)));
IOException ex = Assert.Throws<IOException>(() => CreateFileStream(filePath, mode, FileAccess.Write, FileShare.None, bufferSize: 1, FileOptions.None, tooMuch));
Assert.Contains(filePath, ex.Message);
Assert.Contains(tooMuch.ToString(), ex.Message);

Expand All @@ -146,4 +176,7 @@ private string GetPathToNonExistingFile()
return filePath;
}
}

[CollectionDefinition("NoParallelTests", DisableParallelization = true)]
public partial class NoParallelTests { }
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@
<Compile Include="FileInfo\IsReadOnly.cs" />
<Compile Include="FileInfo\Replace.cs" />
<Compile Include="FileInfo\SymbolicLinks.cs" />
<Compile Include="FileStream\ctor_options_as.cs" />
<Compile Include="FileStream\ctor_options.cs" />
<Compile Include="FileStream\Handle.cs" />
<Compile Include="Directory\GetLogicalDrives.cs" />
<Compile Include="FileStream\LockUnlock.cs" />
Expand Down Expand Up @@ -71,13 +71,13 @@
<Compile Include="Base\SymbolicLinks\BaseSymbolicLinks.Unix.cs" />
<Compile Include="FileSystemTest.Unix.cs" />
<Compile Include="$(CommonPath)Interop\Unix\Interop.Libraries.cs" Link="Interop\Unix\Interop.Libraries.cs" />
<Compile Include="FileStream\ctor_options_as.Unix.cs" />
<Compile Include="FileStream\ctor_options.Unix.cs" />
</ItemGroup>
<ItemGroup Condition="'$(TargetsWindows)' == 'true'">
<Compile Include="Base\SymbolicLinks\BaseSymbolicLinks.Windows.cs" />
<Compile Include="Directory\Delete.Windows.cs" />
<Compile Include="FileSystemTest.Windows.cs" />
<Compile Include="FileStream\ctor_options_as.Windows.cs" />
<Compile Include="FileStream\ctor_options.Windows.cs" />
<Compile Include="FileStream\FileStreamConformanceTests.Windows.cs" />
<Compile Include="Junctions.Windows.cs" />
<Compile Include="RandomAccess\Mixed.Windows.cs" />
Expand All @@ -102,7 +102,7 @@
<ItemGroup Condition="'$(TargetsBrowser)' == 'true'">
<Compile Include="Base\SymbolicLinks\BaseSymbolicLinks.Unix.cs" />
<Compile Include="FileSystemTest.Browser.cs" />
<Compile Include="FileStream\ctor_options_as.Browser.cs" />
<Compile Include="FileStream\ctor_options.Browser.cs" />
</ItemGroup>
<ItemGroup>
<!-- Rewritten -->
Expand Down
Loading