From c30c0858044e55a01ecdf292ece6882105773be8 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Mon, 31 Aug 2020 11:16:06 -0500 Subject: [PATCH 1/3] Add Mmap.madvise! --- stdlib/Mmap/src/Mmap.jl | 60 ++++++++++++++++++++++++++++++++++++ stdlib/Mmap/test/runtests.jl | 14 +++++++++ 2 files changed, 74 insertions(+) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 69d48b3656807..4316d70a4267a 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -349,4 +349,64 @@ function sync!(m::Array{T}, flags::Integer=MS_SYNC) where T end sync!(B::BitArray, flags::Integer=MS_SYNC) = sync!(B.chunks, flags) +@static if Sys.isunix() +const MADV_NORMAL = 0 +const MADV_RANDOM = 1 +const MADV_SEQUENTIAL = 2 +const MADV_WILLNEED = 3 +const MADV_DONTNEED = 4 +if Sys.islinux() + const MADV_FREE = 8 + const MADV_REMOVE = 9 + const MADV_DONTFORK = 10 + const MADV_DOFORK = 11 + const MADV_MERGEABLE = 12 + const MADV_UNMERGEABLE = 13 + const MADV_HUGEPAGE = 14 + const MADV_NOHUGEPAGE = 15 + const MADV_DONTDUMP = 16 + const MADV_DODUMP = 17 + const MADV_WIPEONFORK = 18 + const MADV_KEEPONFORK = 19 + const MADV_COLD = 20 + const MADV_PAGEOUT = 21 + const MADV_HWPOISON = 100 + const MADV_SOFT_OFFLINE = 101 +elseif Sys.isapple() + const MADV_FREE = 5 +elseif Sys.isfreebsd() || Sys.isdragonfly() + const MADV_FREE = 5 + const MADV_NOSYNC = 6 + const MADV_AUTOSYNC = 7 + const MADV_NOCORE = 8 + const MADV_CORE = 9 + if Sys.isfreebsd() + const MADV_PROTECT = 10 + else + const MADV_INVAL = 10 + const MADV_SETMAP = 11 + end +elseif Sys.isopenbsd() || Sys.isnetbsd() + const MADV_SPACEAVAIL = 5 + const MADV_FREE = 6 +end + +""" + Mmap.madvise!(array, flag::Integer = Mmap.MADV_NORMAL) + +Advises the kernel on the intended usage of the memory-mapped `array`, with the intent +`flag` being one of the available `MADV_*` constants. +""" +function madvise!(m::Array{T}, flag::Integer=MADV_NORMAL) where T + offset = rem(UInt(pointer(m)), PAGESIZE) + ptr = pointer(m) - offset + mmaplen = length(m) * sizeof(T) + offset + GC.@preserve m begin + systemerror("madvise", + ccall(:madvise, Cint, (Ptr{Cvoid}, Csize_t, Cint), ptr, mmaplen, flag) != 0) + end +end +madvise!(B::BitArray, flag::Integer=MADV_NORMAL) = madvise!(B.chunks, flag) +end # Sys.isunix() + end # module diff --git a/stdlib/Mmap/test/runtests.jl b/stdlib/Mmap/test/runtests.jl index 46b08992dc699..79d018c5075e7 100644 --- a/stdlib/Mmap/test/runtests.jl +++ b/stdlib/Mmap/test/runtests.jl @@ -307,6 +307,20 @@ n = similar(m, 12) @test size(n) == (12,) finalize(m); m = nothing; GC.gc() +if Sys.isunix() + file = tempname() + write(file, rand(Float64, 20)) + A = Mmap.mmap(file, Vector{Float64}, 20) + @test Mmap.madvise!(A, Mmap.MADV_WILLNEED) === nothing # checking for no error + finalize(A); A = nothing; GC.gc() + + write(file, BitArray(rand(Bool, 20))) + b = Mmap.mmap(file, BitArray, 20) + @test Mmap.madvise!(b, Mmap.MADV_WILLNEED) === nothing + finalize(b); b = nothing; GC.gc() + rm(file) +end + # test #14885 file = tempname() touch(file) From f61c40be3cdb1a2cfa2f52765870cb5cba7da032 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Wed, 2 Sep 2020 22:50:20 -0500 Subject: [PATCH 2/3] Add NEWS --- NEWS.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NEWS.md b/NEWS.md index 98c68b32b7a26..2be930369c7b9 100644 --- a/NEWS.md +++ b/NEWS.md @@ -147,6 +147,10 @@ Standard library changes * Change `uuid1` and `uuid4` to use `Random.RandomDevice()` as default random number generator ([#35872]). * Added `parse(::Type{UUID}, ::AbstractString)` method +#### Mmap +* On Unix systems, the `Mmap.madvise!` function (along with OS-specific `Mmap.MADV_*` + constants) has been added to give advice on handling of memory-mapped arrays. ([#37369]) + Deprecated or removed --------------------- From 2f6c1f459a8aa8093c955205d4ac00bf9e789d88 Mon Sep 17 00:00:00 2001 From: Justin Willmert Date: Fri, 4 Sep 2020 19:28:12 -0500 Subject: [PATCH 3/3] Replace `length(m)*sizeof(eltype(m))` with `sizeof(m)` --- stdlib/Mmap/src/Mmap.jl | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/stdlib/Mmap/src/Mmap.jl b/stdlib/Mmap/src/Mmap.jl index 4316d70a4267a..91db2d43701d5 100644 --- a/stdlib/Mmap/src/Mmap.jl +++ b/stdlib/Mmap/src/Mmap.jl @@ -335,10 +335,10 @@ const MS_SYNC = 4 Forces synchronization between the in-memory version of a memory-mapped `Array` or [`BitArray`](@ref) and the on-disk version. """ -function sync!(m::Array{T}, flags::Integer=MS_SYNC) where T +function sync!(m::Array, flags::Integer=MS_SYNC) offset = rem(UInt(pointer(m)), PAGESIZE) ptr = pointer(m) - offset - mmaplen = length(m) * sizeof(T) + offset + mmaplen = sizeof(m) + offset GC.@preserve m @static if Sys.isunix() systemerror("msync", ccall(:msync, Cint, (Ptr{Cvoid}, Csize_t, Cint), ptr, mmaplen, flags) != 0) @@ -397,10 +397,10 @@ end Advises the kernel on the intended usage of the memory-mapped `array`, with the intent `flag` being one of the available `MADV_*` constants. """ -function madvise!(m::Array{T}, flag::Integer=MADV_NORMAL) where T +function madvise!(m::Array, flag::Integer=MADV_NORMAL) offset = rem(UInt(pointer(m)), PAGESIZE) ptr = pointer(m) - offset - mmaplen = length(m) * sizeof(T) + offset + mmaplen = sizeof(m) + offset GC.@preserve m begin systemerror("madvise", ccall(:madvise, Cint, (Ptr{Cvoid}, Csize_t, Cint), ptr, mmaplen, flag) != 0)