diff --git a/Makefile b/Makefile index 9e9f2c8f0c900..533c8b7e2989d 100644 --- a/Makefile +++ b/Makefile @@ -477,7 +477,7 @@ endif exe: # run Inno Setup to compile installer - $(call spawn,$(JULIAHOME)/dist-extras/inno/iscc.exe /DAppVersion=$(JULIA_VERSION) /DSourceDir="$(call cygpath_w,$(BUILDROOT)/julia-$(JULIA_COMMIT))" /DRepoDir="$(call cygpath_w,$(JULIAHOME))" /F"$(JULIA_BINARYDIST_FILENAME)" /O"$(call cygpath_w,$(BUILDROOT))" $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.iss)) + $(call spawn,$(JULIAHOME)/dist-extras/inno/iscc.exe /DAppVersion=$(JULIA_VERSION) /DSourceDir="$(call cygpath_w,$(BUILDROOT)/julia-$(JULIA_COMMIT))" /DRepoDir="$(call cygpath_w,$(JULIAHOME))" /F"$(JULIA_BINARYDIST_FILENAME)" /O"$(call cygpath_w,$(BUILDROOT))" $(INNO_ARGS) $(call cygpath_w,$(JULIAHOME)/contrib/windows/build-installer.iss)) chmod a+x "$(BUILDROOT)/$(JULIA_BINARYDIST_FILENAME).exe" app: diff --git a/base/compiler/tfuncs.jl b/base/compiler/tfuncs.jl index 15fb81e6643ed..18dec7d839950 100644 --- a/base/compiler/tfuncs.jl +++ b/base/compiler/tfuncs.jl @@ -770,7 +770,7 @@ function getfield_tfunc(@nospecialize(s00), @nospecialize(name)) nv = fieldindex(widenconst(s), nv, false) end if isa(nv, Int) && 1 <= nv <= length(s.fields) - return s.fields[nv] + return unwrapva(s.fields[nv]) end end s = widenconst(s) diff --git a/base/compiler/typeutils.jl b/base/compiler/typeutils.jl index d6eb7305b1c8d..ebb06a33a2d74 100644 --- a/base/compiler/typeutils.jl +++ b/base/compiler/typeutils.jl @@ -70,15 +70,6 @@ function typesubtract(@nospecialize(a), @nospecialize(b)) if isa(a, Union) return Union{typesubtract(a.a, b), typesubtract(a.b, b)} - elseif a isa DataType - if b isa DataType - if a.name === b.name === Tuple.name && length(a.types) == length(b.types) - ta = switchtupleunion(a) - if length(ta) > 1 - return typesubtract(Union{ta...}, b) - end - end - end end return a # TODO: improve this bound? end diff --git a/base/file.jl b/base/file.jl index 66640a20a39a7..c8db25cabc9bd 100644 --- a/base/file.jl +++ b/base/file.jl @@ -458,6 +458,32 @@ function tempdir() end end +""" + prepare_for_deletion(path::AbstractString) + +Prepares the given `path` for deletion by ensuring that all directories within that +`path` have write permissions, so that files can be removed from them. This is +automatically invoked by methods such as `mktempdir()` to ensure that no matter what +weird permissions a user may have created directories with within the temporary prefix, +it will always be deleted. +""" +function prepare_for_deletion(path::AbstractString) + # Nothing to do for non-directories + if !isdir(path) + return + end + + try chmod(path, filemode(path) | 0o333) + catch; end + for (root, dirs, files) in walkdir(path) + for dir in dirs + dpath = joinpath(root, dir) + try chmod(dpath, filemode(dpath) | 0o333) + catch; end + end + end +end + const TEMP_CLEANUP_MIN = Ref(1024) const TEMP_CLEANUP_MAX = Ref(1024) const TEMP_CLEANUP = Dict{String,Bool}() @@ -484,6 +510,7 @@ function temp_cleanup_purge(all::Bool=true) if (all || asap) && ispath(path) need_gc && GC.gc(true) need_gc = false + prepare_for_deletion(path) rm(path, recursive=true, force=true) end !ispath(path) && delete!(TEMP_CLEANUP, path) @@ -682,7 +709,10 @@ function mktempdir(fn::Function, parent::AbstractString=tempdir(); fn(tmpdir) finally try - ispath(tmpdir) && rm(tmpdir, recursive=true) + if ispath(tmpdir) + prepare_for_deletion(tmpdir) + rm(tmpdir, recursive=true) + end catch ex @error "mktempdir cleanup" _group=:file exception=(ex, catch_backtrace()) # might be possible to remove later diff --git a/base/methodshow.jl b/base/methodshow.jl index 6f9c46f929100..1e4b68efb0f66 100644 --- a/base/methodshow.jl +++ b/base/methodshow.jl @@ -13,7 +13,7 @@ function argtype_decl(env, n, sig::DataType, i::Int, nargs, isva::Bool) # -> (ar s = string(n) i = findfirst(isequal('#'), s) if i !== nothing - s = s[1:i-1] + s = s[1:prevind(s, i)] end if t === Any && !isempty(s) return s, "" diff --git a/base/process.jl b/base/process.jl index e664d7411de69..e578db43d3732 100644 --- a/base/process.jl +++ b/base/process.jl @@ -546,7 +546,7 @@ Returns successfully if the process has already exited, but throws an error if killing the process failed for other reasons (e.g. insufficient permissions). """ -function kill(p::Process, signum::Integer) +function kill(p::Process, signum::Integer=SIGTERM) iolock_begin() if process_running(p) @assert p.handle != C_NULL @@ -558,9 +558,8 @@ function kill(p::Process, signum::Integer) iolock_end() nothing end -kill(ps::Vector{Process}) = foreach(kill, ps) -kill(ps::ProcessChain) = foreach(kill, ps.processes) -kill(p::Process) = kill(p, SIGTERM) +kill(ps::Vector{Process}, signum::Integer=SIGTERM) = for p in ps; kill(p, signum); end +kill(ps::ProcessChain, signum::Integer=SIGTERM) = kill(ps.processes, signum) """ getpid(process) -> Int32 diff --git a/contrib/windows/build-installer.iss b/contrib/windows/build-installer.iss index 4e38276d0a038..63eca3e4b56f4 100644 --- a/contrib/windows/build-installer.iss +++ b/contrib/windows/build-installer.iss @@ -74,6 +74,9 @@ UninstallDisplayName={#AppNameLong} UninstallDisplayIcon={app}\{#AppMainExeName} UninstallFilesDir={app}\uninstall +#ifdef Sign +SignTool=mysigntool +#endif [Languages] Name: "english"; MessagesFile: "compiler:Default.isl" diff --git a/deps/Versions.make b/deps/Versions.make index be2303484ba7c..f9e474f287f34 100644 --- a/deps/Versions.make +++ b/deps/Versions.make @@ -30,7 +30,7 @@ CURL_BB_REL = 1 LIBGIT2_VER = 0.28.2 LIBGIT2_BB_REL = 1 LIBUV_VER = 1.29.1 -LIBUV_BB_REL = 0 +LIBUV_BB_REL = 9 OBJCONV_VER = 2.49.0 OBJCONV_BB_REL = 0 ZLIB_VER = 1.2.11 diff --git a/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-gnu.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-gnu.tar.gz/md5 new file mode 100644 index 0000000000000..2955de37e027f --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-gnu.tar.gz/md5 @@ -0,0 +1 @@ +2bd32030f78a45b6195999407f510aa5 diff --git a/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-gnu.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-gnu.tar.gz/sha512 new file mode 100644 index 0000000000000..bef5e4d5d86a6 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-gnu.tar.gz/sha512 @@ -0,0 +1 @@ +1ccc3ffa4f5473e74f4ce342423bddc268a6f70d9743aa2f11124163db27f3e56711e85555895e2ff506a025f9dfbfec686ff457e949aa45d86ffef5ec6c2037 diff --git a/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-musl.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-musl.tar.gz/md5 new file mode 100644 index 0000000000000..d7a9dce808eac --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-musl.tar.gz/md5 @@ -0,0 +1 @@ +c3b6f24843c4be53fa72d995a9296450 diff --git a/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-musl.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-musl.tar.gz/sha512 new file mode 100644 index 0000000000000..713eb1f2f7812 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.aarch64-linux-musl.tar.gz/sha512 @@ -0,0 +1 @@ +1659c0fa80c162411c284bcf85dfac2d6a699bf2b09bbfe7868ebade0c5e424e3bda416b345cc101f946dbbbd7fea677318d733ad12bf3bb6ba5555accbe8d45 diff --git a/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-gnueabihf.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-gnueabihf.tar.gz/md5 new file mode 100644 index 0000000000000..de164951f7fa3 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-gnueabihf.tar.gz/md5 @@ -0,0 +1 @@ +71b09d3e80394d283756348854a65017 diff --git a/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-gnueabihf.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-gnueabihf.tar.gz/sha512 new file mode 100644 index 0000000000000..71ca326cc3196 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-gnueabihf.tar.gz/sha512 @@ -0,0 +1 @@ +b360975c81b574a9ed673c106ab0a520eac42e594b48933a9e0526a1e00462da524b86a516d860945e04fe395ddba31acebe1ed8609aa65ccc7eea784ef0fb68 diff --git a/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-musleabihf.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-musleabihf.tar.gz/md5 new file mode 100644 index 0000000000000..4c68299162b86 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-musleabihf.tar.gz/md5 @@ -0,0 +1 @@ +40bf3062a91d1c24122e71be0479ed94 diff --git a/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-musleabihf.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-musleabihf.tar.gz/sha512 new file mode 100644 index 0000000000000..3da89c2a0c7ee --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.armv7l-linux-musleabihf.tar.gz/sha512 @@ -0,0 +1 @@ +aa66897bd05f13daf682104b16329093f1234922c07d97e8134db0a1f499d17c8716704e4c8b21a00d148afa920f4fa1d2fd884a545c973b8a6734d7aada5fc6 diff --git a/deps/checksums/LibUV.v2.0.0-9.i686-linux-gnu.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.i686-linux-gnu.tar.gz/md5 new file mode 100644 index 0000000000000..b176f993aa66f --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.i686-linux-gnu.tar.gz/md5 @@ -0,0 +1 @@ +11477f56cffc636d48245ad623eea1c7 diff --git a/deps/checksums/LibUV.v2.0.0-9.i686-linux-gnu.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.i686-linux-gnu.tar.gz/sha512 new file mode 100644 index 0000000000000..54542c43f6e91 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.i686-linux-gnu.tar.gz/sha512 @@ -0,0 +1 @@ +da82d92db2ba2bf0c0b52928390bbf0ee5a8212883fd23a772c7735c7f427ca229229b75e8d7ff8964d42a2ab0614f1adfd156e0ef067b682931e0236fe7ef7b diff --git a/deps/checksums/LibUV.v2.0.0-9.i686-linux-musl.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.i686-linux-musl.tar.gz/md5 new file mode 100644 index 0000000000000..609ecb4e59ebd --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.i686-linux-musl.tar.gz/md5 @@ -0,0 +1 @@ +739be10627a93a419f0a9f193232931c diff --git a/deps/checksums/LibUV.v2.0.0-9.i686-linux-musl.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.i686-linux-musl.tar.gz/sha512 new file mode 100644 index 0000000000000..c49cb60652102 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.i686-linux-musl.tar.gz/sha512 @@ -0,0 +1 @@ +8abf48cf61ef7dd1a7d619bb062ba919d97d9d68f2e3d04dd7c4fdfacf01e5bcc864ecd6fe5e8782ba72822764883583d60c5164c96fe030a979d3436278f2bd diff --git a/deps/checksums/LibUV.v2.0.0-9.i686-w64-mingw32.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.i686-w64-mingw32.tar.gz/md5 new file mode 100644 index 0000000000000..d0603d4804576 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.i686-w64-mingw32.tar.gz/md5 @@ -0,0 +1 @@ +3e27de72945d3eda858798b8faaaa2f9 diff --git a/deps/checksums/LibUV.v2.0.0-9.i686-w64-mingw32.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.i686-w64-mingw32.tar.gz/sha512 new file mode 100644 index 0000000000000..47c2c429b9634 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.i686-w64-mingw32.tar.gz/sha512 @@ -0,0 +1 @@ +c0dd0c3dea0f9e2b7319f46733685a5ac1ecb6423886e669aeb14e1bc8aae4017ad994b8802dcc9f57cceaebf138f3f4328b97d2f0a64c317243ce3fe282dd52 diff --git a/deps/checksums/LibUV.v2.0.0-9.powerpc64le-linux-gnu.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.powerpc64le-linux-gnu.tar.gz/md5 new file mode 100644 index 0000000000000..e6a0484d73886 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.powerpc64le-linux-gnu.tar.gz/md5 @@ -0,0 +1 @@ +4231a4b2c1adb333f2ed603ad22b92c5 diff --git a/deps/checksums/LibUV.v2.0.0-9.powerpc64le-linux-gnu.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.powerpc64le-linux-gnu.tar.gz/sha512 new file mode 100644 index 0000000000000..47c6e2b165c24 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.powerpc64le-linux-gnu.tar.gz/sha512 @@ -0,0 +1 @@ +aef5f19dc6b1061f1ebd205e4663779498e0094435dc875e4ebca3dda67b0c11d5cb8eb88a20f9baaf82d59a1e83a5fedcdb7a6be9255d738aca1981656523bf diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-apple-darwin14.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.x86_64-apple-darwin14.tar.gz/md5 new file mode 100644 index 0000000000000..949a55c685a58 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-apple-darwin14.tar.gz/md5 @@ -0,0 +1 @@ +ac719bf85a3619e9d238d0efb1291ef0 diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-apple-darwin14.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.x86_64-apple-darwin14.tar.gz/sha512 new file mode 100644 index 0000000000000..59b8e810d7aca --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-apple-darwin14.tar.gz/sha512 @@ -0,0 +1 @@ +753e640aeca665b6e35deed35860d8e3f3076b9c1cfae382d029827e138bc3ff65a90bbee2f828a8371ac15bca29469c1720b3c1b9c29f56d8b402790c7608e8 diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-gnu.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-gnu.tar.gz/md5 new file mode 100644 index 0000000000000..a86d696b18338 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-gnu.tar.gz/md5 @@ -0,0 +1 @@ +446e2ffac64e64b24c47d17d03cdc3a3 diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-gnu.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-gnu.tar.gz/sha512 new file mode 100644 index 0000000000000..aebd451c8e4d4 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-gnu.tar.gz/sha512 @@ -0,0 +1 @@ +185ec913e17aa00b866c1c7f2b4645c80ab8ff067ee1c176cc33c75e6161450994a795feeb46ca0bcbb80bbc727de8a4532ddee5b658fdc9bbf10b1676a54026 diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-musl.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-musl.tar.gz/md5 new file mode 100644 index 0000000000000..2c40c1f925383 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-musl.tar.gz/md5 @@ -0,0 +1 @@ +0f7913302512ad1002ce77b2c619a4c8 diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-musl.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-musl.tar.gz/sha512 new file mode 100644 index 0000000000000..95213b7850a17 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-linux-musl.tar.gz/sha512 @@ -0,0 +1 @@ +c8cc6d5a27bc21f28a1cb7e75b40947e9870ace136e50040fefe067c30398f3a1655ce28fa43b28e09ff743d5f1bbce061dff2f47ab67ca81cd3d1e6f360317a diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-unknown-freebsd11.1.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.x86_64-unknown-freebsd11.1.tar.gz/md5 new file mode 100644 index 0000000000000..20a55c8a252c3 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-unknown-freebsd11.1.tar.gz/md5 @@ -0,0 +1 @@ +2fd4a5b814194d588186485b8cc5b73d diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-unknown-freebsd11.1.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.x86_64-unknown-freebsd11.1.tar.gz/sha512 new file mode 100644 index 0000000000000..900f3d55128b7 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-unknown-freebsd11.1.tar.gz/sha512 @@ -0,0 +1 @@ +4b72488b5afa6dcc64763155489a4aad0f438266bdc039c03ed8835cf14f6159c5151751cdbc61cdfb54253bf458b2af4a7d3d8cb6ad70ffb154df0082c2149b diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-w64-mingw32.tar.gz/md5 b/deps/checksums/LibUV.v2.0.0-9.x86_64-w64-mingw32.tar.gz/md5 new file mode 100644 index 0000000000000..aec56e6f81632 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-w64-mingw32.tar.gz/md5 @@ -0,0 +1 @@ +bab85c5eae8370f4058e06b6779c7dc7 diff --git a/deps/checksums/LibUV.v2.0.0-9.x86_64-w64-mingw32.tar.gz/sha512 b/deps/checksums/LibUV.v2.0.0-9.x86_64-w64-mingw32.tar.gz/sha512 new file mode 100644 index 0000000000000..9ba2115db44b0 --- /dev/null +++ b/deps/checksums/LibUV.v2.0.0-9.x86_64-w64-mingw32.tar.gz/sha512 @@ -0,0 +1 @@ +78f42e12f99a9dddfec221c03e43fb76ac0ab664b6aec7f3f1a5f55fbc72c9cf03e2e7f99a4bb22b9a0e568cd448d52f5b91f22cf4aff6a3a3e5ddc6e5c963b7 diff --git a/deps/checksums/Pkg-0c3ccb119efd8b3182c114599abe78bc4f89dd4c.tar.gz/md5 b/deps/checksums/Pkg-0c3ccb119efd8b3182c114599abe78bc4f89dd4c.tar.gz/md5 new file mode 100644 index 0000000000000..40286f6e648d9 --- /dev/null +++ b/deps/checksums/Pkg-0c3ccb119efd8b3182c114599abe78bc4f89dd4c.tar.gz/md5 @@ -0,0 +1 @@ +d6a6ee3e1575e3b0cf64adf6d2c7c28d diff --git a/deps/checksums/Pkg-0c3ccb119efd8b3182c114599abe78bc4f89dd4c.tar.gz/sha512 b/deps/checksums/Pkg-0c3ccb119efd8b3182c114599abe78bc4f89dd4c.tar.gz/sha512 new file mode 100644 index 0000000000000..1321023fb170b --- /dev/null +++ b/deps/checksums/Pkg-0c3ccb119efd8b3182c114599abe78bc4f89dd4c.tar.gz/sha512 @@ -0,0 +1 @@ +3aa17ba7f7a9b76be7a8ddcebfa55c2eb62961c9dd3b4d06551a85fb58cacfcd53a40d2c2dd168f25f8ef5630fb01b3a41537dc9e4b4d54b409fe7d52a748cfa diff --git a/deps/checksums/libuv-1fcc6d66f9df74189c74d3d390f02202bb7db953.tar.gz/md5 b/deps/checksums/libuv-1fcc6d66f9df74189c74d3d390f02202bb7db953.tar.gz/md5 new file mode 100644 index 0000000000000..ce4beff6c0bc9 --- /dev/null +++ b/deps/checksums/libuv-1fcc6d66f9df74189c74d3d390f02202bb7db953.tar.gz/md5 @@ -0,0 +1 @@ +9388aa4d36915724f1de4b20b205ed35 diff --git a/deps/checksums/libuv-1fcc6d66f9df74189c74d3d390f02202bb7db953.tar.gz/sha512 b/deps/checksums/libuv-1fcc6d66f9df74189c74d3d390f02202bb7db953.tar.gz/sha512 new file mode 100644 index 0000000000000..863943d5d26bb --- /dev/null +++ b/deps/checksums/libuv-1fcc6d66f9df74189c74d3d390f02202bb7db953.tar.gz/sha512 @@ -0,0 +1 @@ +70b0c738a60b4e476750b7de8d120e735359e2c36fcb3a8a38628a0ae326210ed3b15d793dfd5443d7aa5603e83e7d99f567aa4c1696846d950df9f83648669c diff --git a/deps/libuv.mk b/deps/libuv.mk index 6e07cea10953e..09523d034d9fd 100644 --- a/deps/libuv.mk +++ b/deps/libuv.mk @@ -64,8 +64,8 @@ fastcheck-libuv: #none check-libuv: $(LIBUV_BUILDDIR)/build-checked else # USE_BINARYBUILDER_LIBUV -LIBUV_BB_URL_BASE := https://github.com/JuliaPackaging/Yggdrasil/releases/download/LibUV-v2+$(LIBUV_VER)-julia+$(LIBUV_BB_REL) -LIBUV_BB_NAME := LibUV.v2.0.0+$(LIBUV_VER)-julia +LIBUV_BB_URL_BASE := https://github.com/JuliaBinaryWrappers/LibUV_jll.jl/releases/download/LibUV-v2.0.0+$(LIBUV_BB_REL) +LIBUV_BB_NAME := LibUV.v2.0.0 $(eval $(call bb-install,libuv,LIBUV,false)) endif diff --git a/deps/libuv.version b/deps/libuv.version index cb4d6b8cad0b3..b410875f69ac9 100644 --- a/deps/libuv.version +++ b/deps/libuv.version @@ -1,2 +1,2 @@ LIBUV_BRANCH=julia-uv2-1.29.1 -LIBUV_SHA1=35b1504507a7a4168caae3d78db54d1121b121e1 +LIBUV_SHA1=1fcc6d66f9df74189c74d3d390f02202bb7db953 diff --git a/doc/src/devdocs/locks.md b/doc/src/devdocs/locks.md index 8fa3fd4b6dc47..7591f0df2d356 100644 --- a/doc/src/devdocs/locks.md +++ b/doc/src/devdocs/locks.md @@ -27,6 +27,7 @@ The following are definitely leaf locks (level 1), and must not try to acquire a > * pagealloc > * gc_perm_lock > * flisp +> * jl_in_stackwalk (Win32) > > > flisp itself is already threadsafe, this lock only protects the `jl_ast_context_list_t` pool diff --git a/src/datatype.c b/src/datatype.c index c58c9173bd669..6f9358a2016ec 100644 --- a/src/datatype.c +++ b/src/datatype.c @@ -95,6 +95,9 @@ jl_datatype_t *jl_new_uninitialized_datatype(void) t->zeroinit = 0; t->isinlinealloc = 0; t->has_concrete_subtype = 1; + t->name = NULL; + t->super = NULL; + t->parameters = NULL; t->layout = NULL; t->names = NULL; t->types = NULL; diff --git a/src/debuginfo.cpp b/src/debuginfo.cpp index 1f81ee114a52f..0c53092d3e3a2 100644 --- a/src/debuginfo.cpp +++ b/src/debuginfo.cpp @@ -23,8 +23,6 @@ using namespace llvm; -using llvm_file_magic = file_magic; - #include "julia.h" #include "julia_internal.h" #include "debuginfo.h" @@ -55,11 +53,41 @@ typedef object::SymbolRef SymRef; // and cannot have any interaction with the julia runtime static uv_rwlock_t threadsafe; -extern "C" void jl_init_debuginfo() +extern "C" void jl_init_debuginfo(void) { uv_rwlock_init(&threadsafe); } +extern "C" void jl_lock_profile(void) +{ + uv_rwlock_rdlock(&threadsafe); +} + +extern "C" void jl_unlock_profile(void) +{ + uv_rwlock_rdunlock(&threadsafe); +} + +// some actions aren't signal (especially profiler) safe so we acquire a lock +// around them to establish a mutual exclusion with unwinding from a signal +template +static void jl_profile_atomic(T f) +{ + uv_rwlock_wrlock(&threadsafe); +#ifndef _OS_WINDOWS_ + sigset_t sset; + sigset_t oset; + sigfillset(&sset); + pthread_sigmask(SIG_BLOCK, &sset, &oset); +#endif + f(); +#ifndef _OS_WINDOWS_ + pthread_sigmask(SIG_SETMASK, &oset, NULL); +#endif + uv_rwlock_wrunlock(&threadsafe); +} + + // --- storing and accessing source location metadata --- struct ObjectInfo { @@ -111,8 +139,7 @@ static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnnam mod_size = Size; #endif if (0) { - assert(!jl_in_stackwalk); - jl_in_stackwalk = 1; + JL_LOCK_NOGC(&jl_in_stackwalk); if (mod_size && !SymLoadModuleEx(GetCurrentProcess(), NULL, NULL, NULL, (DWORD64)Section, mod_size, NULL, SLMFLAG_VIRTUAL)) { static int warned = 0; if (!warned) { @@ -132,16 +159,18 @@ static void create_PRUNTIME_FUNCTION(uint8_t *Code, size_t Size, StringRef fnnam jl_printf(JL_STDERR, "WARNING: failed to insert function name %s into debug info: %lu\n", name, GetLastError()); } } - jl_in_stackwalk = 0; + JL_UNLOCK_NOGC(&jl_in_stackwalk); } #if defined(_CPU_X86_64_) - if (!RtlAddFunctionTable(tbl, 1, (DWORD64)Section)) { - static int warned = 0; - if (!warned) { - jl_printf(JL_STDERR, "WARNING: failed to insert function stack unwind info: %lu\n", GetLastError()); - warned = 1; + jl_profile_atomic([&]() { + if (!RtlAddFunctionTable(tbl, 1, (DWORD64)Section)) { + static int warned = 0; + if (!warned) { + jl_printf(JL_STDERR, "WARNING: failed to insert function stack unwind info: %lu\n", GetLastError()); + warned = 1; + } } - } + }); #endif } #endif @@ -279,7 +308,9 @@ class JuliaJITEventListener: public JITEventListener di->u.rti.name_ptr = 0; di->u.rti.table_data = arm_exidx_addr; di->u.rti.table_len = arm_exidx_len; - _U_dyn_register(di); + jl_profile_atomic([&]() { + _U_dyn_register(di); + }); break; } #endif @@ -405,20 +436,20 @@ class JuliaJITEventListener: public JITEventListener codeinst = codeinst_it->second; codeinst_in_flight.erase(codeinst_it); } - uv_rwlock_wrlock(&threadsafe); - if (codeinst) - linfomap[Addr] = std::make_pair(Size, codeinst->def); - if (first) { - ObjectInfo tmp = {&debugObj, - (size_t)SectionSize, - (ptrdiff_t)(SectionAddr - SectionLoadAddr), - *Section, - nullptr, - }; - objectmap[SectionLoadAddr] = tmp; - first = false; - } - uv_rwlock_wrunlock(&threadsafe); + jl_profile_atomic([&]() { + if (codeinst) + linfomap[Addr] = std::make_pair(Size, codeinst->def); + if (first) { + ObjectInfo tmp = {&debugObj, + (size_t)SectionSize, + (ptrdiff_t)(SectionAddr - SectionLoadAddr), + *Section, + nullptr, + }; + objectmap[SectionLoadAddr] = tmp; + first = false; + } + }); } jl_gc_safe_leave(ptls, gc_state); } @@ -432,14 +463,6 @@ class JuliaJITEventListener: public JITEventListener uv_rwlock_rdlock(&threadsafe); return objectmap; } - - Optional*> trygetObjectMap() - { - if (0 == uv_rwlock_tryrdlock(&threadsafe)) { - return &objectmap; - } - return {}; - } }; JL_DLLEXPORT void ORCNotifyObjectEmitted(JITEventListener *Listener, @@ -483,7 +506,7 @@ static std::pair jl_demangle(const char *name) JL_NOTSAFEPOINT } static JuliaJITEventListener *jl_jit_events; -JITEventListener *CreateJuliaJITEventListener() +JITEventListener *CreateJuliaJITEventListener(void) { jl_jit_events = new JuliaJITEventListener(); return jl_jit_events; @@ -723,7 +746,7 @@ openDebugInfo(StringRef debuginfopath, const debug_link_info &info) auto error_splitobj = object::ObjectFile::createObjectFile( SplitFile.get().get()->getMemBufferRef(), - llvm_file_magic::unknown); + file_magic::unknown); if (!error_splitobj) { return error_splitobj.takeError(); } @@ -832,12 +855,12 @@ static void get_function_name_and_base(llvm::object::SectionRef Section, size_t PSYMBOL_INFO pSymbol = (PSYMBOL_INFO)frame_info_func; pSymbol->SizeOfStruct = sizeof(SYMBOL_INFO); pSymbol->MaxNameLen = MAX_SYM_NAME; - jl_in_stackwalk = 1; + JL_LOCK_NOGC(&jl_in_stackwalk); if (SymFromAddr(GetCurrentProcess(), dwAddress, &dwDisplacement64, pSymbol)) { // errors are ignored jl_copy_str(name, pSymbol->Name); } - jl_in_stackwalk = 0; + JL_UNLOCK_NOGC(&jl_in_stackwalk); } #endif } @@ -874,7 +897,7 @@ static objfileentry_t &find_object_file(uint64_t fbase, StringRef fname) JL_NOTS std::unique_ptr membuf = MemoryBuffer::getMemBuffer( StringRef((const char *)fbase, msize), "", false); auto origerrorobj = llvm::object::ObjectFile::createObjectFile( - membuf->getMemBufferRef(), llvm_file_magic::unknown); + membuf->getMemBufferRef(), file_magic::unknown); if (!origerrorobj) return entry; @@ -1053,6 +1076,7 @@ static object::SectionRef getModuleSectionForAddress(const object::ObjectFile *o extern "C" void jl_refresh_dbg_module_list(void); + bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t *slide, llvm::DIContext **context, bool onlySysImg, bool *isSysImg, void **saddr, char **name, char **filename) JL_NOTSAFEPOINT { @@ -1077,11 +1101,12 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * #ifdef _OS_WINDOWS_ IMAGEHLP_MODULE64 ModuleInfo; ModuleInfo.SizeOfStruct = sizeof(IMAGEHLP_MODULE64); + JL_LOCK_NOGC(&jl_in_stackwalk); jl_refresh_dbg_module_list(); - jl_in_stackwalk = 1; bool isvalid = SymGetModuleInfo64(GetCurrentProcess(), (DWORD64)pointer, &ModuleInfo); - jl_in_stackwalk = 0; - if (!isvalid) return false; + JL_UNLOCK_NOGC(&jl_in_stackwalk); + if (!isvalid) + return false; StringRef fname = ModuleInfo.LoadedImageName; if (fname.empty()) // empirically, LoadedImageName might be missing @@ -1147,16 +1172,12 @@ bool jl_dylib_DI_for_fptr(size_t pointer, object::SectionRef *Section, int64_t * static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skipC, int noInline) JL_NOTSAFEPOINT { // This function is not allowed to reference any TLS variables if noInline - // since it can be called from an unmanaged thread on OSX. + // since it can be called from an unmanaged thread (the segfault handler) jl_frame_t *frame0 = *frames; #ifdef _OS_WINDOWS_ static IMAGEHLP_LINE64 frame_info_line; DWORD dwDisplacement = 0; - if (jl_in_stackwalk) { - frame0->fromC = 1; - return 1; - } - jl_in_stackwalk = 1; + JL_LOCK_NOGC(&jl_in_stackwalk); DWORD64 dwAddress = pointer; frame_info_line.SizeOfStruct = sizeof(IMAGEHLP_LINE64); if (SymGetLineFromAddr64(GetCurrentProcess(), dwAddress, &dwDisplacement, &frame_info_line)) { @@ -1166,7 +1187,7 @@ static int jl_getDylibFunctionInfo(jl_frame_t **frames, size_t pointer, int skip jl_copy_str(&frame0->file_name, frame_info_line.FileName); frame0->line = frame_info_line.LineNumber; } - jl_in_stackwalk = 0; + JL_UNLOCK_NOGC(&jl_in_stackwalk); #endif object::SectionRef Section; llvm::DIContext *context = NULL; @@ -1295,11 +1316,13 @@ void register_eh_frames(uint8_t *Addr, size_t Size) // See http://lists.cs.uiuc.edu/pipermail/llvmdev/2013-April/061768.html processFDEs((char*)Addr, Size, [](const char *Entry) { if (!libc_register_frame) { - libc_register_frame = (void(*)(void*))dlsym(RTLD_NEXT,"__register_frame"); + libc_register_frame = (void(*)(void*))dlsym(RTLD_NEXT, "__register_frame"); } assert(libc_register_frame); - libc_register_frame(const_cast(Entry)); - __register_frame(const_cast(Entry)); + jl_profile_atomic([&]() { + libc_register_frame(const_cast(Entry)); + __register_frame(const_cast(Entry)); + }); }); } @@ -1307,16 +1330,19 @@ void deregister_eh_frames(uint8_t *Addr, size_t Size) { processFDEs((char*)Addr, Size, [](const char *Entry) { if (!libc_deregister_frame) { - libc_deregister_frame = (void(*)(void*))dlsym(RTLD_NEXT,"__deregister_frame"); + libc_deregister_frame = (void(*)(void*))dlsym(RTLD_NEXT, "__deregister_frame"); } assert(libc_deregister_frame); - libc_deregister_frame(const_cast(Entry)); - __deregister_frame(const_cast(Entry)); + jl_profile_atomic([&]() { + libc_deregister_frame(const_cast(Entry)); + __deregister_frame(const_cast(Entry)); + }); }); } #elif defined(_OS_LINUX_) && \ - defined(JL_UNW_HAS_FORMAT_IP) && !defined(_CPU_ARM_) + defined(JL_UNW_HAS_FORMAT_IP) && \ + !defined(_CPU_ARM_) // ARM does not have/use .eh_frame, so we handle this elsewhere #include struct unw_table_entry @@ -1502,7 +1528,9 @@ static DW_EH_PE parseCIE(const uint8_t *Addr, const uint8_t *End) void register_eh_frames(uint8_t *Addr, size_t Size) { // System unwinder - __register_frame(Addr); + jl_profile_atomic([&]() { + __register_frame(Addr); + }); // Our unwinder unw_dyn_info_t *di = new unw_dyn_info_t; // In a shared library, this is set to the address of the PLT. @@ -1613,7 +1641,7 @@ void register_eh_frames(uint8_t *Addr, size_t Size) start_ips[cur_entry] = start; cur_entry++; }); - for (size_t i = 0;i < nentries;i++) { + for (size_t i = 0; i < nentries; i++) { table[i].start_ip_offset = safe_trunc((intptr_t)start_ips[i] - (intptr_t)start_ip); } @@ -1624,27 +1652,21 @@ void register_eh_frames(uint8_t *Addr, size_t Size) di->start_ip = start_ip; di->end_ip = end_ip; - _U_dyn_register(di); + jl_profile_atomic([&]() { + _U_dyn_register(di); + }); } void deregister_eh_frames(uint8_t *Addr, size_t Size) { - __deregister_frame(Addr); - // Deregistering with our unwinder requires a lookup table to find the - // the allocated entry above (or we could look in libunwind's internal + jl_profile_atomic([&]() { + __deregister_frame(Addr); + }); + // Deregistering with our unwinder (_U_dyn_cancel) requires a lookup table + // to find the allocated entry above (or looking into libunwind's internal // data structures). } -#elif defined(_CPU_ARM_) - -void register_eh_frames(uint8_t *Addr, size_t Size) -{ -} - -void deregister_eh_frames(uint8_t *Addr, size_t Size) -{ -} - #else void register_eh_frames(uint8_t *Addr, size_t Size) @@ -1670,22 +1692,3 @@ uint64_t jl_getUnwindInfo(uint64_t dwAddr) uv_rwlock_rdunlock(&threadsafe); return ipstart; } - -extern "C" -uint64_t jl_trygetUnwindInfo(uint64_t dwAddr) -{ - // Might be called from unmanaged thread - Optional*> maybeobjmap = jl_jit_events->trygetObjectMap(); - if (maybeobjmap) { - std::map &objmap = **maybeobjmap; - std::map::iterator it = objmap.lower_bound(dwAddr); - uint64_t ipstart = 0; // ip of the start of the section (if found) - if (it != objmap.end() && dwAddr < it->first + it->second.SectionSize) { - ipstart = (uint64_t)(uintptr_t)(*it).first; - } - uv_rwlock_rdunlock(&threadsafe); - return ipstart; - } - return 0; -} - diff --git a/src/dlload.c b/src/dlload.c index 72213c4362311..d0bd2c866c080 100644 --- a/src/dlload.c +++ b/src/dlload.c @@ -25,7 +25,7 @@ extern "C" { static char const *const extensions[] = { "", ".dylib" }; #elif defined(_OS_WINDOWS_) static char const *const extensions[] = { "", ".dll" }; -extern int needsSymRefreshModuleList; +extern volatile int needsSymRefreshModuleList; #else static char const *const extensions[] = { "", ".so" }; #endif @@ -94,12 +94,14 @@ static void win32_formatmessage(DWORD code, char *reason, int len) JL_DLLEXPORT void *jl_dlopen(const char *filename, unsigned flags) { #if defined(_OS_WINDOWS_) - needsSymRefreshModuleList = 1; size_t len = MultiByteToWideChar(CP_UTF8, 0, filename, -1, NULL, 0); if (!len) return NULL; WCHAR *wfilename = (WCHAR*)alloca(len * sizeof(WCHAR)); if (!MultiByteToWideChar(CP_UTF8, 0, filename, -1, wfilename, len)) return NULL; - return LoadLibraryExW(wfilename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + HANDLE lib = LoadLibraryExW(wfilename, NULL, LOAD_WITH_ALTERED_SEARCH_PATH); + if (lib) + needsSymRefreshModuleList = 1; + return lib; #else dlerror(); /* Reset error status. */ return dlopen(filename, diff --git a/src/dump.c b/src/dump.c index f9c0211c61b86..ac81b1a31a2b4 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1434,6 +1434,7 @@ static int64_t write_dependency_list(ios_t *s, jl_array_t **udepsp, jl_array_t * static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_value_t **loc) JL_GC_DISABLED { + assert(pos == backref_list.len - 1 && "nothing should have been deserialized since assigning pos"); int tag = read_uint8(s->s); if (tag == 6 || tag == 7) { jl_typename_t *name = (jl_typename_t*)jl_deserialize_value(s, NULL); @@ -1449,19 +1450,18 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v backref_list.items[pos] = dtv; return dtv; } - size_t size = read_int32(s->s); - uint8_t flags = read_uint8(s->s); - uint8_t memflags = read_uint8(s->s); - jl_datatype_t *dt = NULL; - if (tag == 0 || tag == 5 || tag == 10 || tag == 11 || tag == 12) - dt = jl_new_uninitialized_datatype(); - else { + if (!(tag == 0 || tag == 5 || tag == 10 || tag == 11 || tag == 12)) { assert(0 && "corrupt deserialization state"); abort(); } assert(s->method == NULL && s->mode != MODE_IR && "no new data-types expected during MODE_IR"); - assert(pos == backref_list.len - 1 && "nothing should have been deserialized since assigning pos"); + jl_datatype_t *dt = jl_new_uninitialized_datatype(); backref_list.items[pos] = dt; + if (loc != NULL && loc != HT_NOTFOUND) + *loc = (jl_value_t*)dt; + size_t size = read_int32(s->s); + uint8_t flags = read_uint8(s->s); + uint8_t memflags = read_uint8(s->s); dt->size = size; dt->abstract = flags & 1; dt->mutabl = (flags >> 1) & 1; @@ -1474,11 +1474,6 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v dt->zeroinit = (memflags >> 4) & 1; dt->isinlinealloc = (memflags >> 5) & 1; dt->has_concrete_subtype = (memflags >> 6) & 1; - dt->types = NULL; - dt->parameters = NULL; - dt->name = NULL; - dt->super = NULL; - dt->layout = NULL; if (!dt->abstract) dt->ninitialized = read_uint16(s->s); else @@ -1542,7 +1537,7 @@ static jl_value_t *jl_deserialize_datatype(jl_serializer_state *s, int pos, jl_v return (jl_value_t*)dt; } -static jl_value_t *jl_deserialize_value_svec(jl_serializer_state *s, uint8_t tag) JL_GC_DISABLED +static jl_value_t *jl_deserialize_value_svec(jl_serializer_state *s, uint8_t tag, jl_value_t **loc) JL_GC_DISABLED { int usetable = (s->mode != MODE_IR); size_t i, len; @@ -1550,7 +1545,9 @@ static jl_value_t *jl_deserialize_value_svec(jl_serializer_state *s, uint8_t tag len = read_uint8(s->s); else len = read_int32(s->s); - jl_svec_t *sv = jl_alloc_svec_uninit(len); + jl_svec_t *sv = jl_alloc_svec(len); + if (loc != NULL) + *loc = (jl_value_t*)sv; if (usetable) arraylist_push(&backref_list, (jl_value_t*)sv); jl_value_t **data = jl_svec_data(sv); @@ -2115,7 +2112,7 @@ static jl_value_t *jl_deserialize_value(jl_serializer_state *s, jl_value_t **loc case TAG_LONG_METHODROOT: return jl_array_ptr_ref(s->method->roots, read_uint16(s->s)); case TAG_SVEC: JL_FALLTHROUGH; case TAG_LONG_SVEC: - return jl_deserialize_value_svec(s, tag); + return jl_deserialize_value_svec(s, tag, loc); case TAG_COMMONSYM: return deser_symbols[read_uint8(s->s)]; case TAG_SYMBOL: JL_FALLTHROUGH; case TAG_LONG_SYMBOL: diff --git a/src/init.c b/src/init.c index 86958049b10db..e4ffc79a9b8f8 100644 --- a/src/init.c +++ b/src/init.c @@ -661,6 +661,7 @@ void _julia_init(JL_IMAGE_SEARCH rel) #endif jl_winsock_handle = jl_dlopen("ws2_32.dll", 0); jl_exe_handle = GetModuleHandleA(NULL); + JL_MUTEX_INIT(&jl_in_stackwalk); SymSetOptions(SYMOPT_UNDNAME | SYMOPT_DEFERRED_LOADS | SYMOPT_LOAD_LINES); if (!SymInitialize(GetCurrentProcess(), NULL, 1)) { jl_printf(JL_STDERR, "WARNING: failed to initialize stack walk info\n"); diff --git a/src/julia_internal.h b/src/julia_internal.h index 67df97e4c8212..b917569b080a9 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -758,7 +758,6 @@ typedef struct { // Might be called from unmanaged thread uint64_t jl_getUnwindInfo(uint64_t dwBase); -uint64_t jl_trygetUnwindInfo(uint64_t dwBase); #ifdef _OS_WINDOWS_ #include JL_DLLEXPORT EXCEPTION_DISPOSITION __julia_personality( @@ -773,7 +772,7 @@ typedef struct { CONTEXT context; } bt_cursor_t; #endif -extern volatile int jl_in_stackwalk; +extern jl_mutex_t jl_in_stackwalk; #elif !defined(JL_DISABLE_LIBUNWIND) // This gives unwind only local unwinding options ==> faster code # define UNW_LOCAL_ONLY @@ -795,7 +794,7 @@ size_t rec_backtrace(jl_bt_element_t *bt_data, size_t maxsize, int skip) JL_NOTS // Record backtrace from a signal handler. `ctx` is the context of the code // which was asynchronously interrupted. size_t rec_backtrace_ctx(jl_bt_element_t *bt_data, size_t maxsize, bt_context_t *ctx, - jl_gcframe_t *pgcstack, int lockless) JL_NOTSAFEPOINT; + jl_gcframe_t *pgcstack) JL_NOTSAFEPOINT; #ifdef LIBOSXUNWIND size_t rec_backtrace_ctx_dwarf(jl_bt_element_t *bt_data, size_t maxsize, bt_context_t *ctx, jl_gcframe_t *pgcstack) JL_NOTSAFEPOINT; #endif diff --git a/src/locks.h b/src/locks.h index 0f011c5e0a7f0..6365eb681ca30 100644 --- a/src/locks.h +++ b/src/locks.h @@ -71,6 +71,7 @@ static inline void jl_lock_frame_pop(void) { jl_ptls_t ptls = jl_get_ptls_states(); if (__likely(ptls->current_task)) { + assert(ptls->current_task->locks.len > 0); ptls->current_task->locks.len--; } } diff --git a/src/rtutils.c b/src/rtutils.c index 731a34294940d..f9a8eb59a564e 100644 --- a/src/rtutils.c +++ b/src/rtutils.c @@ -610,6 +610,43 @@ JL_DLLEXPORT jl_value_t *jl_argument_datatype(jl_value_t *argt JL_PROPAGATES_ROO return (jl_value_t*)dt; } +static int is_globfunction(jl_value_t *v, jl_datatype_t *dv, jl_sym_t **globname_out) +{ + jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; + *globname_out = globname; + int globfunc = 0; + if (globname && !strchr(jl_symbol_name(globname), '#') && + !strchr(jl_symbol_name(globname), '@') && dv->name->module && + jl_binding_resolved_p(dv->name->module, globname)) { + jl_binding_t *b = jl_get_module_binding(dv->name->module, globname); + // The `||` makes this function work for both function instances and function types. + if (b && b->value && (b->value == v || jl_typeof(b->value) == v)) { + globfunc = 1; + } + } + return globfunc; +} + +static size_t jl_static_show_x_sym_escaped(JL_STREAM *out, jl_sym_t *name) JL_NOTSAFEPOINT +{ + size_t n = 0; + + char *sn = jl_symbol_name(name); + int hidden = 0; + if (!(jl_is_identifier(sn) || jl_is_operator(sn))) { + hidden = 1; + } + + if (hidden) { + n += jl_printf(out, "var\""); + } + n += jl_printf(out, "%s", sn); + if (hidden) { + n += jl_printf(out, "\""); + } + return n; +} + // `v` might be pointing to a field inlined in a structure therefore // `jl_typeof(v)` may not be the same with `vt` and only `vt` should be // used to determine the type of the value. @@ -678,48 +715,32 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, "UnionAll"); } else if (vt == jl_datatype_type) { + // typeof(v) == DataType, so v is a Type object. + // Types are printed as a fully qualified name, with parameters, e.g. + // `Base.Set{Int}`, and function types are printed as e.g. `typeof(Main.f)` jl_datatype_t *dv = (jl_datatype_t*)v; - jl_sym_t *globname = dv->name->mt != NULL ? dv->name->mt->name : NULL; - int globfunc = 0; - if (globname && !strchr(jl_symbol_name(globname), '#') && - !strchr(jl_symbol_name(globname), '@') && dv->name->module && - jl_binding_resolved_p(dv->name->module, globname)) { - jl_binding_t *b = jl_get_module_binding(dv->name->module, globname); - if (b && b->value && jl_typeof(b->value) == v) - globfunc = 1; - } + jl_sym_t *globname; + int globfunc = is_globfunction(v, dv, &globname); jl_sym_t *sym = globfunc ? globname : dv->name->name; char *sn = jl_symbol_name(sym); - int hidden = !globfunc && strchr(sn, '#'); - size_t i = 0; - int quote = 0; - if (hidden) { - n += jl_printf(out, "getfield("); - } - else if (globfunc) { + size_t quote = 0; + if (globfunc) { n += jl_printf(out, "typeof("); } if (jl_core_module && (dv->name->module != jl_core_module || !jl_module_exports_p(jl_core_module, sym))) { n += jl_static_show_x(out, (jl_value_t*)dv->name->module, depth); - if (!hidden) { - n += jl_printf(out, "."); - if (globfunc && !jl_id_start_char(u8_nextchar(sn, &i))) { - n += jl_printf(out, ":("); - quote = 1; - } + n += jl_printf(out, "."); + size_t i = 0; + if (globfunc && !jl_id_start_char(u8_nextchar(sn, &i))) { + n += jl_printf(out, ":("); + quote = 1; } } - if (hidden) { - n += jl_printf(out, ", Symbol(\""); - n += jl_printf(out, "%s", sn); - n += jl_printf(out, "\"))"); - } - else { - n += jl_printf(out, "%s", sn); - if (globfunc) { + n += jl_static_show_x_sym_escaped(out, sym); + if (globfunc) { + n += jl_printf(out, ")"); + if (quote) { n += jl_printf(out, ")"); - if (quote) - n += jl_printf(out, ")"); } } if (dv->parameters && (jl_value_t*)dv != dv->name->wrapper && @@ -835,7 +856,7 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_printf(out, ")"); n += jl_printf(out, "<:"); } - n += jl_printf(out, "%s", jl_symbol_name(var->name)); + n += jl_static_show_x_sym_escaped(out, var->name); if (showbounds && (ub != (jl_value_t*)jl_any_type || lb != jl_bottom_type)) { // show type-var upper bound if it is defined, or if we showed the lower bound int ua = jl_is_unionall(ub); @@ -981,7 +1002,38 @@ static size_t jl_static_show_x_(JL_STREAM *out, jl_value_t *v, jl_datatype_t *vt n += jl_static_show_x(out, *(jl_value_t**)v, depth); n += jl_printf(out, ")"); } + else if (jl_function_type && jl_isa(v, (jl_value_t*)jl_function_type)) { + // v is function instance (an instance of a Function type). + jl_datatype_t *dv = (jl_datatype_t*)vt; + jl_sym_t *sym = dv->name->mt->name; + char *sn = jl_symbol_name(sym); + + jl_sym_t *globname; + int globfunc = is_globfunction(v, dv, &globname); + int quote = 0; + if (jl_core_module && (dv->name->module != jl_core_module || !jl_module_exports_p(jl_core_module, sym))) { + n += jl_static_show_x(out, (jl_value_t*)dv->name->module, depth); + n += jl_printf(out, "."); + + size_t i = 0; + if (globfunc && !jl_id_start_char(u8_nextchar(sn, &i))) { + n += jl_printf(out, ":("); + quote = 1; + } + } + + n += jl_static_show_x_sym_escaped(out, sym); + + if (globfunc) { + if (quote) { + n += jl_printf(out, ")"); + } + } + } else if (jl_datatype_type && jl_is_datatype(vt)) { + // typeof(v) isa DataType, so v is an *instance of* a type that is a Datatype, + // meaning v is e.g. an instance of a struct. These are printed as a call to a + // type constructor, such as e.g. `Base.UnitRange{Int64}(start=1, stop=2)` int istuple = jl_is_tuple_type(vt), isnamedtuple = jl_is_namedtuple_type(vt); size_t tlen = jl_datatype_nfields(vt); if (isnamedtuple) { diff --git a/src/signal-handling.c b/src/signal-handling.c index 148b1598c2996..53ef1a56b861e 100644 --- a/src/signal-handling.c +++ b/src/signal-handling.c @@ -27,6 +27,8 @@ static const uint64_t GIGA = 1000000000ULL; // Timers to take samples at intervals JL_DLLEXPORT void jl_profile_stop_timer(void); JL_DLLEXPORT int jl_profile_start_timer(void); +void jl_lock_profile(void); +void jl_unlock_profile(void); static uint64_t jl_last_sigint_trigger = 0; static uint64_t jl_disable_sigint_time = 0; @@ -233,7 +235,7 @@ void jl_critical_error(int sig, bt_context_t *context, jl_bt_element_t *bt_data, if (context) { // Must avoid extended backtrace frames here unless we're sure bt_data // is properly rooted. - *bt_size = n = rec_backtrace_ctx(bt_data, JL_MAX_BT_SIZE, context, NULL, 1); + *bt_size = n = rec_backtrace_ctx(bt_data, JL_MAX_BT_SIZE, context, NULL); } for (i = 0; i < n; i += jl_bt_entry_size(bt_data + i)) { jl_print_bt_entry_codeloc(bt_data + i); diff --git a/src/signals-mach.c b/src/signals-mach.c index 280ce00236624..f523420cc8342 100644 --- a/src/signals-mach.c +++ b/src/signals-mach.c @@ -17,6 +17,18 @@ #include "julia_assert.h" +// private keymgr stuff +#define KEYMGR_GCC3_DW2_OBJ_LIST 302 +enum { + NM_ALLOW_RECURSION = 1, + NM_RECURSION_ILLEGAL = 2 +}; +extern void _keymgr_set_and_unlock_processwide_ptr(unsigned int key, void *ptr); +extern int _keymgr_unlock_processwide_ptr(unsigned int key); +extern void *_keymgr_get_and_lock_processwide_ptr(unsigned int key); +extern int _keymgr_get_and_lock_processwide_ptr_2(unsigned int key, void **result); +extern int _keymgr_set_lockmode_processwide_ptr(unsigned int key, unsigned int mode); + static void attach_exception_port(thread_port_t thread, int segv_only); // low 16 bits are the thread id, the next 8 bits are the original gc_state @@ -78,6 +90,17 @@ void *mach_segv_listener(void *arg) static void allocate_segv_handler() { + // ensure KEYMGR_GCC3_DW2_OBJ_LIST is initialized, as this requires malloc + // and thus can deadlock when used without first initializing it. + // Apple caused this problem in their libunwind in 10.9 (circa keymgr-28) + // when they removed this part of the code from keymgr. + // Much thanks to Apple for providing source code, or this would probably + // have simply remained unsolved forever on their platform. + // This is similar to just calling checkKeyMgrRegisteredFDEs + // (this is quite thread-unsafe) + if (_keymgr_set_lockmode_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST, NM_ALLOW_RECURSION)) + jl_error("_keymgr_set_lockmode_processwide_ptr failed"); + arraylist_new(&suspended_threads, jl_n_threads); pthread_t thread; pthread_attr_t attr; @@ -142,7 +165,7 @@ static void jl_throw_in_thread(int tid, mach_port_t thread, jl_value_t *exceptio if (!ptls2->safe_restore) { assert(exception); ptls2->bt_size = rec_backtrace_ctx(ptls2->bt_data, JL_MAX_BT_SIZE, - (bt_context_t*)&state, ptls2->pgcstack, 0); + (bt_context_t*)&state, ptls2->pgcstack); ptls2->sig_exception = exception; } jl_call_in_state(ptls2, &state, &jl_sig_throw); @@ -420,6 +443,9 @@ void *mach_profile_listener(void *arg) HANDLE_MACH_ERROR("mach_msg", ret); // sample each thread, round-robin style in reverse order // (so that thread zero gets notified last) + jl_lock_profile(); + void *unused = NULL; + int keymgr_locked = _keymgr_get_and_lock_processwide_ptr_2(KEYMGR_GCC3_DW2_OBJ_LIST, &unused) == 0; for (i = jl_n_threads; i-- > 0; ) { // if there is no space left, break early if (bt_size_cur >= bt_size_max - 1) @@ -448,7 +474,7 @@ void *mach_profile_listener(void *arg) if (forceDwarf == 0) { // Save the backtrace - bt_size_cur += rec_backtrace_ctx((jl_bt_element_t*)bt_data_prof + bt_size_cur, bt_size_max - bt_size_cur - 1, uc, NULL, 1); + bt_size_cur += rec_backtrace_ctx((jl_bt_element_t*)bt_data_prof + bt_size_cur, bt_size_max - bt_size_cur - 1, uc, NULL); } else if (forceDwarf == 1) { bt_size_cur += rec_backtrace_ctx_dwarf((jl_bt_element_t*)bt_data_prof + bt_size_cur, bt_size_max - bt_size_cur - 1, uc, NULL); @@ -464,14 +490,18 @@ void *mach_profile_listener(void *arg) // Mark the end of this block with 0 bt_data_prof[bt_size_cur++].uintptr = 0; - - // Reset the alarm - kern_return_t ret = clock_alarm(clk, TIME_RELATIVE, timerprof, profile_port); - HANDLE_MACH_ERROR("clock_alarm", ret) } // We're done! Resume the thread. jl_thread_resume(i, 0); } + if (keymgr_locked) + _keymgr_unlock_processwide_ptr(KEYMGR_GCC3_DW2_OBJ_LIST); + jl_unlock_profile(); + if (running) { + // Reset the alarm + kern_return_t ret = clock_alarm(clk, TIME_RELATIVE, timerprof, profile_port); + HANDLE_MACH_ERROR("clock_alarm", ret) + } } } @@ -505,6 +535,7 @@ JL_DLLEXPORT int jl_profile_start_timer(void) timerprof.tv_nsec = nsecprof%GIGA; running = 1; + // ensure the alarm is running ret = clock_alarm(clk, TIME_RELATIVE, timerprof, profile_port); HANDLE_MACH_ERROR("clock_alarm", ret); diff --git a/src/signals-unix.c b/src/signals-unix.c index cd86a77023347..68eb1f5da22f8 100644 --- a/src/signals-unix.c +++ b/src/signals-unix.c @@ -176,7 +176,7 @@ static void jl_throw_in_ctx(jl_ptls_t ptls, jl_value_t *e, int sig, void *sigctx { if (!ptls->safe_restore) ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, - jl_to_bt_context(sigctx), ptls->pgcstack, 0); + jl_to_bt_context(sigctx), ptls->pgcstack); ptls->sig_exception = e; jl_call_in_ctx(ptls, &jl_sig_throw, sig, sigctx); } @@ -665,6 +665,8 @@ static void *signal_listener(void *arg) unw_context_t *signal_context; // sample each thread, round-robin style in reverse order // (so that thread zero gets notified last) + if (critical || profile) + jl_lock_profile(); for (int i = jl_n_threads; i-- > 0; ) { // notify thread to stop jl_thread_suspend_and_get_state(i, &signal_context); @@ -674,7 +676,7 @@ static void *signal_listener(void *arg) if (critical) { bt_size += rec_backtrace_ctx(bt_data + bt_size, JL_MAX_BT_SIZE / jl_n_threads - 1, - signal_context, NULL, 1); + signal_context, NULL); bt_data[bt_size++].uintptr = 0; } @@ -693,7 +695,7 @@ static void *signal_listener(void *arg) } else { // Get backtrace data bt_size_cur += rec_backtrace_ctx((jl_bt_element_t*)bt_data_prof + bt_size_cur, - bt_size_max - bt_size_cur - 1, signal_context, NULL, 1); + bt_size_max - bt_size_cur - 1, signal_context, NULL); } ptls->safe_restore = old_buf; @@ -709,6 +711,8 @@ static void *signal_listener(void *arg) // notify thread to resume jl_thread_resume(i, sig); } + if (critical || profile) + jl_unlock_profile(); #endif // this part is async with the running of the rest of the program diff --git a/src/signals-win.c b/src/signals-win.c index 09a8a546e4362..9ea8a38813ff3 100644 --- a/src/signals-win.c +++ b/src/signals-win.c @@ -96,15 +96,18 @@ void __cdecl crt_sig_handler(int sig, int num) } } +// StackOverflowException needs extra stack space to record the backtrace +// so we keep one around, shared by all threads +static jl_mutex_t backtrace_lock; static jl_ucontext_t collect_backtrace_fiber; static jl_ucontext_t error_return_fiber; -static PCONTEXT error_ctx; +static PCONTEXT stkerror_ctx; +static jl_ptls_t stkerror_ptls; static int have_backtrace_fiber; static void JL_NORETURN start_backtrace_fiber(void) { - jl_ptls_t ptls = jl_get_ptls_states(); // collect the backtrace - ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, error_ctx, ptls->pgcstack, 0); + stkerror_ptls->bt_size = rec_backtrace_ctx(stkerror_ptls->bt_data, JL_MAX_BT_SIZE, stkerror_ctx, stkerror_ptls->pgcstack); // switch back to the execution fiber jl_setcontext(&error_return_fiber); abort(); @@ -130,11 +133,14 @@ void jl_throw_in_ctx(jl_value_t *excpt, PCONTEXT ctxThread) assert(excpt != NULL); ptls->bt_size = 0; if (excpt != jl_stackovf_exception) { - ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, ctxThread, ptls->pgcstack, 0); + ptls->bt_size = rec_backtrace_ctx(ptls->bt_data, JL_MAX_BT_SIZE, ctxThread, ptls->pgcstack); } else if (have_backtrace_fiber) { - error_ctx = ctxThread; + JL_LOCK(&backtrace_lock); + stkerror_ctx = ctxThread; + stkerror_ptls = ptls; jl_swapcontext(&error_return_fiber, &collect_backtrace_fiber); + JL_UNLOCK_NOGC(&backtrace_lock); } ptls->sig_exception = excpt; } @@ -155,13 +161,16 @@ HANDLE hMainThread = INVALID_HANDLE_VALUE; static void jl_try_deliver_sigint(void) { jl_ptls_t ptls2 = jl_all_tls_states[0]; + jl_lock_profile(); jl_safepoint_enable_sigint(); jl_wake_libuv(); if ((DWORD)-1 == SuspendThread(hMainThread)) { // error jl_safe_printf("error: SuspendThread failed\n"); + jl_unlock_profile(); return; } + jl_unlock_profile(); int force = jl_check_force_sigint(); if (force || (!ptls2->defer_signal && ptls2->io_wait)) { jl_safepoint_consume_sigint(); @@ -326,11 +335,14 @@ static DWORD WINAPI profile_bt( LPVOID lparam ) hBtThread = 0; return 0; } + timeBeginPeriod(tc.wPeriodMin); while (1) { - if (bt_size_cur < bt_size_max) { - DWORD timeout = nsecprof/GIGA; - timeout = min(max(timeout, tc.wPeriodMin*2), tc.wPeriodMax/2); - Sleep(timeout); + DWORD timeout = nsecprof / GIGA; + timeout += tc.wPeriodMin; + Sleep(timeout); + if (bt_size_cur < bt_size_max && running) { + JL_LOCK_NOGC(&jl_in_stackwalk); + jl_lock_profile(); if ((DWORD)-1 == SuspendThread(hMainThread)) { fputs("failed to suspend main thread. aborting profiling.", stderr); break; @@ -341,25 +353,35 @@ static DWORD WINAPI profile_bt( LPVOID lparam ) ctxThread.ContextFlags = CONTEXT_CONTROL | CONTEXT_INTEGER; if (!GetThreadContext(hMainThread, &ctxThread)) { fputs("failed to get context from main thread. aborting profiling.", stderr); - break; + running = 0; + } + else { + // Get backtrace data + bt_size_cur += rec_backtrace_ctx((jl_bt_element_t*)bt_data_prof + bt_size_cur, + bt_size_max - bt_size_cur - 1, &ctxThread, NULL); + // Mark the end of this block with 0 + if (bt_size_cur < bt_size_max) + bt_data_prof[bt_size_cur++].uintptr = 0; } - // Get backtrace data - bt_size_cur += rec_backtrace_ctx((jl_bt_element_t*)bt_data_prof + bt_size_cur, - bt_size_max - bt_size_cur - 1, &ctxThread, NULL, 1); - // Mark the end of this block with 0 - bt_data_prof[bt_size_cur].uintptr = 0; - bt_size_cur++; } + jl_unlock_profile(); + JL_UNLOCK_NOGC(&jl_in_stackwalk); if ((DWORD)-1 == ResumeThread(hMainThread)) { + timeEndPeriod(tc.wPeriodMin); fputs("failed to resume main thread! aborting.", stderr); gc_debug_critical_error(); abort(); } } else { + timeEndPeriod(tc.wPeriodMin); SuspendThread(GetCurrentThread()); + timeBeginPeriod(tc.wPeriodMin); } } + jl_unlock_profile(); + JL_UNLOCK_NOGC(&jl_in_stackwalk); + timeEndPeriod(tc.wPeriodMin); hBtThread = 0; return 0; } @@ -379,7 +401,7 @@ JL_DLLEXPORT int jl_profile_start_timer(void) } else { if ((DWORD)-1 == ResumeThread(hBtThread)) { - fputs("failed to resume profiling thread.",stderr); + fputs("failed to resume profiling thread.", stderr); return -2; } } @@ -420,5 +442,6 @@ void jl_install_thread_signal_handler(jl_ptls_t ptls) collect_backtrace_fiber.uc_stack.ss_sp = (void*)stk; collect_backtrace_fiber.uc_stack.ss_size = ssize; jl_makecontext(&collect_backtrace_fiber, start_backtrace_fiber); + JL_MUTEX_INIT(&backtrace_lock); have_backtrace_fiber = 1; } diff --git a/src/stackwalk.c b/src/stackwalk.c index ecff626e9a03e..95ecaff38462f 100644 --- a/src/stackwalk.c +++ b/src/stackwalk.c @@ -13,6 +13,7 @@ // define `jl_unw_get` as a macro, since (like setjmp) // returning from the callee function will invalidate the context #ifdef _OS_WINDOWS_ +jl_mutex_t jl_in_stackwalk; #define jl_unw_get(context) RtlCaptureContext(context) #elif !defined(JL_DISABLE_LIBUNWIND) #define jl_unw_get(context) unw_getcontext(context) @@ -24,8 +25,8 @@ void jl_unw_get(void *context) {}; extern "C" { #endif -static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *context, int lockless) JL_NOTSAFEPOINT; -static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lockless) JL_NOTSAFEPOINT; +static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *context) JL_NOTSAFEPOINT; +static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp) JL_NOTSAFEPOINT; static jl_gcframe_t *is_enter_interpreter_frame(jl_gcframe_t **ppgcstack, uintptr_t sp) JL_NOTSAFEPOINT { @@ -67,15 +68,14 @@ static jl_gcframe_t *is_enter_interpreter_frame(jl_gcframe_t **ppgcstack, uintpt // elements written to bt_data (and sp if non-NULL) are returned in bt_size. int jl_unw_stepn(bt_cursor_t *cursor, jl_bt_element_t *bt_data, size_t *bt_size, uintptr_t *sp, size_t maxsize, int skip, jl_gcframe_t **ppgcstack, - int from_signal_handler, int lockless) JL_NOTSAFEPOINT + int from_signal_handler) JL_NOTSAFEPOINT { volatile size_t n = 0; volatile int need_more_space = 0; uintptr_t return_ip = 0; uintptr_t thesp = 0; #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - assert(!jl_in_stackwalk); - jl_in_stackwalk = 1; + JL_LOCK_NOGC(&jl_in_stackwalk); if (!from_signal_handler) { // Workaround 32-bit windows bug missing top frame // See for example https://bugs.chromium.org/p/crashpad/issues/detail?id=53 @@ -96,7 +96,7 @@ int jl_unw_stepn(bt_cursor_t *cursor, jl_bt_element_t *bt_data, size_t *bt_size, need_more_space = 1; break; } - have_more_frames = jl_unw_step(cursor, &return_ip, &thesp, lockless); + have_more_frames = jl_unw_step(cursor, &return_ip, &thesp); if (skip > 0) { skip--; continue; @@ -172,20 +172,20 @@ int jl_unw_stepn(bt_cursor_t *cursor, jl_bt_element_t *bt_data, size_t *bt_size, ptls->safe_restore = old_buf; #endif #if defined(_OS_WINDOWS_) && !defined(_CPU_X86_64_) - jl_in_stackwalk = 0; + JL_UNLOCK_NOGC(&jl_in_stackwalk); #endif *bt_size = n; return need_more_space; } NOINLINE size_t rec_backtrace_ctx(jl_bt_element_t *bt_data, size_t maxsize, - bt_context_t *context, jl_gcframe_t *pgcstack, int lockless) JL_NOTSAFEPOINT + bt_context_t *context, jl_gcframe_t *pgcstack) JL_NOTSAFEPOINT { bt_cursor_t cursor; - if (!jl_unw_init(&cursor, context, lockless)) + if (!jl_unw_init(&cursor, context)) return 0; size_t bt_size = 0; - jl_unw_stepn(&cursor, bt_data, &bt_size, NULL, maxsize, 0, &pgcstack, 1, lockless); + jl_unw_stepn(&cursor, bt_data, &bt_size, NULL, maxsize, 0, &pgcstack, 1); return bt_size; } @@ -201,10 +201,10 @@ NOINLINE size_t rec_backtrace(jl_bt_element_t *bt_data, size_t maxsize, int skip jl_unw_get(&context); jl_gcframe_t *pgcstack = jl_pgcstack; bt_cursor_t cursor; - if (!jl_unw_init(&cursor, &context, 0)) + if (!jl_unw_init(&cursor, &context)) return 0; size_t bt_size = 0; - jl_unw_stepn(&cursor, bt_data, &bt_size, NULL, maxsize, skip + 1, &pgcstack, 0, 0); + jl_unw_stepn(&cursor, bt_data, &bt_size, NULL, maxsize, skip + 1, &pgcstack, 0); return bt_size; } @@ -235,7 +235,7 @@ JL_DLLEXPORT jl_value_t *jl_backtrace_from_here(int returnsp, int skip) memset(&context, 0, sizeof(context)); jl_unw_get(&context); jl_gcframe_t *pgcstack = jl_pgcstack; - if (jl_unw_init(&cursor, &context, 0)) { + if (jl_unw_init(&cursor, &context)) { // Skip frame for jl_backtrace_from_here itself skip += 1; size_t offset = 0; @@ -249,7 +249,7 @@ JL_DLLEXPORT jl_value_t *jl_backtrace_from_here(int returnsp, int skip) } size_t size_incr = 0; have_more_frames = jl_unw_stepn(&cursor, (jl_bt_element_t*)jl_array_data(ip) + offset, - &size_incr, sp_ptr, maxincr, skip, &pgcstack, 0, 0); + &size_incr, sp_ptr, maxincr, skip, &pgcstack, 0); skip = 0; offset += size_incr; } @@ -359,6 +359,7 @@ JL_DLLEXPORT jl_value_t *jl_get_excstack(jl_task_t* task, int include_bt, int ma } #if defined(_OS_WINDOWS_) +// XXX: these caches should be per-thread #ifdef _CPU_X86_64_ static UNWIND_HISTORY_TABLE HistoryTable; #else @@ -375,13 +376,11 @@ static PVOID CALLBACK JuliaFunctionTableAccess64( #ifdef _CPU_X86_64_ DWORD64 ImageBase; PRUNTIME_FUNCTION fn = RtlLookupFunctionEntry(AddrBase, &ImageBase, &HistoryTable); - if (fn) return fn; - if (jl_in_stackwalk) { - return 0; - } - jl_in_stackwalk = 1; + if (fn) + return fn; + JL_LOCK_NOGC(&jl_in_stackwalk); PVOID ftable = SymFunctionTableAccess64(hProcess, AddrBase); - jl_in_stackwalk = 0; + JL_UNLOCK_NOGC(&jl_in_stackwalk); return ftable; #else return SymFunctionTableAccess64(hProcess, AddrBase); @@ -395,16 +394,15 @@ static DWORD64 WINAPI JuliaGetModuleBase64( #ifdef _CPU_X86_64_ DWORD64 ImageBase; PRUNTIME_FUNCTION fn = RtlLookupFunctionEntry(dwAddr, &ImageBase, &HistoryTable); - if (fn) return ImageBase; - if (jl_in_stackwalk) { - return 0; - } - jl_in_stackwalk = 1; + if (fn) + return ImageBase; + JL_LOCK_NOGC(&jl_in_stackwalk); DWORD64 fbase = SymGetModuleBase64(hProcess, dwAddr); - jl_in_stackwalk = 0; + JL_UNLOCK_NOGC(&jl_in_stackwalk); return fbase; #else - if (dwAddr == HistoryTable.dwAddr) return HistoryTable.ImageBase; + if (dwAddr == HistoryTable.dwAddr) + return HistoryTable.ImageBase; DWORD64 ImageBase = jl_getUnwindInfo(dwAddr); if (ImageBase) { HistoryTable.dwAddr = dwAddr; @@ -415,45 +413,23 @@ static DWORD64 WINAPI JuliaGetModuleBase64( #endif } -static DWORD64 WINAPI JuliaAsyncGetModuleBase64( - _In_ HANDLE hProcess, - _In_ DWORD64 dwAddr) -{ - //jl_printf(JL_STDOUT, "lookup base %d\n", dwAddr); -#ifdef _CPU_X86_64_ - return JuliaGetModuleBase64(hProcess, dwAddr); -#else - if (dwAddr == HistoryTable.dwAddr) return HistoryTable.ImageBase; - DWORD64 ImageBase = jl_trygetUnwindInfo(dwAddr); - if (ImageBase) { - HistoryTable.dwAddr = dwAddr; - HistoryTable.ImageBase = ImageBase; - return ImageBase; - } - return SymGetModuleBase64(hProcess, dwAddr); -#endif -} - // Might be called from unmanaged thread. -int needsSymRefreshModuleList; +volatile int needsSymRefreshModuleList; BOOL (WINAPI *hSymRefreshModuleList)(HANDLE); + void jl_refresh_dbg_module_list(void) { - if (needsSymRefreshModuleList && hSymRefreshModuleList != 0 && !jl_in_stackwalk) { - jl_in_stackwalk = 1; + if (needsSymRefreshModuleList && hSymRefreshModuleList != NULL) { hSymRefreshModuleList(GetCurrentProcess()); - jl_in_stackwalk = 0; needsSymRefreshModuleList = 0; } } -static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context, int lockless) +static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context) { + int result; + JL_LOCK_NOGC(&jl_in_stackwalk); jl_refresh_dbg_module_list(); #if !defined(_CPU_X86_64_) - if (jl_in_stackwalk) { - return 0; - } - jl_in_stackwalk = 1; memset(&cursor->stackframe, 0, sizeof(cursor->stackframe)); cursor->stackframe.AddrPC.Offset = Context->Eip; cursor->stackframe.AddrStack.Offset = Context->Esp; @@ -462,15 +438,15 @@ static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *Context, int lockless) cursor->stackframe.AddrStack.Mode = AddrModeFlat; cursor->stackframe.AddrFrame.Mode = AddrModeFlat; cursor->context = *Context; - BOOL result = StackWalk64(IMAGE_FILE_MACHINE_I386, GetCurrentProcess(), hMainThread, - &cursor->stackframe, &cursor->context, NULL, JuliaFunctionTableAccess64, - lockless ? JuliaAsyncGetModuleBase64 : JuliaGetModuleBase64, NULL); - jl_in_stackwalk = 0; - return result; + result = StackWalk64(IMAGE_FILE_MACHINE_I386, GetCurrentProcess(), hMainThread, + &cursor->stackframe, &cursor->context, NULL, JuliaFunctionTableAccess64, + JuliaGetModuleBase64, NULL); #else *cursor = *Context; - return 1; + result = 1; #endif + JL_UNLOCK_NOGC(&jl_in_stackwalk); + return result; } static int readable_pointer(LPCVOID pointer) @@ -487,10 +463,8 @@ static int readable_pointer(LPCVOID pointer) return 1; } -static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lockless) +static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp) { - DWORD64 WINAPI (*GetModuleBase64)(HANDLE, DWORD64) = lockless ? - JuliaAsyncGetModuleBase64 : JuliaGetModuleBase64; // Might be called from unmanaged thread. #ifndef _CPU_X86_64_ *ip = (uintptr_t)cursor->stackframe.AddrPC.Offset; @@ -504,7 +478,7 @@ static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lo } BOOL result = StackWalk64(IMAGE_FILE_MACHINE_I386, GetCurrentProcess(), hMainThread, - &cursor->stackframe, &cursor->context, NULL, JuliaFunctionTableAccess64, GetModuleBase64, NULL); + &cursor->stackframe, &cursor->context, NULL, JuliaFunctionTableAccess64, JuliaGetModuleBase64, NULL); return result; #else *ip = (uintptr_t)cursor->Rip; @@ -517,20 +491,15 @@ static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lo return cursor->Rip != 0; } - DWORD64 ImageBase = GetModuleBase64(GetCurrentProcess(), cursor->Rip); + DWORD64 ImageBase = JuliaGetModuleBase64(GetCurrentProcess(), cursor->Rip); if (!ImageBase) return 0; PRUNTIME_FUNCTION FunctionEntry = (PRUNTIME_FUNCTION)JuliaFunctionTableAccess64( GetCurrentProcess(), cursor->Rip); - if (!FunctionEntry) { // assume this is a NO_FPO RBP-based function - cursor->Rsp = cursor->Rbp; // MOV RSP, RBP - if (!readable_pointer((LPCVOID)cursor->Rsp)) - return 0; - cursor->Rbp = *(DWORD64*)cursor->Rsp; // POP RBP - cursor->Rsp += sizeof(void*); - cursor->Rip = *(DWORD64*)cursor->Rsp; // POP RIP (aka RET) - cursor->Rsp += sizeof(void*); + if (!FunctionEntry) { + // Not code or bad unwind? + return 0; } else { PVOID HandlerData; @@ -552,12 +521,12 @@ static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lo #elif !defined(JL_DISABLE_LIBUNWIND) // stacktrace using libunwind -static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *context, int lockless) +static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *context) { return unw_init_local(cursor, context) == 0; } -static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lockless) +static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp) { unw_word_t reg; if (unw_get_reg(cursor, UNW_REG_IP, ®) < 0) @@ -577,7 +546,7 @@ NOINLINE size_t rec_backtrace_ctx_dwarf(jl_bt_element_t *bt_data, size_t maxsize bt_cursor_t cursor; if (unw_init_local_dwarf(&cursor, context) != UNW_ESUCCESS) return 0; - jl_unw_stepn(&cursor, bt_data, &bt_size, NULL, maxsize, 0, &pgcstack, 1, 0); + jl_unw_stepn(&cursor, bt_data, &bt_size, NULL, maxsize, 0, &pgcstack, 1); return bt_size; } #endif @@ -589,7 +558,7 @@ static int jl_unw_init(bt_cursor_t *cursor, bt_context_t *context) return 0; } -static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp, int lockless) +static int jl_unw_step(bt_cursor_t *cursor, uintptr_t *ip, uintptr_t *sp) { return 0; } diff --git a/src/staticdata.c b/src/staticdata.c index 440926d0876a7..b15fec248e09a 100644 --- a/src/staticdata.c +++ b/src/staticdata.c @@ -166,9 +166,9 @@ typedef enum { } jl_callingconv_t; -// this supports up to 1 GB images and 16 RefTags +// this supports up to 8 RefTags, 512MB of pointer data, and 4/2 (64/32-bit) GB of constant data. // if a larger size is required, will need to add support for writing larger relocations in many cases below -#define RELOC_TAG_OFFSET 28 +#define RELOC_TAG_OFFSET 29 /* read and write in host byte order */ @@ -1394,6 +1394,12 @@ static void jl_save_system_image_to_stream(ios_t *f) jl_write_gv_ints(&s); } + if (sysimg.size > ((uintptr_t)1 << RELOC_TAG_OFFSET) || + const_data.size > ((uintptr_t)1 << RELOC_TAG_OFFSET)*sizeof(void*)) { + jl_printf(JL_STDERR, "ERROR: system image too large\n"); + jl_exit(1); + } + // step 3: combine all of the sections into one file write_uint32(f, sysimg.size - sizeof(uint32_t)); ios_seek(&sysimg, sizeof(uint32_t)); diff --git a/src/task.c b/src/task.c index 9d88306dd4eee..06e6d4895e7d7 100644 --- a/src/task.c +++ b/src/task.c @@ -52,9 +52,7 @@ static inline void sanitizer_start_switch_fiber(const void* bottom, size_t size) static inline void sanitizer_finish_switch_fiber(void) {} #endif -#if defined(_OS_WINDOWS_) -volatile int jl_in_stackwalk = 0; -#else +#if !defined(_OS_WINDOWS_) #ifdef JL_HAVE_UCONTEXT #include #endif diff --git a/stdlib/Pkg.version b/stdlib/Pkg.version index f80c75c33cefa..8136ba368449b 100644 --- a/stdlib/Pkg.version +++ b/stdlib/Pkg.version @@ -1,2 +1,2 @@ PKG_BRANCH = master -PKG_SHA1 = 1542f285243b8ac31e666dee3c6690bdaf26a0be +PKG_SHA1 = 0c3ccb119efd8b3182c114599abe78bc4f89dd4c diff --git a/stdlib/Profile/test/runtests.jl b/stdlib/Profile/test/runtests.jl index 8ff9fa9c76005..007492745ca47 100644 --- a/stdlib/Profile/test/runtests.jl +++ b/stdlib/Profile/test/runtests.jl @@ -111,3 +111,25 @@ end end @test getline(values(fdictc)) == getline(values(fdict0)) + 2 end + +# Profile deadlocking in compilation (debuginfo registration) +let cmd = Base.julia_cmd() + script = """ + using Profile + f(::Val) = GC.safepoint() + @profile for i = 1:10^3; f(Val(i)); end + print(Profile.len_data()) + """ + p = open(`$cmd -e $script`) + t = Timer(120) do t + # should be under 10 seconds, so give it 2 minutes then report failure + println("KILLING BY PROFILE TEST WATCHDOG\n") + kill(p, Base.SIGTERM) + sleep(10) + kill(p, Base.SIGKILL) + end + s = read(p, String) + close(t) + @test success(p) + @test parse(Int, s) > 1000 +end diff --git a/test/compiler/inference.jl b/test/compiler/inference.jl index 4fd9a020252b9..226196fa74a49 100644 --- a/test/compiler/inference.jl +++ b/test/compiler/inference.jl @@ -2562,47 +2562,6 @@ end @test map(>:, [Int], [Int]) == [true] -# issue 35566 -module Issue35566 -function step(acc, x) - xs, = acc - y = x > 0.0 ? x : missing - if y isa eltype(xs) - ys = push!(xs, y) - else - ys = vcat(xs, [y]) - end - return (ys,) -end - -function probe(y) - if y isa Tuple{Vector{Missing}} - return Val(:missing) - else - return Val(:expected) - end -end - -function _foldl_iter(rf, val::T, iter, state) where {T} - while true - ret = iterate(iter, state) - ret === nothing && break - x, state = ret - y = rf(val, x) - if y isa T - val = y - else - return probe(y) - end - end - return Val(:expected) -end - -f() = _foldl_iter(step, (Missing[],), [0.0], 1) -end -@test Core.Compiler.typesubtract(Tuple{Union{Int,Char}}, Tuple{Char}) == Tuple{Int} -@test Base.return_types(Issue35566.f) == [Val{:expected}] - # constant prop through keyword arguments _unstable_kw(;x=1,y=2) = x == 1 ? 0 : "" _use_unstable_kw_1() = _unstable_kw(x = 2) @@ -2647,3 +2606,6 @@ for badf in [getfield_const_typename_bad1, getfield_const_typename_bad2] @test code[end] == Expr(:unreachable) @test_throws TypeError badf() end + +# issue #37638 +@test !(Core.Compiler.return_type(() -> (nothing, Any[]...)[2], Tuple{}) <: Vararg) diff --git a/test/file.jl b/test/file.jl index 3a7b4710281b5..27e60dd0c5ddf 100644 --- a/test/file.jl +++ b/test/file.jl @@ -268,6 +268,20 @@ no_error_logging(f::Function) = end end +@testset "mktempdir() permissions correction" begin + # Test that `mktempdir()` with children with weird permissions get cleared + # before we would delete the directory + local temp_dir_path + mktempdir() do dir + temp_dir_path = dir + mkdir(joinpath(dir, "foo")) + touch(joinpath(dir, "foo", "bar")) + chmod(joinpath(dir, "foo"), 0o444) + @test isdir(temp_dir_path) + end + @test !isdir(temp_dir_path) +end + ####################################################################### # This section tests some of the features of the stat-based file info # ####################################################################### diff --git a/test/show.jl b/test/show.jl index 4e944633c29a8..1e173b5c64fee 100644 --- a/test/show.jl +++ b/test/show.jl @@ -1329,6 +1329,39 @@ end @test static_shown(QuoteNode(:x)) == ":(:x)" +# PR #38049 +@test static_shown(sum) == "Base.sum" +@test static_shown(+) == "Base.:(+)" +@test static_shown(typeof(+)) == "typeof(Base.:(+))" + +struct var"#X#" end +var"#f#"() = 2 +struct var"%X%" end # Invalid name without '#' + +# (Just to make this test more sustainable,) we don't necesssarily need to test the exact +# output format, just ensure that it prints at least the parts we expect: +@test occursin(".var\"#X#\"", static_shown(var"#X#")) # Leading `.` tests it printed a module name. +@test occursin(r"Set{var\"[^\"]+\"} where var\"[^\"]+\"", static_shown(Set{<:Any})) + +# Test that static_shown is returning valid, correct julia expressions +@testset "static_show() prints valid julia" begin + @testset for v in ( + var"#X#", + var"#X#"(), + var"%X%", + var"%X%"(), + Vector, + Vector{<:Any}, + Vector{var"#X#"}, + +, + typeof(+), + var"#f#", + typeof(var"#f#"), + ) + @test v == eval(Meta.parse(static_shown(v))) + end +end + # Test @show let fname = tempname() try @@ -1986,3 +2019,12 @@ end @test sprint(show, skipmissing([1,2,missing])) == "skipmissing(Union{Missing, $Int}[1, 2, missing])" @test sprint(show, skipmissing((missing,1.0,'a'))) == "skipmissing((missing, 1.0, 'a'))" end + +@testset "unicode in method table" begin + αsym = gensym(:α) + ℓsym = gensym(:ℓ) + eval(:(foo($αsym) = $αsym)) + eval(:(bar($ℓsym) = $ℓsym)) + @test contains(string(methods(foo)), "foo(α)") + @test contains(string(methods(bar)), "bar(ℓ)") +end