diff --git a/base/loading.jl b/base/loading.jl index ddca6283d848f..c3198408596c9 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2,84 +2,6 @@ # Base.require is the implementation for the `import` statement -# Cross-platform case-sensitive path canonicalization - -if is_unix() && !is_apple() - # assume case-sensitive filesystems, don't have to do anything - isfile_casesensitive(path) = isfile(path) -elseif is_windows() - # GetLongPathName Win32 function returns the case-preserved filename on NTFS. - function isfile_casesensitive(path) - isfile(path) || return false # Fail fast - Filesystem.longpath(path) == path - end -elseif is_apple() - # HFS+ filesystem is case-preserving. The getattrlist API returns - # a case-preserved filename. In the rare event that HFS+ is operating - # in case-sensitive mode, this will still work but will be redundant. - - # Constants from - const ATRATTR_BIT_MAP_COUNT = 5 - const ATTR_CMN_NAME = 1 - const BITMAPCOUNT = 1 - const COMMONATTR = 5 - const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links - - const attr_list = zeros(UInt8, 24) - attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT - attr_list[COMMONATTR] = ATTR_CMN_NAME - - # This essentially corresponds to the following C code: - # attrlist attr_list; - # memset(&attr_list, 0, sizeof(attr_list)); - # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT; - # attr_list.commonattr = ATTR_CMN_NAME; - # struct Buffer { - # u_int32_t total_length; - # u_int32_t filename_offset; - # u_int32_t filename_length; - # char filename[max_filename_length]; - # }; - # Buffer buf; - # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW); - function isfile_casesensitive(path) - isfile(path) || return false - path_basename = String(basename(path)) - local casepreserved_basename - const header_size = 12 - buf = Array{UInt8}(length(path_basename) + header_size + 1) - while true - ret = ccall(:getattrlist, Cint, - (Cstring, Ptr{Void}, Ptr{Void}, Csize_t, Culong), - path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW) - systemerror(:getattrlist, ret ≠ 0) - filename_length = unsafe_load( - convert(Ptr{UInt32}, pointer(buf) + 8)) - if (filename_length + header_size) > length(buf) - resize!(buf, filename_length + header_size) - continue - end - casepreserved_basename = - view(buf, (header_size+1):(header_size+filename_length-1)) - break - end - # Hack to compensate for inability to create a string from a subarray with no allocations. - path_basename.data == casepreserved_basename && return true - - # If there is no match, it's possible that the file does exist but HFS+ - # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html. - isascii(path_basename) && return false - normalize_string(path_basename, :NFD).data == casepreserved_basename - end -else - # Generic fallback that performs a slow directory listing. - function isfile_casesensitive(path) - isfile(path) || return false - dir, filename = splitdir(path) - any(readdir(dir) .== filename) - end -end - function try_path(prefix::String, base::String, name::String) path = joinpath(prefix, name) isfile_casesensitive(path) && return abspath(path) diff --git a/base/pkg/entry.jl b/base/pkg/entry.jl index 6a0cf4dda1f6b..95109f07dbb9a 100644 --- a/base/pkg/entry.jl +++ b/base/pkg/entry.jl @@ -7,7 +7,7 @@ import ..Reqs, ..Read, ..Query, ..Resolve, ..Cache, ..Write, ..Dir import ...LibGit2 importall ...LibGit2 import ...Pkg.PkgError -using ..Types +using ..Types, Base.Filesystem macro recover(ex) quote @@ -48,25 +48,23 @@ end function add(pkg::AbstractString, vers::VersionSet) outdated = :maybe - @sync begin - @async if !edit(Reqs.add,pkg,vers) - ispath(pkg) || throw(PkgError("unknown package $pkg")) - info("Nothing to be done") - end - branch = Dir.getmetabranch() - outdated = with(GitRepo, "METADATA") do repo - if LibGit2.branch(repo) == branch - if LibGit2.isdiff(repo, "origin/$branch") - outdated = :yes - else - try - LibGit2.fetch(repo) - outdated = LibGit2.isdiff(repo, "origin/$branch") ? (:yes) : (:no) - end - end + if !edit(Reqs.add,pkg,vers) + ispath_casesensitive(pkg) || throw(PkgError("unknown package $pkg")) + info("Nothing to be done") + end + branch = Dir.getmetabranch() + outdated = with(GitRepo, "METADATA") do repo + if LibGit2.branch(repo) == branch + if LibGit2.isdiff(repo, "origin/$branch") + outdated = :yes else - :no # user is doing something funky with METADATA + try + LibGit2.fetch(repo) + outdated = LibGit2.isdiff(repo, "origin/$branch") ? (:yes) : (:no) + end end + else + :no # user is doing something funky with METADATA end end if outdated != :no @@ -79,7 +77,7 @@ add(pkg::AbstractString, vers::VersionNumber...) = add(pkg,VersionSet(vers...)) function rm(pkg::AbstractString) edit(Reqs.rm,pkg) && return - ispath(pkg) || return info("Nothing to be done") + ispath_casesensitive(pkg) || return info("Nothing to be done") info("Removing $pkg (unregistered)") Write.remove(pkg) end @@ -113,7 +111,7 @@ function installed(pkg::AbstractString) avail = Read.available(pkg) if Read.isinstalled(pkg) res = typemin(VersionNumber) - if ispath(joinpath(pkg,".git")) + if Read.isgitrepo(pkg) LibGit2.with(GitRepo, pkg) do repo res = Read.installed_version(pkg, repo, avail) end @@ -159,7 +157,7 @@ function status(io::IO, pkg::AbstractString, ver::VersionNumber, fix::Bool) @printf io " - %-29s " pkg fix || return println(io,ver) @printf io "%-19s" ver - if ispath(pkg,".git") + if Read.isgitrepo(pkg) prepo = GitRepo(pkg) try with(LibGit2.head(prepo)) do phead @@ -222,7 +220,8 @@ function clone(url_or_pkg::AbstractString) end function checkout(pkg::AbstractString, branch::AbstractString, do_merge::Bool, do_pull::Bool) - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + isdir_casesensitive(pkg) || throw(PkgError("$pkg is not installed")) + Read.isgitrepo(pkg) || throw(PkgError("$pkg is not a git repo")) info("Checking out $pkg $branch...") with(GitRepo, pkg) do r LibGit2.transact(r) do repo @@ -240,7 +239,7 @@ function checkout(pkg::AbstractString, branch::AbstractString, do_merge::Bool, d end function free(pkg::AbstractString) - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + Read.isgitrepo(pkg) || throw(PkgError("$pkg is not a git repo")) Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be freed – not an installed package")) avail = Read.available(pkg) isempty(avail) && throw(PkgError("$pkg cannot be freed – not a registered package")) @@ -267,7 +266,7 @@ end function free(pkgs) try for pkg in pkgs - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + Read.isgitrepo(pkg) || throw(PkgError("$pkg is not a git repo")) Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be freed – not an installed package")) avail = Read.available(pkg) isempty(avail) && throw(PkgError("$pkg cannot be freed – not a registered package")) @@ -291,7 +290,8 @@ function free(pkgs) end function pin(pkg::AbstractString, head::AbstractString) - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + isdir_casesensitive(pkg) || throw(PkgError("$pkg is not installed")) + Read.isgitrepo(pkg) || throw(PkgError("$pkg is not a git repo")) should_resolve = true with(GitRepo, pkg) do repo id = if isempty(head) # get HEAD commit @@ -341,7 +341,7 @@ end pin(pkg::AbstractString) = pin(pkg, "") function pin(pkg::AbstractString, ver::VersionNumber) - ispath(pkg,".git") || throw(PkgError("$pkg is not a git repo")) + Read.isgitrepo(pkg) || throw(PkgError("$pkg is not a git repo")) Read.isinstalled(pkg) || throw(PkgError("$pkg cannot be pinned – not an installed package")) avail = Read.available(pkg) isempty(avail) && throw(PkgError("$pkg cannot be pinned – not a registered package")) @@ -411,7 +411,7 @@ function update(branch::AbstractString, upkgs::Set{String}) try stopupdate = false for (pkg,ver) in fixed - ispath(pkg,".git") || continue + Read.isgitrepo(pkg) || continue pkg in dont_update && continue with(GitRepo, pkg) do repo if LibGit2.isattached(repo) @@ -685,6 +685,10 @@ function test!(pkg::AbstractString, errs::Vector{AbstractString}, nopkgs::Vector{AbstractString}, notests::Vector{AbstractString}; coverage::Bool=false) + if !isdir_casesensitive(pkg) + push!(nopkgs, pkg) + return false + end reqs_path = abspath(pkg,"test","REQUIRE") if isfile(reqs_path) tests_require = Reqs.parse(reqs_path) @@ -694,9 +698,7 @@ function test!(pkg::AbstractString, end end test_path = abspath(pkg,"test","runtests.jl") - if !isdir(pkg) - push!(nopkgs, pkg) - elseif !isfile(test_path) + if !isfile(test_path) push!(notests, pkg) else info("Testing $pkg") diff --git a/base/pkg/pkg.jl b/base/pkg/pkg.jl index 828a7d9bef50e..3a6390798752b 100644 --- a/base/pkg/pkg.jl +++ b/base/pkg/pkg.jl @@ -15,6 +15,7 @@ type PkgError <: Exception end PkgError(msg::AbstractString) = PkgError(msg, Nullable{Exception}()) function Base.showerror(io::IO, pkgerr::PkgError) + print(io, "PkgError: ") print(io, pkgerr.msg) if !isnull(pkgerr.ex) pkgex = get(pkgerr.ex) diff --git a/base/pkg/read.jl b/base/pkg/read.jl index faf2a91301006..b0e61b1f7e055 100644 --- a/base/pkg/read.jl +++ b/base/pkg/read.jl @@ -3,18 +3,29 @@ module Read import ...LibGit2, ..Cache, ..Reqs, ...Pkg.PkgError, ..Dir -using ..Types +using ..Types, Base.Filesystem -readstrip(path...) = strip(readstring(joinpath(path...))) +isgitrepo(pkg) = + isdir_casesensitive(pkg) && ispath(pkg, ".git") -url(pkg::AbstractString) = readstrip(Dir.path("METADATA"), pkg, "url") -sha1(pkg::AbstractString, ver::VersionNumber) = readstrip(Dir.path("METADATA"), pkg, "versions", string(ver), "sha1") +readstrip(path...) = + strip(readstring(joinpath(path...))) + +url(pkg::AbstractString) = + readstrip(Dir.path("METADATA"), pkg, "url") + +sha1(pkg::AbstractString, ver::VersionNumber) = + readstrip(Dir.path("METADATA"), pkg, "versions", string(ver), "sha1") function available(names=readdir("METADATA")) pkgs = Dict{String,Dict{VersionNumber,Available}}() for pkg in names - isfile("METADATA", pkg, "url") || continue - versdir = joinpath("METADATA", pkg, "versions") + pkgdir = joinpath("METADATA", pkg) + if !isdir_casesensitive(pkgdir) + continue + end + isfile(pkgdir, "url") || continue + versdir = joinpath(pkgdir, "versions") isdir(versdir) || continue for ver in readdir(versdir) ismatch(Base.VERSION_REGEX, ver) || continue @@ -28,13 +39,18 @@ function available(names=readdir("METADATA")) end return pkgs end -available(pkg::AbstractString) = get(available([pkg]),pkg,Dict{VersionNumber,Available}()) +available(pkg::AbstractString) = + get(available([pkg]),pkg,Dict{VersionNumber,Available}()) function latest(names=readdir("METADATA")) pkgs = Dict{String,Available}() for pkg in names - isfile("METADATA", pkg, "url") || continue - versdir = joinpath("METADATA", pkg, "versions") + pkgdir = joinpath("METADATA", pkg) + if !isdir_casesensitive(pkgdir) + continue + end + isfile(pkgdir, "url") || continue + versdir = joinpath(pkgdir, "versions") isdir(versdir) || continue pkgversions = VersionNumber[] for ver in readdir(versdir) @@ -53,12 +69,12 @@ function latest(names=readdir("METADATA")) end isinstalled(pkg::AbstractString) = - pkg != "METADATA" && pkg != "REQUIRE" && pkg[1] != '.' && isdir(pkg) + pkg != "METADATA" && pkg != "REQUIRE" && pkg[1] != '.' && isdir_casesensitive(pkg) function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=available(pkg)) isinstalled(pkg) || throw(PkgError("$pkg is not an installed package.")) isfile("METADATA", pkg, "url") || return true - ispath(pkg, ".git") || return true + isgitrepo(pkg) || return true LibGit2.isdirty(prepo) && return true LibGit2.isattached(prepo) && return true @@ -101,7 +117,7 @@ function isfixed(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=availa end function ispinned(pkg::AbstractString) - ispath(pkg,".git") || return false + isgitrepo(pkg) || return false LibGit2.with(LibGit2.GitRepo, pkg) do repo return ispinned(repo) end @@ -115,7 +131,7 @@ function ispinned(prepo::LibGit2.GitRepo) end function installed_version(pkg::AbstractString, prepo::LibGit2.GitRepo, avail::Dict=available(pkg)) - ispath(pkg,".git") || return typemin(VersionNumber) + isgitrepo(pkg) || return typemin(VersionNumber) # get package repo head hash local head @@ -176,7 +192,7 @@ end function requires_path(pkg::AbstractString, avail::Dict=available(pkg)) pkgreq = joinpath(pkg,"REQUIRE") - ispath(pkg,".git") || return pkgreq + isgitrepo(pkg) || return pkgreq repo = LibGit2.GitRepo(pkg) head = LibGit2.with(LibGit2.GitRepo, pkg) do repo LibGit2.isdirty(repo, "REQUIRE") && return pkgreq @@ -203,7 +219,7 @@ function installed(avail::Dict=available()) for pkg in readdir() isinstalled(pkg) || continue ap = get(avail,pkg,Dict{VersionNumber,Available}()) - if ispath(pkg,".git") + if isgitrepo(pkg) LibGit2.with(LibGit2.GitRepo, pkg) do repo ver = installed_version(pkg, repo, ap) fixed = isfixed(pkg, repo, ap) @@ -238,7 +254,7 @@ function free(inst::Dict=installed(), dont_update::Set{String}=Set{String}()) end function issue_url(pkg::AbstractString) - ispath(pkg,".git") || return "" + isgitrepo(pkg) || return "" m = match(LibGit2.GITHUB_REGEX, url(pkg)) m === nothing && return "" return "https://github.com/" * m.captures[1] * "/issues" diff --git a/base/stat.jl b/base/stat.jl index 9b1b5391edf99..1ee3188a0ccc9 100644 --- a/base/stat.jl +++ b/base/stat.jl @@ -10,11 +10,14 @@ export isblockdev, ischardev, isdir, + isdir_casesensitive, isfifo, isfile, + isfile_casesensitive, islink, ismount, ispath, + ispath_casesensitive, issetgid, issetuid, issocket, @@ -163,3 +166,91 @@ function ismount(path...) (s1.inode == s2.inode) && return true false end + +# Cross-platform case-sensitive path canonicalization. +# _ispath_casesensitive(path) assumes that ispath(path) is true, +# and returns whether path matches the case stored in the filesystem. +if is_unix() && !is_apple() + # assume case-sensitive filesystems, don't have to do anything + _ispath_casesensitive(path) = true +elseif is_windows() + # GetLongPathName Win32 function returns the case-preserved filename on NTFS. + _ispath_casesensitive(path) = Filesystem.longpath(path) == path +elseif is_apple() + # HFS+ filesystem is case-preserving. The getattrlist API returns + # a case-preserved filename. In the rare event that HFS+ is operating + # in case-sensitive mode, this will still work but will be redundant. + + # Constants from + const ATRATTR_BIT_MAP_COUNT = 5 + const ATTR_CMN_NAME = 1 + const BITMAPCOUNT = 1 + const COMMONATTR = 5 + const FSOPT_NOFOLLOW = 1 # Don't follow symbolic links + + const attr_list = zeros(UInt8, 24) + attr_list[BITMAPCOUNT] = ATRATTR_BIT_MAP_COUNT + attr_list[COMMONATTR] = ATTR_CMN_NAME + + # This essentially corresponds to the following C code: + # attrlist attr_list; + # memset(&attr_list, 0, sizeof(attr_list)); + # attr_list.bitmapcount = ATTR_BIT_MAP_COUNT; + # attr_list.commonattr = ATTR_CMN_NAME; + # struct Buffer { + # u_int32_t total_length; + # u_int32_t filename_offset; + # u_int32_t filename_length; + # char filename[max_filename_length]; + # }; + # Buffer buf; + # getattrpath(path, &attr_list, &buf, sizeof(buf), FSOPT_NOFOLLOW); + function _ispath_casesensitive(path) + # we need to strip out trailing '/' for directories + path = rstrip(path, '/') + const header_size = 12 + path_basename = String(basename(path)) + buf = Array{UInt8}(length(path_basename) + header_size + 1) + local casepreserved_basename + while true + ret = ccall(:getattrlist, Cint, + (Cstring, Ptr{Void}, Ptr{Void}, Csize_t, Culong), + path, attr_list, buf, sizeof(buf), FSOPT_NOFOLLOW) + systemerror(:getattrlist, ret ≠ 0) + filename_length = unsafe_load( + convert(Ptr{UInt32}, pointer(buf) + 8)) + if (filename_length + header_size) > length(buf) + resize!(buf, filename_length + header_size) + continue + end + casepreserved_basename = + view(buf, (header_size+1):(header_size+filename_length-1)) + break + end + # Hack to compensate for inability to create a string from a subarray with no allocations. + path_basename.data == casepreserved_basename && return true + + # If there is no match, it's possible that the file does exist but HFS+ + # performed unicode normalization. See https://developer.apple.com/library/mac/qa/qa1235/_index.html. + isascii(path_basename) && return false + normalize_string(path_basename, :NFD).data == casepreserved_basename + end +else + # Generic fallback that performs a slow directory listing. + function _ispath_casesensitive(path) + dir, filename = splitdir(path) + any(readdir(dir) .== filename) + end +end +for f in (:ispath, :isfile, :isdir) + fc = Symbol(f,"_casesensitive") + @eval @doc let f=$(string(f)); """ + $(f)_casesensitive(path) + +This function implements a case-sensitive `$f` on filesystems (e.g. Mac and Windows) +that are case-insensitive but case-preserving. It is identical to `$f` except +that it returns `false` if `basename(path)` does not also match the *case* +of the path stored in the filesystem. (It is equivalent to `$f` on case-sensitive +filesystems.)"""; end -> $fc(path) = $f(path) && _ispath_casesensitive(path) + @eval $fc(path...) = $fc(joinpath(path...)) +end diff --git a/test/pkg.jl b/test/pkg.jl index 7fb6848d270eb..11c3208b714b3 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -99,8 +99,24 @@ temp_pkg_dir() do Pkg.setprotocol!("") @test Pkg.Cache.rewrite_url_to === nothing Pkg.setprotocol!("https") + + # test that Pkg.add is case sensitive + @test_throws PkgError Pkg.add("example") === nothing + Pkg.add("Example") @test [keys(Pkg.installed())...] == ["Example"] + + # test that Pkg.installed, Pkg.checkout, + # Pkg.status, Pkg.free, Pkg.test, and Pkg.available are case sensitive + @test_throws PkgError Pkg.installed("example") + @test_throws PkgError Pkg.checkout("example") + @test Pkg.status("example") === nothing + @test_throws PkgError Pkg.free("example") + @test_throws PkgError Pkg.test("example") + Pkg.rm("example") + @test [keys(Pkg.installed())...] == ["Example"] + @test_throws PkgError Pkg.available("example") + iob = IOBuffer() Pkg.checkout("Example") Pkg.status("Example", iob) @@ -206,8 +222,8 @@ temp_pkg_dir() do Pkg.add("REPL") error("unexpected") catch err - @test isa(err.exceptions[1].ex, PkgError) - @test err.exceptions[1].ex.msg == "REPL can't be installed because " * + @test isa(err, PkgError) + @test err.msg == "REPL can't be installed because " * "it has no versions that support $VERSION of julia. You may " * "need to update METADATA by running `Pkg.update()`" end @@ -217,8 +233,8 @@ temp_pkg_dir() do Pkg.add("NonexistentPackage") error("unexpected") catch err - @test isa(err.exceptions[1].ex, PkgError) - @test err.exceptions[1].ex.msg == "unknown package NonexistentPackage" + @test isa(err, PkgError) + @test err.msg == "unknown package NonexistentPackage" end try Pkg.available("NonexistentPackage")