add Downloads stdlib; replace Base.download

- Base.download is deprecated in favor of Downloads.download
- the former is now implemented by calling the latter

NEWS.md

@@ -58,8 +58,9 @@ Build system changes
 
 Library functions
 -----------------
-* The `Base.Grisu` code has been officially removed (float printing was switched to the ryu algorithm code in 1.4)
 
+* The `Base.download` function has been deprecated (silently, by default) in favor of the new `Downloads.download` standard library function ([#37340]).
+* The `Base.Grisu` code has been officially removed (float printing was switched to the ryu algorithm code in 1.4)
 
 New library functions
 ---------------------
@@ -81,6 +82,7 @@ New library features
 
 Standard library changes
 ------------------------
+
 * The `nextprod` function now accepts tuples and other array types for its first argument ([#35791]).
 * The `reverse(A; dims)` function for multidimensional `A` can now reverse multiple dimensions at once
   by passing a tuple for `dims`, and defaults to reversing all dimensions; there is also a multidimensional
@@ -99,6 +101,7 @@ Standard library changes
 * `RegexMatch` objects can now be probed for whether a named capture group exists within it through `haskey()` ([#36717]).
 * For consistency `haskey(r::RegexMatch, i::Integer)` has also been added and returns if the capture group for `i` exists ([#37300]).
 * A new standard library `TOML` has been added for parsing and printing [TOML files](https://toml.io) ([#37034]).
+* A new standard library `Downloads` has been added, which replaces the old `Base.download` function with `Downloads.download`, providing cross-platform, multi-protocol, in-process download functionality implemented with [libcurl](https://curl.haxx.se/libcurl/) ([#37340]).
 * The `Pkg.BinaryPlatforms` module has been moved into `Base` as `Base.BinaryPlatforms` and heavily reworked.
   Applications that want to be compatible with the old API should continue to import `Pkg.BinaryPlatforms`,
   however new users should use `Base.BinaryPlatforms` directly. ([#37320])
@@ -107,6 +110,7 @@ Standard library changes
   all of `Pkg` alongside. ([#37320])
 
 #### LinearAlgebra
+
 * New method `LinearAlgebra.issuccess(::CholeskyPivoted)` for checking whether pivoted Cholesky factorization was successful ([#36002]).
 * `UniformScaling` can now be indexed into using ranges to return dense matrices and vectors ([#24359]).
 * New function `LinearAlgebra.BLAS.get_num_threads()` for getting the number of BLAS threads. ([#36360])

base/download.jl

@@ -1,51 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# file downloading
-
-if Sys.iswindows()
-    function download_powershell(url::AbstractString, filename::AbstractString)
-        ps = joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\WindowsPowerShell\\v1.0\\powershell.exe")
-        tls12 = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-        client = "New-Object System.Net.Webclient"
-        # in the following we escape ' with '' (see https://ss64.com/ps/syntax-esc.html)
-        downloadfile = "($client).DownloadFile('$(replace(url, "'" => "''"))', '$(replace(filename, "'" => "''"))')"
-        # PowerShell v3 or later is required for Tls12
-        proc = run(pipeline(`$ps -Version 3 -NoProfile -Command "$tls12; $downloadfile"`; stderr=stderr); wait=false)
-        if !success(proc)
-            if proc.exitcode % Int32 == -393216
-                # appears to be "wrong version" exit code, based on
-                # https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-startup-tasks-common
-                @error "Downloading files requires Windows Management Framework 3.0 or later."
-            end
-            pipeline_error(proc)
-        end
-        return filename
-    end
-end
-
-function find_curl()
-    if Sys.isapple() && Sys.isexecutable("/usr/bin/curl")
-        "/usr/bin/curl"
-    elseif Sys.iswindows() && Sys.isexecutable(joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe"))
-        joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe")
-    elseif !Sys.iswindows() && Sys.which("curl") !== nothing
-        "curl"
-    else
-        nothing
-    end
-end
-
-function download_curl(curl_exe::AbstractString, url::AbstractString, filename::AbstractString)
-    err = PipeBuffer()
-    process = run(pipeline(`$curl_exe -s -S -g -L -f -o $filename $url`, stderr=err), wait=false)
-    if !success(process)
-        error_msg = readline(err)
-        @error "Download failed: $error_msg"
-        pipeline_error(process)
-    end
-    return filename
-end
-
 const DOWNLOAD_HOOKS = Callable[]
 
 function download_url(url::AbstractString)
@@ -55,50 +9,27 @@ function download_url(url::AbstractString)
     return url
 end
 
-function download(url::AbstractString, filename::AbstractString)
-    url = download_url(url)
-    curl_exe = find_curl()
-    if curl_exe !== nothing
-        return download_curl(curl_exe, url, filename)
-    elseif Sys.iswindows()
-        return download_powershell(url, filename)
-    elseif Sys.which("wget") !== nothing
-        try
-            run(`wget -O $filename $url`)
-        catch
-            rm(filename, force=true)  # wget always creates a file
-            rethrow()
-        end
-    elseif Sys.which("busybox") !== nothing
-        try
-            run(`busybox wget -O $filename $url`)
-        catch
-            rm(filename, force=true)  # wget always creates a file
-            rethrow()
-        end
-    elseif Sys.which("fetch") !== nothing
-        run(`fetch -f $filename $url`)
-    else
-        error("No download agent available; install curl, wget, busybox or fetch.")
-    end
-    return filename
-end
-
-function download(url::AbstractString)
-    filename = tempname()
-    download(url, filename)
-end
+Downloads() = require(PkgId(
+        UUID((0xf43a241f_c20a_4ad4, 0x852c_f6b1247861c6)),
+        "Downloads",
+    ))
 
 """
-    download(url::AbstractString, [localfile::AbstractString])
+    download(url::AbstractString, [path::AbstractString = tempname()]) -> path
 
-Download a file from the given url, optionally renaming it to the given local file name. If
-no filename is given this will download into a randomly-named file in your temp directory.
-Note that this function relies on the availability of external tools such as `curl`, `wget`
-or `fetch` to download the file and is provided for convenience. For production use or
-situations in which more options are needed, please use a package that provides the desired
-functionality instead.
+Download a file from the given url, saving it to the location `path`, or if not
+specified, a temporary path. Returns the path of the downloaded file.
 
-Returns the filename of the downloaded file.
+!!! note
+    Since Julia 1.6, this function is deprecated and is just a thin wrapper
+    around `Downloads.download`. In new code, you should use that function
+    directly instead of calling this.
 """
-download(url, filename)
+function download(url::AbstractString, path::AbstractString)
+    depwarn("Base.download is deprecated; use Downloads.download instead", :download)
+    invokelatest(Downloads().download, download_url(url), path)
+end
+function download(url::AbstractString)
+    depwarn("Base.download is deprecated; use Downloads.download instead", :download)
+    invokelatest(Downloads().download, download_url(url))
+end

base/sysimg.jl

@@ -55,6 +55,7 @@ let
             :MozillaCACerts_jll,
             :LibCURL_jll,
             :LibCURL,
+            :Downloads,
         ]
 
     maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0)

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/md5

@@ -0,0 +1 @@
+24a8b8fc2398d20c24a13ce73482a3d7

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/sha512

@@ -0,0 +1 @@
+4b652be535dce6a5cf36e546a31d3221f4c2534d1a3ac710f31f9ed2dfbeabd21b36c0186e4abe12cfee992ff557317aefe2bdeb9cb8b1c44da085b030719329

stdlib/.gitignore

@@ -5,3 +5,5 @@
 /Statistics
 /LibCURL-*
 /LibCURL
+/Downloads-*
+/Downloads

stdlib/Downloads.version

@@ -0,0 +1,2 @@
+DOWNLOADS_BRANCH = master
+DOWNLOADS_SHA1 = 1a1d2e0a10209512f5b29e585bfd78e7a47f8f61

stdlib/Makefile

@@ -20,13 +20,15 @@ STDLIBS = Artifacts Base64 CRC32c Dates DelimitedFiles Distributed FileWatching
           SharedArrays Sockets SparseArrays SuiteSparse Test TOML Unicode UUIDs \
           MozillaCACerts_jll LibCURL_jll
 
-STDLIBS_EXT = Pkg Statistics LibCURL
+STDLIBS_EXT = Pkg Statistics LibCURL Downloads
 PKG_GIT_URL := git://github.com/JuliaLang/Pkg.jl.git
 PKG_TAR_URL = https://api.github.com/repos/JuliaLang/Pkg.jl/tarball/$1
 STATISTICS_GIT_URL := git://github.com/JuliaLang/Statistics.jl.git
 STATISTICS_TAR_URL = https://api.github.com/repos/JuliaLang/Statistics.jl/tarball/$1
 LIBCURL_GIT_URL := git://github.com/JuliaWeb/LibCURL.jl.git
 LIBCURL_TAR_URL = https://api.github.com/repos/JuliaWeb/LibCURL.jl/tarball/$1
+DOWNLOADS_GIT_URL := git://github.com/JuliaLang/Downloads.jl.git
+DOWNLOADS_TAR_URL = https://api.github.com/repos/JuliaLang/Downloads.jl/tarball/$1
 
 $(foreach module, $(STDLIBS_EXT), $(eval $(call stdlib-external,$(module),$(shell echo $(module) | tr a-z A-Z))))
 

test/choosetests.jl

@@ -111,7 +111,7 @@ function choosetests(choices = [])
         filter!(x -> (x != "Profile"), tests)
     end
 
-    net_required_for = ["Sockets", "LibGit2", "LibCURL"]
+    net_required_for = ["Sockets", "LibGit2", "LibCURL", "Downloads"]
     net_on = true
     try
         ipa = getipaddr()

test/download.jl

@@ -1,59 +1,6 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# Test that `Base.download_url()` is altered by `Base.DOWNLOAD_HOOKS`.
-let urls = ["http://httpbin.julialang.org/ip", "https://httpbin.julialang.org/ip"]
-    for url in urls
-        @test Base.download_url(url) == url
-    end
-    push!(Base.DOWNLOAD_HOOKS, url->replace(url, r"^http://" => "https://"))
-    for url in urls
-        @test Base.download_url(url) == urls[end]
-    end
-    pop!(Base.DOWNLOAD_HOOKS)
-    for url in urls
-        @test Base.download_url(url) == url
-    end
-end
-
-mktempdir() do temp_dir
-    # Download a file
-    file = joinpath(temp_dir, "ip")
-    @test download("https://httpbin.julialang.org/ip", file) == file
-    @test isfile(file)
-    @test !isempty(read(file))
-    ip = read(file, String)
-
-    # Test download rewrite hook
-    push!(Base.DOWNLOAD_HOOKS, url->replace(url, r"/status/404$" => "/ip"))
-    @test download("https://httpbin.julialang.org/status/404", file) == file
-    @test isfile(file)
-    @test !isempty(read(file))
-    @test ip == read(file, String)
-    pop!(Base.DOWNLOAD_HOOKS)
-
-    # Download an empty file
-    empty_file = joinpath(temp_dir, "empty")
-    @test download("https://httpbin.julialang.org/status/200", empty_file) == empty_file
-
-    # Windows and older versions of curl do not create the empty file (https://github.com/curl/curl/issues/183)
-    @test !isfile(empty_file) || isempty(read(empty_file))
-
-    # Make sure that failed downloads do not leave files around
-    missing_file = joinpath(temp_dir, "missing")
-    @test_throws ProcessFailedException download("https://httpbin.julialang.org/status/404", missing_file)
-    @test !isfile(missing_file)
-
-    # Make sure we properly handle metachar ' on windows with ^ escaping
-    if Sys.iswindows()
-        metachar_file = joinpath(temp_dir, "metachar")
-        Base.download_powershell("https://httpbin.julialang.org/get?test='^'", metachar_file)
-        metachar_string = read(metachar_file, String)
-        m = match(r"\"test\"\s*:\s*\"(.*)\"", metachar_string)
-        @test m.captures[1] == "'^'"
-    end
-
-    # Use a TEST-NET (192.0.2.0/24) address which shouldn't be bound
-    invalid_host_file = joinpath(temp_dir, "invalid_host")
-    @test_throws ProcessFailedException download("http://192.0.2.1", invalid_host_file)
-    @test !isfile(invalid_host_file)
+cmd = `$(Base.julia_cmd()) --depwarn=no --startup-file=no download_exec.jl`
+if !success(pipeline(cmd; stdout=stdout, stderr=stderr))
+    error("download test failed, cmd : $cmd")
 end

test/download_exec.jl

@@ -0,0 +1,56 @@
+# This file is a part of Julia. License is MIT: https://julialang.org/license
+
+module TestDownload
+
+using Test
+
+# Test that `Base.download_url()` is altered by `Base.DOWNLOAD_HOOKS`.
+let urls = ["http://httpbin.julialang.org/ip", "https://httpbin.julialang.org/ip"]
+    for url in urls
+        @test Base.download_url(url) == url
+    end
+    push!(Base.DOWNLOAD_HOOKS, url->replace(url, r"^http://" => "https://"))
+    for url in urls
+        @test Base.download_url(url) == urls[end]
+    end
+    pop!(Base.DOWNLOAD_HOOKS)
+    for url in urls
+        @test Base.download_url(url) == url
+    end
+end
+
+mktempdir() do temp_dir
+    # Download a file
+    file = joinpath(temp_dir, "ip")
+    @test download("https://httpbin.julialang.org/ip", file) == file
+    @test isfile(file)
+    @test !isempty(read(file))
+    ip = read(file, String)
+
+    # Test download rewrite hook
+    push!(Base.DOWNLOAD_HOOKS, url->replace(url, r"/status/404$" => "/ip"))
+    @test download("https://httpbin.julialang.org/status/404", file) == file
+    @test isfile(file)
+    @test !isempty(read(file))
+    @test ip == read(file, String)
+    pop!(Base.DOWNLOAD_HOOKS)
+
+    # Download an empty file
+    empty_file = joinpath(temp_dir, "empty")
+    @test download("https://httpbin.julialang.org/status/200", empty_file) == empty_file
+
+    # Windows and older versions of curl do not create the empty file (https://github.com/curl/curl/issues/183)
+    @test !isfile(empty_file) || isempty(read(empty_file))
+
+    # Make sure that failed downloads do not leave files around
+    missing_file = joinpath(temp_dir, "missing")
+    @test_throws ErrorException download("https://httpbin.julialang.org/status/404", missing_file)
+    @test !isfile(missing_file)
+
+    # Use a TEST-NET (192.0.2.0/24) address which shouldn't be bound
+    invalid_host_file = joinpath(temp_dir, "invalid_host")
+    @test_throws ErrorException download("http://192.0.2.1", invalid_host_file)
+    @test !isfile(invalid_host_file)
+end
+
+end # module

test/precompile.jl

@@ -292,7 +292,7 @@ try
                  :Future, :Libdl, :LinearAlgebra, :Logging, :Mmap, :Printf,
                  :Profile, :Random, :Serialization, :SharedArrays, :SparseArrays, :SuiteSparse, :Test,
                  :Unicode, :REPL, :InteractiveUtils, :Pkg, :LibGit2, :SHA, :UUIDs, :Sockets,
-                 :Statistics, :TOML, :MozillaCACerts_jll, :LibCURL_jll, :LibCURL,]),
+                 :Statistics, :TOML, :MozillaCACerts_jll, :LibCURL_jll, :LibCURL, :Downloads,]),
                 # Plus precompilation module generated at build time
                 let id = Base.PkgId("__PackagePrecompilationStatementModule")
                     Dict(id => Base.module_build_id(Base.root_module(id)))