add Downloads stdlib

Make.inc

@@ -49,6 +49,7 @@ USE_SYSTEM_LIBUV:=0
 USE_SYSTEM_UTF8PROC:=0
 USE_SYSTEM_MBEDTLS:=0
 USE_SYSTEM_LIBSSH2:=0
+USE_SYSTEM_NGHTTP2:=0
 USE_SYSTEM_CURL:=0
 USE_SYSTEM_LIBGIT2:=0
 USE_SYSTEM_PATCHELF:=0
@@ -1099,7 +1100,7 @@ USE_BINARYBUILDER ?= 0
 endif
 
 # This is the set of projects that BinaryBuilder dependencies are hooked up for.
-BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
+BB_PROJECTS := OPENBLAS LLVM SUITESPARSE OPENLIBM GMP MBEDTLS LIBSSH2 NGHTTP2 MPFR CURL LIBGIT2 PCRE LIBUV LIBUNWIND DSFMT OBJCONV ZLIB P7ZIP
 define SET_BB_DEFAULT
 # First, check to see if BB is disabled on a global setting
 ifeq ($$(USE_BINARYBUILDER),0)

Makefile

@@ -170,6 +170,7 @@ JL_PRIVATE_LIBS-$(USE_SYSTEM_DSFMT) += libdSFMT
 JL_PRIVATE_LIBS-$(USE_SYSTEM_GMP) += libgmp
 JL_PRIVATE_LIBS-$(USE_SYSTEM_MPFR) += libmpfr
 JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBSSH2) += libssh2
+JL_PRIVATE_LIBS-$(USE_SYSTEM_NGHTTP2) += libnghttp2
 JL_PRIVATE_LIBS-$(USE_SYSTEM_MBEDTLS) += libmbedtls libmbedcrypto libmbedx509
 JL_PRIVATE_LIBS-$(USE_SYSTEM_CURL) += libcurl
 JL_PRIVATE_LIBS-$(USE_SYSTEM_LIBGIT2) += libgit2

NEWS.md

@@ -58,8 +58,9 @@ Build system changes
 
 Library functions
 -----------------
-* The `Base.Grisu` code has been officially removed (float printing was switched to the ryu algorithm code in 1.4)
 
+* The `Base.download` function has been deprecated (silently, by default) in favor of the new `Downloads.download` standard library function ([#37340]).
+* The `Base.Grisu` code has been officially removed (float printing was switched to the ryu algorithm code in 1.4)
 
 New library functions
 ---------------------
@@ -81,6 +82,7 @@ New library features
 
 Standard library changes
 ------------------------
+
 * The `nextprod` function now accepts tuples and other array types for its first argument ([#35791]).
 * The `reverse(A; dims)` function for multidimensional `A` can now reverse multiple dimensions at once
   by passing a tuple for `dims`, and defaults to reversing all dimensions; there is also a multidimensional
@@ -99,6 +101,7 @@ Standard library changes
 * `RegexMatch` objects can now be probed for whether a named capture group exists within it through `haskey()` ([#36717]).
 * For consistency `haskey(r::RegexMatch, i::Integer)` has also been added and returns if the capture group for `i` exists ([#37300]).
 * A new standard library `TOML` has been added for parsing and printing [TOML files](https://toml.io) ([#37034]).
+* A new standard library `Downloads` has been added, which replaces the old `Base.download` function with `Downloads.download`, providing cross-platform, multi-protocol, in-process download functionality implemented with [libcurl](https://curl.haxx.se/libcurl/) ([#37340]).
 * The `Pkg.BinaryPlatforms` module has been moved into `Base` as `Base.BinaryPlatforms` and heavily reworked.
   Applications that want to be compatible with the old API should continue to import `Pkg.BinaryPlatforms`,
   however new users should use `Base.BinaryPlatforms` directly. ([#37320])
@@ -107,6 +110,7 @@ Standard library changes
   all of `Pkg` alongside. ([#37320])
 
 #### LinearAlgebra
+
 * New method `LinearAlgebra.issuccess(::CholeskyPivoted)` for checking whether pivoted Cholesky factorization was successful ([#36002]).
 * `UniformScaling` can now be indexed into using ranges to return dense matrices and vectors ([#24359]).
 * New function `LinearAlgebra.BLAS.get_num_threads()` for getting the number of BLAS threads. ([#36360])

base/download.jl

@@ -1,51 +1,5 @@
 # This file is a part of Julia. License is MIT: https://julialang.org/license
 
-# file downloading
-
-if Sys.iswindows()
-    function download_powershell(url::AbstractString, filename::AbstractString)
-        ps = joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\WindowsPowerShell\\v1.0\\powershell.exe")
-        tls12 = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-        client = "New-Object System.Net.Webclient"
-        # in the following we escape ' with '' (see https://ss64.com/ps/syntax-esc.html)
-        downloadfile = "($client).DownloadFile('$(replace(url, "'" => "''"))', '$(replace(filename, "'" => "''"))')"
-        # PowerShell v3 or later is required for Tls12
-        proc = run(pipeline(`$ps -Version 3 -NoProfile -Command "$tls12; $downloadfile"`; stderr=stderr); wait=false)
-        if !success(proc)
-            if proc.exitcode % Int32 == -393216
-                # appears to be "wrong version" exit code, based on
-                # https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-startup-tasks-common
-                @error "Downloading files requires Windows Management Framework 3.0 or later."
-            end
-            pipeline_error(proc)
-        end
-        return filename
-    end
-end
-
-function find_curl()
-    if Sys.isapple() && Sys.isexecutable("/usr/bin/curl")
-        "/usr/bin/curl"
-    elseif Sys.iswindows() && Sys.isexecutable(joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe"))
-        joinpath(get(ENV, "SYSTEMROOT", "C:\\Windows"), "System32\\curl.exe")
-    elseif !Sys.iswindows() && Sys.which("curl") !== nothing
-        "curl"
-    else
-        nothing
-    end
-end
-
-function download_curl(curl_exe::AbstractString, url::AbstractString, filename::AbstractString)
-    err = PipeBuffer()
-    process = run(pipeline(`$curl_exe -s -S -g -L -f -o $filename $url`, stderr=err), wait=false)
-    if !success(process)
-        error_msg = readline(err)
-        @error "Download failed: $error_msg"
-        pipeline_error(process)
-    end
-    return filename
-end
-
 const DOWNLOAD_HOOKS = Callable[]
 
 function download_url(url::AbstractString)
@@ -55,50 +9,25 @@ function download_url(url::AbstractString)
     return url
 end
 
-function download(url::AbstractString, filename::AbstractString)
-    url = download_url(url)
-    curl_exe = find_curl()
-    if curl_exe !== nothing
-        return download_curl(curl_exe, url, filename)
-    elseif Sys.iswindows()
-        return download_powershell(url, filename)
-    elseif Sys.which("wget") !== nothing
-        try
-            run(`wget -O $filename $url`)
-        catch
-            rm(filename, force=true)  # wget always creates a file
-            rethrow()
-        end
-    elseif Sys.which("busybox") !== nothing
-        try
-            run(`busybox wget -O $filename $url`)
-        catch
-            rm(filename, force=true)  # wget always creates a file
-            rethrow()
-        end
-    elseif Sys.which("fetch") !== nothing
-        run(`fetch -f $filename $url`)
-    else
-        error("No download agent available; install curl, wget, busybox or fetch.")
-    end
-    return filename
-end
-
-function download(url::AbstractString)
-    filename = tempname()
-    download(url, filename)
-end
+Downloads() = require(PkgId(
+        UUID((0xf43a241f_c20a_4ad4, 0x852c_f6b1247861c6)),
+        "Downloads",
+    ))
 
 """
-    download(url::AbstractString, [localfile::AbstractString])
+    download(url::AbstractString, [path::AbstractString = tempname()]) -> path
 
-Download a file from the given url, optionally renaming it to the given local file name. If
-no filename is given this will download into a randomly-named file in your temp directory.
-Note that this function relies on the availability of external tools such as `curl`, `wget`
-or `fetch` to download the file and is provided for convenience. For production use or
-situations in which more options are needed, please use a package that provides the desired
-functionality instead.
+Download a file from the given url, saving it to the location `path`, or if not
+specified, a temporary path. Returns the path of the downloaded file.
 
-Returns the filename of the downloaded file.
+!!! note
+    Since Julia 1.6, this function is deprecated and is just a thin wrapper
+    around `Downloads.download`. In new code, you should use that function
+    directly instead of calling this.
 """
-download(url, filename)
+function download(url::AbstractString, path::AbstractString)
+    invokelatest(Downloads().download, download_url(url), path)
+end
+function download(url::AbstractString)
+    invokelatest(Downloads().download, download_url(url))
+end

base/sysimg.jl

@@ -52,6 +52,10 @@ let
             :Test,
             :REPL,
             :Statistics,
+            :MozillaCACerts_jll,
+            :LibCURL_jll,
+            :LibCURL,
+            :Downloads,
         ]
 
     maxlen = reduce(max, textwidth.(string.(stdlibs)); init=0)

contrib/refresh_bb_tarballs.sh

@@ -12,7 +12,7 @@
 TRIPLETS="i686-linux-gnu x86_64-linux-gnu aarch64-linux-gnu armv7l-linux-gnueabihf powerpc64le-linux-gnu i686-linux-musl x86_64-linux-musl aarch64-linux-musl armv7l-linux-musleabihf x86_64-apple-darwin14 x86_64-unknown-freebsd11.1 i686-w64-mingw32 x86_64-w64-mingw32"
 
 # These are the projects currently using BinaryBuilder; both GCC-expanded and non-GCC-expanded:
-BB_PROJECTS="mbedtls libssh2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv p7zip zlib suitesparse openlibm"
+BB_PROJECTS="mbedtls libssh2 nghttp2 mpfr curl libgit2 pcre libuv unwind osxunwind dsfmt objconv p7zip zlib suitesparse openlibm"
 BB_GCC_EXPANDED_PROJECTS="openblas"
 BB_CXX_EXPANDED_PROJECTS="gmp llvm"
 

deps/Makefile

@@ -111,11 +111,13 @@ ifeq ($(USE_SYSTEM_LIBSSH2), 0)
 DEP_LIBS += libssh2
 endif
 
-ifneq ($(OS), WINNT)
+ifeq ($(USE_SYSTEM_NGHTTP2), 0)
+DEP_LIBS += nghttp2
+endif
+
 ifeq ($(USE_SYSTEM_CURL), 0)
 DEP_LIBS += curl
 endif
-endif
 
 DEP_LIBS += libgit2
 endif # USE_SYSTEM_LIBGIT2
@@ -178,7 +180,7 @@ install: $(addprefix install-, $(DEP_LIBS))
 cleanall: $(addprefix clean-, $(DEP_LIBS))
 distcleanall: $(addprefix distclean-, $(DEP_LIBS))
 	rm -rf $(build_prefix)
-getall: get-llvm get-libuv get-pcre get-openlibm get-dsfmt get-openblas get-lapack get-suitesparse get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-objconv get-mbedtls get-libssh2 get-curl get-libgit2 get-libwhich
+getall: get-llvm get-libuv get-pcre get-openlibm get-dsfmt get-openblas get-lapack get-suitesparse get-unwind get-osxunwind get-gmp get-mpfr get-patchelf get-utf8proc get-objconv get-mbedtls get-libssh2 get-nghttp2 get-curl get-libgit2 get-libwhich
 
 include $(SRCDIR)/llvm.mk
 include $(SRCDIR)/libuv.mk
@@ -195,6 +197,7 @@ include $(SRCDIR)/mpfr.mk
 include $(SRCDIR)/patchelf.mk
 include $(SRCDIR)/mbedtls.mk
 include $(SRCDIR)/libssh2.mk
+include $(SRCDIR)/nghttp2.mk
 include $(SRCDIR)/curl.mk
 include $(SRCDIR)/libgit2.mk
 include $(SRCDIR)/libwhich.mk

deps/Versions.make

@@ -20,12 +20,14 @@ GMP_BB_REL = 1
 MPFR_VER = 4.1.0
 MPFR_BB_REL = 1
 PATCHELF_VER = 0.9
-MBEDTLS_VER = 2.16.0
-MBEDTLS_BB_REL = 1
+MBEDTLS_VER = 2.16.8
+MBEDTLS_BB_REL = 0
 LIBSSH2_VER = 1.9.0
 LIBSSH2_BB_REL = 1
-CURL_VER = 7.66.0
-CURL_BB_REL = 1
+CURL_VER = 7.71.1
+CURL_BB_REL = 0
+NGHTTP2_VER = 1.40.0
+NGHTTP2_BB_REL = 2
 LIBGIT2_VER = 1.0.1
 LIBGIT2_BB_REL = 0
 LIBUV_VER = 1.29.1

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/md5

@@ -0,0 +1 @@
+24a8b8fc2398d20c24a13ce73482a3d7

deps/checksums/Downloads-1a1d2e0a10209512f5b29e585bfd78e7a47f8f61.tar.gz/sha512

@@ -0,0 +1 @@
+4b652be535dce6a5cf36e546a31d3221f4c2534d1a3ac710f31f9ed2dfbeabd21b36c0186e4abe12cfee992ff557317aefe2bdeb9cb8b1c44da085b030719329

deps/checksums/LibCURL-70aa2cc3f8fa4488897136a0a700b5425bfffdfa.tar.gz/md5

@@ -0,0 +1 @@
+829dd514bd1e0d2f9d8cc6c8d5085bba

deps/checksums/LibCURL-70aa2cc3f8fa4488897136a0a700b5425bfffdfa.tar.gz/sha512

@@ -0,0 +1 @@
+080fa389e9a924484aeb50d4bccd314da7d264613d0951f3b8dd53f28f4080abe126cb3960dcb69434a9c6d915866de01977625960117bc5e9d77928efead1cb

deps/checksums/LibCURL.v7.71.1-0.aarch64-linux-gnu.tar.gz/md5

@@ -0,0 +1 @@
+9429a6debbee10b7013ae6741658c2f8

deps/checksums/LibCURL.v7.71.1-0.aarch64-linux-gnu.tar.gz/sha512

@@ -0,0 +1 @@
+b301b37c0d71d13d6d08ea5888f9f62d83c3c9f0be080e5852954e418814c34aa6024c338dd074b51343de5fc4026caf4310c10fb7243da25f2de3554064fb41

deps/checksums/LibCURL.v7.71.1-0.aarch64-linux-musl.tar.gz/md5

@@ -0,0 +1 @@
+ce8f2be6ca2d9b355888b059087c4910

deps/checksums/LibCURL.v7.71.1-0.aarch64-linux-musl.tar.gz/sha512

@@ -0,0 +1 @@
+a672d2aa069a90e941eac69d0aac55e84be479c9f18f03884bb586e558df2a77c4272e926f1e808b8e17bbb2b1eab670d9e1a8057571b776d30fdcc69af88895

deps/checksums/LibCURL.v7.71.1-0.armv7l-linux-gnueabihf.tar.gz/md5

@@ -0,0 +1 @@
+8679fcc7fc6ccfc9689184d8ec3f99a1

deps/checksums/LibCURL.v7.71.1-0.armv7l-linux-gnueabihf.tar.gz/sha512

@@ -0,0 +1 @@
+6257d64d749c1fa518d64f243b65290152026f7f28a3aca2218d72fddf0289e2410d4936cecd9577721bb9c4f664042caf11e5672695cd0eeb1d692517eac05b

deps/checksums/LibCURL.v7.71.1-0.armv7l-linux-musleabihf.tar.gz/md5

@@ -0,0 +1 @@
+99505f07a8be8097535cd11261c3c115

deps/checksums/LibCURL.v7.71.1-0.armv7l-linux-musleabihf.tar.gz/sha512

@@ -0,0 +1 @@
+dbf099ce8a1930fba08ba3369934d6377ef9665794ffe2ae6e89cccd2f58b9983b7070d1a86fc4f85c8b0667e209a498d429f392ee85a23344ba9e2e561ebb4f

deps/checksums/LibCURL.v7.71.1-0.i686-linux-gnu.tar.gz/md5

@@ -0,0 +1 @@
+2efc802e9e6d84f280d3e10f76564082

deps/checksums/LibCURL.v7.71.1-0.i686-linux-gnu.tar.gz/sha512

@@ -0,0 +1 @@
+0099f74b49a5dd53479dce05677219fb31cbdbf3a509234fba7f6f19ccaf1bee6be8e894f849f0130f1ed41b3f00bbcc3a4c8888d934da11774461f5d9799633

deps/checksums/LibCURL.v7.71.1-0.i686-linux-musl.tar.gz/md5

@@ -0,0 +1 @@
+5fc430d5b4dda597f9f739ee33eeb18c

deps/checksums/LibCURL.v7.71.1-0.i686-linux-musl.tar.gz/sha512

@@ -0,0 +1 @@
+bdff1b186e4f6615da45dd4f9ee99ef0868f5a1d6e305fa4c3a57059f0aec101916b97de14cb62eb7e8ba95a3ddb587b452edf63c99994f93bcdd079217d0f3c

deps/checksums/LibCURL.v7.71.1-0.i686-w64-mingw32.tar.gz/md5

@@ -0,0 +1 @@
+5be87ee4e9675d31c3b9fd58cc0789d5

deps/checksums/LibCURL.v7.71.1-0.i686-w64-mingw32.tar.gz/sha512

@@ -0,0 +1 @@
+fce0707d0c42744d4b93fc9ff2ff7ea329d2d89ad597d832e8c4c2b942755f63443d99667040c42659b283070bd69d0572eced0a3f90ac054d9bfd51f42ed611

deps/checksums/LibCURL.v7.71.1-0.powerpc64le-linux-gnu.tar.gz/md5

@@ -0,0 +1 @@
+19c2d84f551de1154a2b7021bb045198

deps/checksums/LibCURL.v7.71.1-0.powerpc64le-linux-gnu.tar.gz/sha512

@@ -0,0 +1 @@
+77bc7838aedd69ea836420585a5b1ad51dd99daa2b22e130ea4f217ca210d8d524511d1b86bc4ff18325c7417fed13c8d05d2e2ffeca81ebc05e24bc66fe62a5

deps/checksums/LibCURL.v7.71.1-0.x86_64-apple-darwin14.tar.gz/md5

@@ -0,0 +1 @@
+756ae7b6f45d27ca91f0d3f0b1b912ed

deps/checksums/LibCURL.v7.71.1-0.x86_64-apple-darwin14.tar.gz/sha512

@@ -0,0 +1 @@
+ec0442d72fa08d589097521b82360f154d54c51f37702b887ee9a8d9e36c189d66b146f7a7d981c0483def3a5b0cb2f5f1363b56f1cf911a53b62e33c776a2c3

deps/checksums/LibCURL.v7.71.1-0.x86_64-linux-gnu.tar.gz/md5

@@ -0,0 +1 @@
+7fa98a535ebaeac2632befb6d1fbbaae

deps/checksums/LibCURL.v7.71.1-0.x86_64-linux-gnu.tar.gz/sha512

@@ -0,0 +1 @@
+6711f413d3d1f73f388b7605fe4620d420a777b0f450ca1c0c153ed9d2033f6fa15229ce113a6a3da6f5454930603cb4d80ae35c90e8b1eb08fa4e69069acd2a

deps/checksums/LibCURL.v7.71.1-0.x86_64-linux-musl.tar.gz/md5

@@ -0,0 +1 @@
+bc4829f55dca10f7a7f690448d7c21e2

deps/checksums/LibCURL.v7.71.1-0.x86_64-linux-musl.tar.gz/sha512

@@ -0,0 +1 @@
+e5e0c585a61566053238c78e6c049df34d6001cec6a1d1b79609f28c91bd2bc03eb852bd99540bd475d8fa603cafd113ab8ef8ae0d78910c1e800136de40ad77

deps/checksums/LibCURL.v7.71.1-0.x86_64-unknown-freebsd11.1.tar.gz/md5

@@ -0,0 +1 @@
+d20046a11ae0e75b23a44425f5d55437

deps/checksums/LibCURL.v7.71.1-0.x86_64-unknown-freebsd11.1.tar.gz/sha512

@@ -0,0 +1 @@
+3478d6259bc7846662a49e95b2c37c452e635e47717ed4132a1a44bc2bb292603b49c895f75e807d09ed3d4f946694f10c9108dd313deb8ede8b3ea2f1f9779e

deps/checksums/LibCURL.v7.71.1-0.x86_64-w64-mingw32.tar.gz/md5

@@ -0,0 +1 @@
+4e15d303c24763383bb6ac6f9f5241e2

deps/checksums/LibCURL.v7.71.1-0.x86_64-w64-mingw32.tar.gz/sha512

@@ -0,0 +1 @@
+658119a4a842672d451dc167d01d7629a0726e8be0520d288977141c35a4e8d40b91398d0d7abebb63d547d4c1db8c6c1343f85dbf3a0bd8d6d3d51435f1922e