Pkg client auth: support connecting to authenticated Pkg servers

What's implemented here is support for HTTP authorization with bearer tokens as
standardized in [RFC6750](https://www.rfc-editor.org/rfc/rfc6750.html). This
means that authorized access is accomplished by the client by making an HTTPS
request including a `Authorization: Bearer $access_token` header. The Pkg client
also supports automatic token refresh, since bearer tokens are recommended to be
short-lived (no more than a day). The authorization information is saved locally
in `$(DEPOT_PATH[1])/servers/$server/auth.toml` which is a TOML file with the
following fields:

- `access_token` (REQUIRED): the bearer token used to authorize normal requests
- `expires_at` (OPTIONAL): an absolute expiration time
- `expires_in` (OPTIONAL): a relative expiration time
- `refresh_token` (OPTIONAL): bearer token used to authorize refresh requests
- `refresh_url` (OPTIONAL): URL to fetch new a new token from

The `auth.toml` file may contain other fields (e.g. user name, user email), but
they are ignored by Pkg. The two other fields mentioned in RFC6750 are
`token_type` and `scope`: these are omitted since only tokens of type `Bearer`
are supported currently and the scope is always implicitly to provide access to
Pkg protocol URLs. Pkg servers should, however, not send `auth.toml` files with
`token_type` or `scope` fields, as these names may be used in the future, e.g.
to support other kinds of tokens or to limit the scope of an authorization to a
subset of Pkg protocol URLs.

Initially, the user or user agent (IDE) must acquire a `auth.toml` file and save
it to the correct location. After that, Pkg will determine whether the access
token needs to be refreshed by examining the `expires_at` and/or `exipres_in`
fields of the auth file. The expiration time is the minimum of `expires_at` and
`mtime(auth_file) + expires_in`. When the Pkg client downloads a new `auth.toml`
file, if there is a relative `exipres_in` field, an absolute `exipres_at` value
is computed based on the client's current clock time. This combination of
policies allows expiration to work gracefully even in the presence of clock skew
between the server and the client.

If the access token is expired and there are `refresh_token` and `refresh_url`
fields in `auth.toml`, a new auth file is requested by making a request to
`refresh_url` with an `Authorization: Bearer $refresh_token` header. Pkg will
refuse to make unless `refresh_url` is an HTTPS URL. Note that `refresh_url`
need not be a URL on the Pkg server: token refresh can be handled by separate
server. If the request is successful and the returned `auth.toml` file is a
well-formed TOML file with at least an `access_token` field, it is saved to
`$(DEPOT_PATH[1])/servers/$server/auth.toml`.

Checking for access token expiry and refreshing `auth.toml` is done before each
Pkg client request to a Pkg server, and if the auth file is updated the new
access token is used, so the token should in theory always be up to date.
Practice is different from theory, of course, and if the Pkg server considers
the access token expired, it may return an HTTP 401 Unauthorized response, and
the Pkg client should attempt to refresh the auth token. If, after attempting to
refresh the access token, the server still returns HTTP 401 Unauthorized, the
Pkg client server will present the body of the error response to the user or
user agent (IDE); we'll add a hook to allow the user agent may to handle an auth
failure, e.g. by presenting a login page to get a new auth token.

For testing purposes, the client considers HTTP connections to localhost to be
secure; for any other host it refuses to send access or refresh tokens over a
non-HTTPS connection.

src/PlatformEngines.jl

@@ -4,6 +4,7 @@
 
 module PlatformEngines
 using SHA, Logging
+import ...Pkg: TOML, pkg_server, depots1
 
 export probe_platform_engines!, parse_7z_list, parse_tar_list, verify,
        download_verify, unpack, package, download_verify_unpack,
@@ -588,11 +589,109 @@ function parse_tar_list(output::AbstractString)
     return Sys.iswindows() ? replace.(lines, ['/' => '\\']) : lines
 end
 
+is_secure_url(url::AbstractString) =
+    occursin(r"^(https://|\w+://(127\.0\.0\.1|localhost)(:\d+)?($|/))"i, url)
+
+function get_auth_header(url::AbstractString; verbose::Bool = false)
+    server = pkg_server()
+    server === nothing && return
+    startswith(url, server) || return
+    # find and parse auth file
+    m = match(r"^(\w+)://([^\\/]+)$", server)
+    if m === nothing
+        @warn "malformed Pkg server value" server=server
+        return
+    end
+    proto, host = m.captures
+    auth_file = joinpath(depots1(), "servers", host, "auth.toml")
+    isfile(auth_file) || return
+    # TODO: check for insecure auth file permissions
+    if !is_secure_url(url)
+        @warn "refusing to send auth info over insecure connection" url=url
+        return
+    end
+    # parse the auth file
+    auth_info = try
+        TOML.parsefile(auth_file)
+    catch err
+        @error "malformed auth file" file=auth_file err=err
+        return
+    end
+    # check for an auth token
+    if !haskey(auth_info, "access_token")
+        @warn "auth file without access_token field" file=auth_file
+        return
+    end
+    auth_header = "Authorization: Bearer $(auth_info["access_token"])"
+    # handle token expiration and refresh
+    expires_at = Inf
+    if haskey(auth_info, "expires_at")
+        expires_at = min(expires_at, auth_info["expires_at"]::Integer)
+    end
+    if haskey(auth_info, "expires_in")
+        expires_at = min(expires_at, mtime(auth_file) + auth_info["expires_in"]::Integer)
+    end
+    # if token is good until ten minutes from now, use it
+    time_now = time()
+    if expires_at ≥ time_now + 10*60 # ten minutes
+        return auth_header
+    end
+    if !haskey(auth_info, "refresh_url") || !haskey(auth_info, "refresh_token")
+        if expires_at ≤ time_now
+            @warn "expired auth without refresh keys" file=auth_file
+        end
+        # try it anyway since we can't refresh
+        return auth_header
+    end
+    refresh_url = auth_info["refresh_url"]
+    if !is_secure_url(refresh_url)
+        @warn "ignoring insecure auth refresh URL" url=refresh_url
+        return auth_header
+    end
+    verbose && @info "Refreshing expired auth token..." file=auth_file
+    tmp = tempname()
+    refresh_auth = "Authorization: Bearer $(auth_info["refresh_token"])"
+    try download(refresh_url, tmp, auth_header=refresh_auth, verbose=verbose)
+    catch err
+        @warn "token refresh failure" file=auth_file url=refresh_url err=err
+        rm(tmp, force=true)
+        return
+    end
+    auth_info = try TOML.parsefile(tmp)
+    catch err
+        @warn "discarding malformed auth file" url=refresh_url err=err
+        rm(tmp, force=true)
+        return auth_header
+    end
+    if !haskey(auth_info, "access_token")
+        if haskey(auth_info, "refresh_token")
+            auth_info["refresh_token"] = "*"^64
+        end
+        @warn "discarding auth file without access token" auth=auth_info
+        rm(tmp, force=true)
+        return auth_header
+    end
+    if haskey(auth_info, "expires_in")
+        expires_in = auth_info["expires_in"]
+        if expires_in isa Number
+            expires_at = floor(Int64, time_now + expires_in)
+            # overwrite expires_at (avoids clock skew issues)
+            auth_info["expires_at"] = expires_at
+        end
+    end
+    open(tmp, write=true) do io
+        TOML.print(io, auth_info, sorted=true)
+    end
+    mv(tmp, auth_file, force=true)
+    return "Authorization: Bearer $(auth_info["access_token"])"
+end
+
 """
     download(
         url::AbstractString,
         dest::AbstractString;
         verbose::Bool = false,
+        auth_header::Union{AbstractString, Nothing} = nothing,
     )
 
 Download file located at `url`, store it at `dest`, continuing if `dest`
@@ -602,8 +701,16 @@ function download(
     url::AbstractString,
     dest::AbstractString;
     verbose::Bool = false,
+    auth_header::Union{AbstractString, Nothing} = nothing,
 )
-    download_cmd = gen_download_cmd(url, dest)
+    if auth_header === nothing
+        auth_header = get_auth_header(url, verbose=verbose)
+    end
+    if auth_header === nothing
+        download_cmd = gen_download_cmd(url, dest)
+    else
+        download_cmd = gen_download_cmd(url, dest, auth_header)
+    end
     if verbose
         @info("Downloading $(url) to $(dest)...")
     end