diff --git a/quickget b/quickget index b4b71541b2..19215e5db9 100755 --- a/quickget +++ b/quickget @@ -3339,6 +3339,12 @@ handle_curl_error() { 7) echo "Failed to contact Microsoft servers! Is there an Internet connection or is the server down?" ;; + 8) + echo "Microsoft servers returned a malformed HTTP response!" + ;; + 22) + echo "Microsoft servers returned a failing HTTP status code!" + ;; 23) echo "Failed at writing Windows media to disk! Out of disk space or permission error? Exiting..." return "$fatal_error_action" @@ -3350,15 +3356,15 @@ handle_curl_error() { 36) echo "Failed to continue earlier download!" ;; - 22) - echo "Microsoft servers returned failing HTTP status code!" + 63) + echo "Microsoft servers returned an unexpectedly large response!" ;; # POSIX defines exit statuses 1-125 as usable by us # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02 $((error_code <= 125))) - # Must be some other server error (possibly with this specific request/file) - # This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and HTTP(S) request, using only the curl features we're using, and a sane build - echo "Server returned an error status!" + # Must be some other server or network error (possibly with this specific request/file) + # This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and an HTTP(S) request, using only the curl features we're using, and a sane build + echo "Miscellaneous server or network error!" ;; 126 | 127) echo "Curl command not found! Please install curl and try again. Exiting..." @@ -3386,18 +3392,63 @@ handle_curl_error() { return 1 } -function download_windows-server() { - # Download enterprise evaluation windows versions +function curl_windows() { + local part_ext=".PART" + local vm_path="$1" + local out_file="$2" + local tls_version="$3" + local url="$4" + + mkdir -p "${vm_path}" + + real_file="${vm_path}/${out_file}" + part_file="${vm_path}/${out_file}${part_ext}" + + # --location: Microsoft likes to change which endpoint these downloads are stored on but is usually kind enough to add redirects + # --fail: Return an error on server errors where the HTTP response code is 400 or greater + curl --progress-bar --location --output "${part_file}" --continue-at - --max-filesize 10G --fail --proto =https "--tlsv$tls_version" --http1.1 -- "$url" || { + error_code=$? + handle_curl_error "$error_code" + error_action=$? + + # Clean up and make sure a future resume doesn't happen from a bad download resume file + if [ -f "${part_file}" ]; then + # If file is empty, bad HTTP code, or bad download resume file + if [ ! -s "${part_file}" ] || [ "$error_code" = 22 ] || [ "$error_code" = 36 ]; then + echo "- Deleting failed download..." + rm -f "${part_file}" + fi + fi + + return "$error_action" + } + + # Full downloaded succeeded + mv "${part_file}" "${real_file}" +} + +function download_windows_server() { + # Copyright (C) 2024 Elliot Killick + # This function is adapted from the Mido project: + # https://github.com/ElliotKillick/Mido + + # Download enterprise evaluation Windows versions local windows_version="$1" local enterprise_type="$2" local url="https://www.microsoft.com/en-us/evalcenter/download-$windows_version" - local iso_download_page_html="$(curl --silent --location --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { + local iso_download_page_html="$(curl --silent --location --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { handle_curl_error $? return $? } + if ! [ "$iso_download_page_html" ]; then + # This should only happen if there's been some change to where this download page is located + echo " - Windows server download page gave us an empty response" + return 1 + fi + local CULTURE="" local COUNTRY="" local PRETTY_RELEASE="" @@ -3408,7 +3459,6 @@ function download_windows-server() { *) PRETTY_RELEASE="$RELEASE";; esac - case "$LANG" in "English (Great Britain)") CULTURE="en-gb" @@ -3448,40 +3498,53 @@ function download_windows-server() { COUNTRY="US";; esac - iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=$CULTURE&country=$COUNTRY" | head -c 1024)" + iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=$CULTURE&country=$COUNTRY")" || { + # This should only happen if there's been some change to the download endpoint web address + echo "- Windows server download page gave us no download link" + return 1 + } + + # Limit untrusted size for input validation + iso_download_links="$(echo "$iso_download_links" | head -c 1024)" case "$enterprise_type" in + # Select x64 download link + "enterprise") iso_download_link=$(echo "$iso_download_links" | head -n 2 | tail -n 1) ;; # Select x64 LTSC download link "ltsc") iso_download_link=$(echo "$iso_download_links" | head -n 4 | tail -n 1) ;; *) iso_download_link="$iso_download_links" ;; esac - iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" + # Follow redirect so proceeding log message is useful + # This is a request we make this Fido doesn't + # We don't need to set "--max-filesize" here because this is a HEAD request and the output is to /dev/null anyway + iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" || { + # This should only happen if the Microsoft servers are down + handle_curl_error $? + return $? + } + + # Limit untrusted size for input validation + iso_download_link="$(echo "$iso_download_link" | head -c 1024)" - if [ "${COUNTRY}" != "US" ]; then - echo Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} in "${LANG}" from "$iso_download_link" - else - echo Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} from "$iso_download_link" - fi + echo "Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} (${LANG}): $iso_download_link" + + # Use highest TLS version for endpoints that support it + case "$iso_download_link" in + "https://download.microsoft.com"*) tls_version="1.2" ;; + *) tls_version="1.3" ;; + esac + # Download ISO FILE_NAME="${iso_download_link##*/}" - web_get "$iso_download_link" "${VM_PATH}" "${FILE_NAME}" + curl_windows "${VM_PATH}" "${FILE_NAME}" "$tls_version" "$iso_download_link" OS="windows-server" } -function download_windows() { +function download_windows_workstation() { + # This function is adapted from the Mido project: + # https://github.com/ElliotKillick/Mido # Download newer consumer Windows versions from behind gated Microsoft API - # This function aims to precisely emulate what Fido does down to the URL requests and HTTP headers (exceptions: updated user agent and referer adapts to Windows version instead of always being "windows11") but written in POSIX sh (with coreutils) and curl instead of PowerShell (also simplified to greatly reduce attack surface) - # However, differences such as the order of HTTP headers and TLS stacks (could be used to do TLS fingerprinting) still exist - # - # Command translated: ./Fido -Win 10 -Lang English -Verbose - # "English" = "English (United States)" (as opposed to the default "English (International)") - # For testing Fido, replace all "https://" with "http://" and remove all instances of "-MaximumRedirection 0" (to allow redirection of HTTP traffic to HTTPS) so HTTP requests can easily be inspected in Wireshark - # Fido (command-line only) works under PowerShell for Linux if that makes it easier for you - # UPDATE: Fido v1.4.2+ no longer works without being edited on Linux due to these issues on the Fido GitHub repo (and possibly others after these): #56 and #58 - # - # If this function in Mido fails to work for you then please test with the Fido script before creating an issue because we basically just copy what Fido does exactly: - # https://github.com/pbatard/Fido # Either 8, 10, or 11 local windows_version="$1" @@ -3500,13 +3563,11 @@ function download_windows() { # This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically # Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden # Remove "Accept" header that curl sends by default - local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { + local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { handle_curl_error $? return $? } - # Limit untrusted size for input validation - iso_download_page_html="$(echo "$iso_download_page_html" | head -c 102400)" # tr: Filter for only numerics to prevent HTTP parameter injection # head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407 local product_edition_id="$(echo "$iso_download_page_html" | grep -Eo '