From 9d2c8c9584b59e97cc7142f2c64d06afae7d9dc6 Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Tue, 16 Apr 2024 17:11:17 +0100 Subject: [PATCH 1/7] refactor: update handle_curl_error() based Mido --- quickget | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/quickget b/quickget index b4b71541b2..2f6d4b173f 100755 --- a/quickget +++ b/quickget @@ -3339,6 +3339,12 @@ handle_curl_error() { 7) echo "Failed to contact Microsoft servers! Is there an Internet connection or is the server down?" ;; + 8) + echo "Microsoft servers returned a malformed HTTP response!" + ;; + 22) + echo "Microsoft servers returned a failing HTTP status code!" + ;; 23) echo "Failed at writing Windows media to disk! Out of disk space or permission error? Exiting..." return "$fatal_error_action" @@ -3350,15 +3356,15 @@ handle_curl_error() { 36) echo "Failed to continue earlier download!" ;; - 22) - echo "Microsoft servers returned failing HTTP status code!" + 63) + echo "Microsoft servers returned an unexpectedly large response!" ;; # POSIX defines exit statuses 1-125 as usable by us # https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02 $((error_code <= 125))) - # Must be some other server error (possibly with this specific request/file) - # This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and HTTP(S) request, using only the curl features we're using, and a sane build - echo "Server returned an error status!" + # Must be some other server or network error (possibly with this specific request/file) + # This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and an HTTP(S) request, using only the curl features we're using, and a sane build + echo "Miscellaneous server or network error!" ;; 126 | 127) echo "Curl command not found! Please install curl and try again. Exiting..." From b51dd788aa74ea1ade9e704846ffffff73023b1f Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Tue, 16 Apr 2024 19:12:54 +0100 Subject: [PATCH 2/7] feat: add curl_windows() from Mido --- quickget | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/quickget b/quickget index 2f6d4b173f..8222543968 100755 --- a/quickget +++ b/quickget @@ -3392,6 +3392,41 @@ handle_curl_error() { return 1 } +function curl_windows() { + local part_ext=".PART" + local vm_path="$1" + local out_file="$2" + local tls_version="$3" + local url="$4" + + mkdir -p "${vm_path}" + + real_file="${vm_path}/${out_file}" + part_file="${vm_path}/${out_file}${part_ext}" + + # --location: Microsoft likes to change which endpoint these downloads are stored on but is usually kind enough to add redirects + # --fail: Return an error on server errors where the HTTP response code is 400 or greater + curl --progress-bar --location --output "${part_file}" --continue-at - --max-filesize 10G --fail --proto =https "--tlsv$tls_version" --http1.1 -- "$url" || { + error_code=$? + handle_curl_error "$error_code" + error_action=$? + + # Clean up and make sure a future resume doesn't happen from a bad download resume file + if [ -f "${part_file}" ]; then + # If file is empty, bad HTTP code, or bad download resume file + if [ ! -s "${part_file}" ] || [ "$error_code" = 22 ] || [ "$error_code" = 36 ]; then + echo "- Deleting failed download..." + rm -f "${part_file}" + fi + fi + + return "$error_action" + } + + # Full downloaded succeeded + mv "${part_file}" "${real_file}" +} + function download_windows-server() { # Download enterprise evaluation windows versions local windows_version="$1" From 12a30698f737c57ae8d97cc0eae03534a5490f5a Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Tue, 16 Apr 2024 19:14:37 +0100 Subject: [PATCH 3/7] refactor: rename download_windows functions --- quickget | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/quickget b/quickget index 8222543968..de65f91a97 100755 --- a/quickget +++ b/quickget @@ -3427,7 +3427,7 @@ function curl_windows() { mv "${part_file}" "${real_file}" } -function download_windows-server() { +function download_windows_server() { # Download enterprise evaluation windows versions local windows_version="$1" local enterprise_type="$2" @@ -3510,7 +3510,7 @@ function download_windows-server() { OS="windows-server" } -function download_windows() { +function download_windows_workstation() { # Download newer consumer Windows versions from behind gated Microsoft API # This function aims to precisely emulate what Fido does down to the URL requests and HTTP headers (exceptions: updated user agent and referer adapts to Windows version instead of always being "windows11") but written in POSIX sh (with coreutils) and curl instead of PowerShell (also simplified to greatly reduce attack surface) # However, differences such as the order of HTTP headers and TLS stacks (could be used to do TLS fingerprinting) still exist @@ -3659,11 +3659,11 @@ function download_windows() { function get_windows() { if [ "${RELEASE}" == "10-ltsc" ]; then - download_windows-server windows-10-enterprise ltsc + download_windows_workstation windows-10-enterprise ltsc elif [ "${OS}" == "windows-server" ]; then - download_windows-server windows-server-${RELEASE} + download_windows_server "windows-server-${RELEASE}" else - download_windows "${RELEASE}" + download_windows_workstation "${RELEASE}" fi if [ "${download_iso}" == 'on' ]; then From 7c7a7b26be358b2d4451b83447d208dc1e9bb053 Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Tue, 16 Apr 2024 19:15:42 +0100 Subject: [PATCH 4/7] docs: update downloads_windows function attributions --- quickget | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/quickget b/quickget index de65f91a97..341d0a8b37 100755 --- a/quickget +++ b/quickget @@ -3428,7 +3428,11 @@ function curl_windows() { } function download_windows_server() { - # Download enterprise evaluation windows versions + # Copyright (C) 2024 Elliot Killick + # This function is adapted from the Mido project: + # https://github.com/ElliotKillick/Mido + + # Download enterprise evaluation Windows versions local windows_version="$1" local enterprise_type="$2" @@ -3511,18 +3515,9 @@ function download_windows_server() { } function download_windows_workstation() { + # This function is adapted from the Mido project: + # https://github.com/ElliotKillick/Mido # Download newer consumer Windows versions from behind gated Microsoft API - # This function aims to precisely emulate what Fido does down to the URL requests and HTTP headers (exceptions: updated user agent and referer adapts to Windows version instead of always being "windows11") but written in POSIX sh (with coreutils) and curl instead of PowerShell (also simplified to greatly reduce attack surface) - # However, differences such as the order of HTTP headers and TLS stacks (could be used to do TLS fingerprinting) still exist - # - # Command translated: ./Fido -Win 10 -Lang English -Verbose - # "English" = "English (United States)" (as opposed to the default "English (International)") - # For testing Fido, replace all "https://" with "http://" and remove all instances of "-MaximumRedirection 0" (to allow redirection of HTTP traffic to HTTPS) so HTTP requests can easily be inspected in Wireshark - # Fido (command-line only) works under PowerShell for Linux if that makes it easier for you - # UPDATE: Fido v1.4.2+ no longer works without being edited on Linux due to these issues on the Fido GitHub repo (and possibly others after these): #56 and #58 - # - # If this function in Mido fails to work for you then please test with the Fido script before creating an issue because we basically just copy what Fido does exactly: - # https://github.com/pbatard/Fido # Either 8, 10, or 11 local windows_version="$1" From 5d278b3e3ce79c359ec751b01ff2f6630042d0ac Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Tue, 16 Apr 2024 19:18:15 +0100 Subject: [PATCH 5/7] refactor: update download_windows_server() based on Mido --- quickget | 46 +++++++++++++++++++++++++++++++++++++--------- 1 file changed, 37 insertions(+), 9 deletions(-) diff --git a/quickget b/quickget index 341d0a8b37..6ac2435694 100755 --- a/quickget +++ b/quickget @@ -3438,11 +3438,17 @@ function download_windows_server() { local url="https://www.microsoft.com/en-us/evalcenter/download-$windows_version" - local iso_download_page_html="$(curl --silent --location --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { + local iso_download_page_html="$(curl --silent --location --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { handle_curl_error $? return $? } + if ! [ "$iso_download_page_html" ]; then + # This should only happen if there's been some change to where this download page is located + echo " - Windows server download page gave us an empty response" + return 1 + fi + local CULTURE="" local COUNTRY="" local PRETTY_RELEASE="" @@ -3493,24 +3499,46 @@ function download_windows_server() { COUNTRY="US";; esac - iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=$CULTURE&country=$COUNTRY" | head -c 1024)" + iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=$CULTURE&country=$COUNTRY")" || { + # This should only happen if there's been some change to the download endpoint web address + echo "- Windows server download page gave us no download link" + return 1 + } + + # Limit untrusted size for input validation + iso_download_links="$(echo "$iso_download_links" | head -c 1024)" case "$enterprise_type" in + # Select x64 download link + "enterprise") iso_download_link=$(echo "$iso_download_links" | head -n 2 | tail -n 1) ;; # Select x64 LTSC download link "ltsc") iso_download_link=$(echo "$iso_download_links" | head -n 4 | tail -n 1) ;; *) iso_download_link="$iso_download_links" ;; esac - iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" + # Follow redirect so proceeding log message is useful + # This is a request we make this Fido doesn't + # We don't need to set "--max-filesize" here because this is a HEAD request and the output is to /dev/null anyway + iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" || { + # This should only happen if the Microsoft servers are down + handle_curl_error $? + return $? + } - if [ "${COUNTRY}" != "US" ]; then - echo Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} in "${LANG}" from "$iso_download_link" - else - echo Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} from "$iso_download_link" - fi + # Limit untrusted size for input validation + iso_download_link="$(echo "$iso_download_link" | head -c 1024)" + + echo "Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} (${LANG}): $iso_download_link" + # Use highest TLS version for endpoints that support it + case "$iso_download_link" in + "https://download.microsoft.com"*) tls_version="1.2" ;; + *) tls_version="1.3" ;; + esac + + # Download ISO FILE_NAME="${iso_download_link##*/}" - web_get "$iso_download_link" "${VM_PATH}" "${FILE_NAME}" + curl_windows "${VM_PATH}" "${FILE_NAME}" "$tls_version" "$iso_download_link" OS="windows-server" } From 3ffbe2c3c070256536041722a7bb85c5bb2eeada Mon Sep 17 00:00:00 2001 From: Martin Wimpress Date: Tue, 16 Apr 2024 19:19:21 +0100 Subject: [PATCH 6/7] refactor: update download_windows_workstation() based on Mido --- quickget | 51 ++++++++++++++++++--------------------------------- 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/quickget b/quickget index 6ac2435694..ce87708b4a 100755 --- a/quickget +++ b/quickget @@ -3564,13 +3564,11 @@ function download_windows_workstation() { # This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically # Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden # Remove "Accept" header that curl sends by default - local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { + local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || { handle_curl_error $? return $? } - # Limit untrusted size for input validation - iso_download_page_html="$(echo "$iso_download_page_html" | head -c 102400)" # tr: Filter for only numerics to prevent HTTP parameter injection # head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407 local product_edition_id="$(echo "$iso_download_page_html" | grep -Eo '