Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Refactor Windows downloading based on Mido #1042

Merged
merged 7 commits into from
Apr 16, 2024
Merged

Refactor Windows downloading based on Mido #1042

Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
9d2c8c9
refactor: update handle_curl_error() based Mido
flexiondotorg Apr 16, 2024
b51dd78
feat: add curl_windows() from Mido
flexiondotorg Apr 16, 2024
12a3069
refactor: rename download_windows functions
flexiondotorg Apr 16, 2024
7c7a7b2
docs: update downloads_windows function attributions
flexiondotorg Apr 16, 2024
5d278b3
refactor: update download_windows_server() based on Mido
flexiondotorg Apr 16, 2024
3ffbe2c
refactor: update download_windows_workstation() based on Mido
flexiondotorg Apr 16, 2024
9a5036e
chore: white space clean up
flexiondotorg Apr 16, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 113 additions & 65 deletions quickget
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3339,6 +3339,12 @@ handle_curl_error() {
7)
echo "Failed to contact Microsoft servers! Is there an Internet connection or is the server down?"
;;
8)
echo "Microsoft servers returned a malformed HTTP response!"
;;
22)
echo "Microsoft servers returned a failing HTTP status code!"
;;
23)
echo "Failed at writing Windows media to disk! Out of disk space or permission error? Exiting..."
return "$fatal_error_action"
Expand All @@ -3350,15 +3356,15 @@ handle_curl_error() {
36)
echo "Failed to continue earlier download!"
;;
22)
echo "Microsoft servers returned failing HTTP status code!"
63)
echo "Microsoft servers returned an unexpectedly large response!"
;;
# POSIX defines exit statuses 1-125 as usable by us
# https://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_08_02
$((error_code <= 125)))
# Must be some other server error (possibly with this specific request/file)
# This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and HTTP(S) request, using only the curl features we're using, and a sane build
echo "Server returned an error status!"
# Must be some other server or network error (possibly with this specific request/file)
# This is when accounting for all possible errors in the curl manual assuming a correctly formed curl command and an HTTP(S) request, using only the curl features we're using, and a sane build
echo "Miscellaneous server or network error!"
;;
126 | 127)
echo "Curl command not found! Please install curl and try again. Exiting..."
Expand Down Expand Up @@ -3386,18 +3392,63 @@ handle_curl_error() {
return 1
}

function download_windows-server() {
# Download enterprise evaluation windows versions
function curl_windows() {
local part_ext=".PART"
local vm_path="$1"
local out_file="$2"
local tls_version="$3"
local url="$4"

mkdir -p "${vm_path}"

real_file="${vm_path}/${out_file}"
part_file="${vm_path}/${out_file}${part_ext}"

# --location: Microsoft likes to change which endpoint these downloads are stored on but is usually kind enough to add redirects
# --fail: Return an error on server errors where the HTTP response code is 400 or greater
curl --progress-bar --location --output "${part_file}" --continue-at - --max-filesize 10G --fail --proto =https "--tlsv$tls_version" --http1.1 -- "$url" || {
error_code=$?
handle_curl_error "$error_code"
error_action=$?

# Clean up and make sure a future resume doesn't happen from a bad download resume file
if [ -f "${part_file}" ]; then
# If file is empty, bad HTTP code, or bad download resume file
if [ ! -s "${part_file}" ] || [ "$error_code" = 22 ] || [ "$error_code" = 36 ]; then
echo "- Deleting failed download..."
rm -f "${part_file}"
fi
fi

return "$error_action"
}

# Full downloaded succeeded
mv "${part_file}" "${real_file}"
}

function download_windows_server() {
# Copyright (C) 2024 Elliot Killick <contact@elliotkillick.com>
# This function is adapted from the Mido project:
# https://github.com/ElliotKillick/Mido

# Download enterprise evaluation Windows versions
local windows_version="$1"
local enterprise_type="$2"

local url="https://www.microsoft.com/en-us/evalcenter/download-$windows_version"

local iso_download_page_html="$(curl --silent --location --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
local iso_download_page_html="$(curl --silent --location --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
handle_curl_error $?
return $?
}

if ! [ "$iso_download_page_html" ]; then
# This should only happen if there's been some change to where this download page is located
echo " - Windows server download page gave us an empty response"
return 1
fi

local CULTURE=""
local COUNTRY=""
local PRETTY_RELEASE=""
Expand All @@ -3408,7 +3459,6 @@ function download_windows-server() {
*) PRETTY_RELEASE="$RELEASE";;
esac


case "$LANG" in
"English (Great Britain)")
CULTURE="en-gb"
Expand Down Expand Up @@ -3448,40 +3498,53 @@ function download_windows-server() {
COUNTRY="US";;
esac

iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=$CULTURE&country=$COUNTRY" | head -c 1024)"
iso_download_links="$(echo "$iso_download_page_html" | grep -o "https://go.microsoft.com/fwlink/p/?LinkID=[0-9]\+&clcid=0x[0-9a-z]\+&culture=$CULTURE&country=$COUNTRY")" || {
# This should only happen if there's been some change to the download endpoint web address
echo "- Windows server download page gave us no download link"
return 1
}

# Limit untrusted size for input validation
iso_download_links="$(echo "$iso_download_links" | head -c 1024)"

case "$enterprise_type" in
# Select x64 download link
"enterprise") iso_download_link=$(echo "$iso_download_links" | head -n 2 | tail -n 1) ;;
# Select x64 LTSC download link
"ltsc") iso_download_link=$(echo "$iso_download_links" | head -n 4 | tail -n 1) ;;
*) iso_download_link="$iso_download_links" ;;
esac

iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")"
# Follow redirect so proceeding log message is useful
# This is a request we make this Fido doesn't
# We don't need to set "--max-filesize" here because this is a HEAD request and the output is to /dev/null anyway
iso_download_link="$(curl --silent --location --output /dev/null --silent --write-out "%{url_effective}" --head --fail --proto =https --tlsv1.2 --http1.1 -- "$iso_download_link")" || {
# This should only happen if the Microsoft servers are down
handle_curl_error $?
return $?
}

# Limit untrusted size for input validation
iso_download_link="$(echo "$iso_download_link" | head -c 1024)"

if [ "${COUNTRY}" != "US" ]; then
echo Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} in "${LANG}" from "$iso_download_link"
else
echo Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} from "$iso_download_link"
fi
echo "Downloading $(pretty_name "${OS}") ${PRETTY_RELEASE} (${LANG}): $iso_download_link"

# Use highest TLS version for endpoints that support it
case "$iso_download_link" in
"https://download.microsoft.com"*) tls_version="1.2" ;;
*) tls_version="1.3" ;;
esac

# Download ISO
FILE_NAME="${iso_download_link##*/}"
web_get "$iso_download_link" "${VM_PATH}" "${FILE_NAME}"
curl_windows "${VM_PATH}" "${FILE_NAME}" "$tls_version" "$iso_download_link"
OS="windows-server"
}

function download_windows() {
function download_windows_workstation() {
# This function is adapted from the Mido project:
# https://github.com/ElliotKillick/Mido
# Download newer consumer Windows versions from behind gated Microsoft API
# This function aims to precisely emulate what Fido does down to the URL requests and HTTP headers (exceptions: updated user agent and referer adapts to Windows version instead of always being "windows11") but written in POSIX sh (with coreutils) and curl instead of PowerShell (also simplified to greatly reduce attack surface)
# However, differences such as the order of HTTP headers and TLS stacks (could be used to do TLS fingerprinting) still exist
#
# Command translated: ./Fido -Win 10 -Lang English -Verbose
# "English" = "English (United States)" (as opposed to the default "English (International)")
# For testing Fido, replace all "https://" with "http://" and remove all instances of "-MaximumRedirection 0" (to allow redirection of HTTP traffic to HTTPS) so HTTP requests can easily be inspected in Wireshark
# Fido (command-line only) works under PowerShell for Linux if that makes it easier for you
# UPDATE: Fido v1.4.2+ no longer works without being edited on Linux due to these issues on the Fido GitHub repo (and possibly others after these): #56 and #58
#
# If this function in Mido fails to work for you then please test with the Fido script before creating an issue because we basically just copy what Fido does exactly:
# https://github.com/pbatard/Fido

# Either 8, 10, or 11
local windows_version="$1"
Expand All @@ -3500,22 +3563,20 @@ function download_windows() {
# This is the *only* request we make that Fido doesn't. Fido manually maintains a list of all the Windows release/edition product edition IDs in its script (see: $WindowsVersions array). This is helpful for downloading older releases (e.g. Windows 10 1909, 21H1, etc.) but we always want to get the newest release which is why we get this value dynamically
# Also, keeping a "$WindowsVersions" array like Fido does would be way too much of a maintenance burden
# Remove "Accept" header that curl sends by default
local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
local iso_download_page_html="$(curl --silent --user-agent "$user_agent" --header "Accept:" --max-filesize 1M --fail --proto =https --tlsv1.2 --http1.1 -- "$url")" || {
handle_curl_error $?
return $?
}

# Limit untrusted size for input validation
iso_download_page_html="$(echo "$iso_download_page_html" | head -c 102400)"
# tr: Filter for only numerics to prevent HTTP parameter injection
# head -c was recently added to POSIX: https://austingroupbugs.net/view.php?id=407
local product_edition_id="$(echo "$iso_download_page_html" | grep -Eo '<option value="[0-9]+">Windows' | cut -d '"' -f 2 | head -n 1 | tr -cd '0-9' | head -c 16)"
# echo " - Product edition ID: $product_edition_id"

# Permit Session ID
# "org_id" is always the same value
curl --silent --output /dev/null --user-agent "$user_agent" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=$session_id" || {
# This should only happen if there's been some change to how this API works (copy whatever fix Fido implements)
curl --silent --output /dev/null --user-agent "$user_agent" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://vlscppe.microsoft.com/tags?org_id=y6jn8c31&session_id=$session_id" || {
# This should only happen if there's been some change to how this API works
handle_curl_error $?
return $?
}
Expand All @@ -3527,7 +3588,7 @@ function download_windows() {
# SKU ID: This specifies the language of the ISO. We always use "English (United States)", however, the SKU for this changes with each Windows release
# We must make this request so our next one will be allowed
# --data "" is required otherwise no "Content-Length" header will be sent causing HTTP response "411 Length Required"
local language_skuid_table_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=a8f8f489-4c7f-463a-9ca6-5cff94d8d041&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=getskuinformationbyproductedition&sessionId=$session_id&productEditionId=$product_edition_id&sdVersion=2")" || {
local language_skuid_table_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --header "Accept:" --max-filesize 10K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=a8f8f489-4c7f-463a-9ca6-5cff94d8d041&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=getskuinformationbyproductedition&sessionId=$session_id&productEditionId=$product_edition_id&sdVersion=2")" || {
handle_curl_error $?
return $?
}
Expand All @@ -3542,35 +3603,21 @@ function download_windows() {
# Get ISO download link
# If any request is going to be blocked by Microsoft it's always this last one (the previous requests always seem to succeed)
# --referer: Required by Microsoft servers to allow request
local iso_download_link_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --referer "$url" --header "Accept:" --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=6e2a1789-ef16-4f27-a296-74ef7ef5d96b&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=GetProductDownloadLinksBySku&sessionId=$session_id&skuId=$sku_id&language=English&sdVersion=2")" || {
# This should only happen if there's been some change to how this API works
handle_curl_error $?
return $?
}
local iso_download_link_html="$(curl --silent --request POST --user-agent "$user_agent" --data "" --referer "$url" --header "Accept:" --max-filesize 100K --fail --proto =https --tlsv1.2 --http1.1 -- "https://www.microsoft.com/en-US/api/controls/contentinclude/html?pageId=6e2a1789-ef16-4f27-a296-74ef7ef5d96b&host=www.microsoft.com&segments=software-download,$url_segment_parameter&query=&action=GetProductDownloadLinksBySku&sessionId=$session_id&skuId=$sku_id&language=English&sdVersion=2")"

local failed=0

if [ "${LANG}" == "English (United States)" ]; then
LANG="English"
fi

local HASH=$(echo "$iso_download_link_html" | sed 's/<tr><td>/\n<tr><td>/g' | grep "$LANG 64-bit" | grep -o -P '(?<=</td><td>).*(?=</td></tr>)')

# Limit untrusted size for input validation
iso_download_link_html="$(echo "$iso_download_link_html" | head -c 4096)"

if ! [ "$iso_download_link_html" ]; then
# This should only happen if there's been some change to how this API works
echo " - Microsoft servers gave us an empty response to our request for an automated download."
failed=1
fi

if echo "$iso_download_link_html" | grep -q "We are unable to complete your request at this time."; then
echo " - Microsoft blocked the automated download request based on your IP address."
if [ "${show_iso_url}" == 'on' ] || [ "${test_iso_url}" == 'on' ]; then
echo " - Failed to get URL: Microsoft blocked the automated download request based on your IP address."
exit 1
fi
echo " - Microsoft blocked the automated download request based on your IP address."
failed=1
fi

Expand All @@ -3594,35 +3641,36 @@ function download_windows() {
fi

if [ "${show_iso_url}" == 'on' ]; then
echo -e " Windows ${RELEASE} Download (valid for 24 hours):\n${iso_download_link}"
exit 0
echo -e " Windows ${RELEASE} Download (valid for 24 hours):\n${iso_download_link}"
exit 0
elif [ "${test_iso_url}" == 'on' ]; then
wget --spider "${iso_download_link}"
exit 0
wget --spider "${iso_download_link}"
exit 0
fi

if [ "${LANG}" != "English International" ]; then
echo Downloading Windows ${RELEASE} in "${LANG}" from "$iso_download_link"
else
echo Downloading Windows ${RELEASE} from "$iso_download_link"
fi
echo "Downloading Windows ${RELEASE} (${LANG}): $iso_download_link"

# Download ISO
FILE_NAME="$(echo "$iso_download_link" | cut -d'?' -f1 | cut -d'/' -f5)"
web_get "$iso_download_link" "${VM_PATH}" "${FILE_NAME}"
curl_windows "${VM_PATH}" "${FILE_NAME}" "1.3" "$iso_download_link"
# Only Windows 11 hashes can be found directly from Microsoft's page.
if [ "${windows_version}" == 11 ]; then
check_hash "${FILE_NAME}" "${HASH}"
local HASH_LANG="${LANG}"
if [ "${LANG}" == "English (United States)" ]; then
HASH_LANG="English"
fi
local HASH=$(echo "$iso_download_link_html" | sed 's/<tr><td>/\n<tr><td>/g' | grep "$HASH_LANG 64-bit" | grep -o -P '(?<=</td><td>).*(?=</td></tr>)')
check_hash "${VM_PATH}/${FILE_NAME}" "${HASH}"
fi
}

function get_windows() {
if [ "${RELEASE}" == "10-ltsc" ]; then
download_windows-server windows-10-enterprise ltsc
download_windows_workstation windows-10-enterprise ltsc
elif [ "${OS}" == "windows-server" ]; then
download_windows-server windows-server-${RELEASE}
download_windows_server "windows-server-${RELEASE}"
else
download_windows "${RELEASE}"
download_windows_workstation "${RELEASE}"
fi

if [ "${download_iso}" == 'on' ]; then
Expand Down