From 82922761ba6de692b2ba9b3dfb831a471e9e1d2c Mon Sep 17 00:00:00 2001 From: Benjamin Lupton Date: Fri, 1 Sep 2023 14:31:24 +0800 Subject: [PATCH] github-download: prevent rate limiting by supporting GITHUB_TOKEN fetch: - support `--bearer-token` ci: - pass GITHUB_TOKEN to env --- .github/workflows/dorothy-workflow.yml | 8 +++ commands/fetch | 74 ++++++++++++++++++++------ commands/github-download | 8 +-- 3 files changed, 69 insertions(+), 21 deletions(-) diff --git a/.github/workflows/dorothy-workflow.yml b/.github/workflows/dorothy-workflow.yml index 5a7eb87f6..627f2bbdf 100755 --- a/.github/workflows/dorothy-workflow.yml +++ b/.github/workflows/dorothy-workflow.yml @@ -32,6 +32,8 @@ jobs: path: ${{ env.DOROTHY }} - name: 'Dorothy Development' shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # prevent rate limiting run: | # ensure clone set correct permissions chmod +x "$DOROTHY/commands/"* @@ -84,6 +86,8 @@ jobs: steps: - name: 'Dorothy Test' shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # prevent rate limiting run: | # clone dorothy and run tests bash -ic "$(curl -fsSL 'https://dorothy.bevry.me/run?branch=${{ github.ref_name }}&commit=${{ github.sha }}')" -- dorothy test @@ -97,6 +101,8 @@ jobs: bash -ic "$(curl -fsSL 'https://raw.githubusercontent.com/Homebrew/install/master/uninstall.sh')" - name: 'Dorothy Test' shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # prevent rate limiting run: | # clone dorothy and run tests bash -ic "$(curl -fsSL 'https://dorothy.bevry.me/run?branch=${{ github.ref_name }}&commit=${{ github.sha }}')" -- dorothy test @@ -171,6 +177,8 @@ jobs: fi - name: 'Dorothy Remote Tests' shell: bash + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # prevent rate limiting run: | # clone dorothy and run tests bash -ic "$(curl -fsSL 'https://dorothy.bevry.me/run?branch=${{ github.ref_name }}&commit=${{ github.sha }}')" -- dorothy test diff --git a/commands/fetch b/commands/fetch index 473c615d1..e41255fe6 100755 --- a/commands/fetch +++ b/commands/fetch @@ -15,6 +15,9 @@ function fetch() ( fetch [...options] OPTIONS: + --bearer-token= + If provided, include this in a bearer token header. + --[no-]status[=] If truthy, output only the status code, rather than the entire contents of the fetch URL. @@ -32,12 +35,13 @@ function fetch() ( } # process - local item option_url='' option_status='no' option_ok='' + local item option_bearer_token='' option_url='' option_status='no' option_ok='' while test "$#" -ne 0; do item="$1" shift case "$item" in '--help' | '-h') help ;; + '--bearer-token='*) option_bearer_token="${item#*--bearer-token=}" ;; '--no-status'* | '--status'*) option_status="$(get-flag-value status --missing="$option_status" -- "$item" | echo-affirmative)" ;; @@ -69,36 +73,72 @@ function fetch() ( return "$status" } - # -I, --head Show document info only - # -m, --max-time Maximum time allowed for the transfer - # -o, --output Write to file instead of stdout - # -s, --silent Silent mode - # -A, --user-agent Send User-Agent to server - # -w, --write-out Use output FORMAT after completion + # -I, --head Show document info only + # -m, --max-time Maximum time allowed for the transfer + # -o, --output Write to file instead of stdout + # -s, --silent No curl output, only content output. + # -w, --write-out Use output FORMAT after completion + # -L, --location Follow redirects + # -f, --fail Fail on server errors. + # -A, --user-agent Send User-Agent to server + # -S, --show-error When used with -s, --silent, it makes curl show an error message if it fails. + function do_curl { + local options=("$@") + if test "$option_bearer_token"; then + options+=( + '--header' + "Authorization: Bearer $option_bearer_token" + ) + fi + # use echo-wait to fix [curl: (23) Failure writing output to destination] when piping to ripgrep + curl "${options[@]}" "$option_url" | echo-wait || log_failure + } + function do_wget { + local options=("$@") + if test "$option_bearer_token"; then + options+=( + "--header=Authorization: Bearer $option_bearer_token" + ) + fi + wget "${options[@]}" "$option_url" || log_failure + } + function fetch_status { - local args=( - --head - --max-time 3 - --output /dev/null - --silent - --write-out '%{http_code}' + local options=( + --head # only fetch head + --max-time 3 # wait three seconds + --output /dev/null # no content output + --silent # no curl output + --write-out '%{http_code}' # output the http code + --location # follow redirects ) setup-util-curl --quiet - curl "${args[@]}" --location "$option_url" || log_failure + do_curl "${options[@]}" } function fetch_ok { + # no wget equivalent local status status="$(fetch_status "$option_url")" test "$status" -ge 200 -a "$status" -le 300 } function fetch_contents { + local options if command-exists curl; then - # use echo-wait to fix [curl: (23) Failure writing output to destination] when piping to ripgrep - curl -fsSL "$option_url" | echo-wait || log_failure + options=( + --fail # fail if server fails + --silent # only output content + --show-error # output errors + --location # follow redirects + ) + do_curl "${options[@]}" elif command-exists wget; then - wget "$option_url" -O - || log_failure + options=( + -quiet # only output content + -O- # output to stdout + ) + do_wget "${options[@]}" else get-installer --first-success --invoke --quiet -- curl wget fetch_contents diff --git a/commands/github-download b/commands/github-download index 201396e41..9a585cc78 100755 --- a/commands/github-download +++ b/commands/github-download @@ -160,9 +160,9 @@ function github_download() ( # as [/releases/$release] returns asset information, use that with .assets[] instead of .[] local url url="$GITHUB_API_URL/repos/$slug/releases/$release" - if fetch "$url" | grep --quiet --fixed-strings --regexp='Not Found'; then + if fetch --bearer-token="${GITHUB_TOKEN-}" "$url" | grep --quiet --fixed-strings --regexp='Not Found'; then url="$( - fetch "$GITHUB_API_URL/repos/$slug/releases" | jq -r ".[] | select(.tag_name==\"$release\") | .url" + fetch --bearer-token="${GITHUB_TOKEN-}" "$GITHUB_API_URL/repos/$slug/releases" | jq -r ".[] | select(.tag_name==\"$release\") | .url" )" fi if command-missing jq; then @@ -170,9 +170,9 @@ function github_download() ( # [tail -n+2] excludes the first line, which is the name of the repo # this is fragile as if the order of arguments change, it is all over # sed fixes minified json such as `},{` and `","` - fetch "$url" | sed -E $'s/,/,\\\n/g' | grep --extended-regexp --regexp='"(name|browser_download_url)":' | tail -n+2 | sed -E 's/.+: *"(.+)".*/\1/' + fetch --bearer-token="${GITHUB_TOKEN-}" "$url" | sed -E $'s/,/,\\\n/g' | grep --extended-regexp --regexp='"(name|browser_download_url)":' | tail -n+2 | sed -E 's/.+: *"(.+)".*/\1/' else - fetch "$url" | jq -r '.assets[] | (.name, .browser_download_url)' + fetch --bearer-token="${GITHUB_TOKEN-}" "$url" | jq -r '.assets[] | (.name, .browser_download_url)' fi } function get_matched_assets_for_release {