onshape-download-std-library

#!/usr/bin/env bash
# ^ to get bash 5 on my mac - needed for inherit_errexit and friends

set -ueC -o pipefail
shopt -s inherit_errexit
shopt -u dotglob # so that we can easily remove all files in the mirror repo except for .git and friends. note that we could also do this via shopt -s extglob, then e.g. rm $git_repo_dir/!(.git)

# (despite all of the above, lots of stuff - like missing variable errors in nested substitutions - still doesn't get caught. you (and future me) have been forewarned.)

if [ -z "${AUTH_HEADER:-}" ]; then
  if [ -x local/read-auth-header ]; then
    AUTH_HEADER="$(local/read-auth-header)"
  else
    echo 'No Onshape authentication header found in $AUTH_HEADER and no local/read-auth-header script found to look it up. Bailing.' >&2
    exit 1
  fi
fi

ONSHAPE_STD_DOC_ID=12312312345abcabcabcdeff
ONSHAPE_STD_VWM_ID=w/a855e4161c814f2e9ab3698a

REPO_REMOTE=git@github.com:javawizard/onshape-std-library-mirror

export GIT_AUTHOR_NAME="Onshape Standard Library Importer"
export GIT_AUTHOR_EMAIL="alex+onshape-std-library-importer@opengroove.org"
export GIT_COMMITTER_NAME="$GIT_AUTHOR_NAME"
export GIT_COMMITTER_EMAIL="$GIT_AUTHOR_EMAIL"

GITHUB_SSH_KNOWN_HOSTS="
github.com ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIOMqqnkVzrm0SdG6UOoqKLsabgH5C9okWi0dh2l9GKJl
github.com ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEmKSENjQEezOmxkZMy7opKgwFB9nkt5YRrYMjNuG5N87uRgg6CLrbo5wAdT/y6v0mKV0U2w0WZ2YB/++Tpockg=
github.com ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCj7ndNxQowgcQnjshcLrqPEiiphnt+VTTvDP6mHBL9j1aNUkY4Ue1gvwnGLVlOhGeYrnZaMgRK6+PKCUXaDbC7qtbW8gIkhL7aGCsOr/C56SJMy/BCZfxd1nWzAOxSDPgVsmerOBYfNqltV9/hWCqBywINIR+5dIg6JTJ72pcEpEjcYgXkE2YEFXV1JHnsKgbLWNlhScqb2UmyRkQyytRLtL+38TGxkxCflmO+5Z8CSSNY7GidjMIZ7Q4zMjA2n1nGrlTDkzwDCsw+wqFPGQA179cnfGWOWRVruj16z6XyvxvjJwbz0wQZ75XK5tKSb7FNyeIEs4TT4jk+S4dhPeAUC5y+bDYirYgM4GC7uEnztnZyaVWQ7B381AK4Qdrwt51ZqExKbQpTUNn+EjqoTwvqNj4kqx5QUCI0ThS/YkOxJCXmPUWZbhjpCg56i+2aB6CmK2JGhn57K5mj0MNdBXA4/WnwH6XoPWJzK5Nyu2zB3nAZp+S5hpQs+p1vN1/wsjk=
"

call_api() {
  sleep 0.5 # to avoid ratelimiting
  curl --no-progress-meter -H "Authorization: $AUTH_HEADER" https://cad.onshape.com/api/v6/"$1" "${@:2}"
}

json_field() {
  echo "$1" | jq -r ."$2"
}

repo_dir="$(dirname "$0")/local/onshape-std-library-mirror"

if [ -n "${GITHUB_SSH_KEY:-}" ]; then
  echo "$GITHUB_SSH_KNOWN_HOSTS" > /tmp/ssh_known_hosts
  chmod 400 /tmp/ssh_known_hosts

  echo "$GITHUB_SSH_KEY" > /tmp/id_rsa_github_onshape_mirror
  chmod 400 /tmp/id_rsa_github_onshape_mirror

  export GIT_SSH_COMMAND="ssh -o UserKnownHostsFile=/tmp/ssh_known_hosts -i /tmp/id_rsa_github_onshape_mirror"
fi

# Arguments: the git repo/folder to download to and the the WVM ID to download
download_elements() {
  local git_repo_dir="$1" wvm_id="$2" # the set -u up top will cause this to blow up if we aren't passed at least two arguments

  # safety check
  if [ ! -d "$git_repo_dir"/.git ]; then
    echo "$git_repo_dir does not contain a .git folder" >&2
    exit 1
  fi

  if [ "$(ls "$git_repo_dir")" != "" ]; then
    echo "[download_elements] Clearing non-hidden files from $git_repo_dir ..." >&2
    rm "$git_repo_dir"/*
  fi

  echo "[download_elements] Downloading elements from $wvm_id into $git_repo_dir ..." >&2

  call_api documents/d/"$ONSHAPE_STD_DOC_ID"/"$wvm_id"/elements?withThumbnails=false | jq -c '.[] | {"name", "id", "elementType", "dataType"}' | while read e; do
    local name="$(json_field "$e" name)"
    local id="$(json_field "$e" id)"
    local elementType="$(json_field "$e" elementType)"
    local dataType="$(json_field "$e" dataType)"

    local suffix
    case "$dataType" in
      application/pdf)
        suffix=.pdf
        ;;
      text/plain)
        suffix=.txt
        ;;
      onshape/featurestudio|"")
        suffix=
        ;;
      *)
        echo "[download_elements] unknown datatype, skipping suffix: $dataType" >&2
        suffix=
        ;;
    esac

    echo "[download_elements] name: $name, type: $elementType, id: $id, datatype: $dataType, suffix: $suffix" >&2

    case "$elementType" in
      FEATURESTUDIO)
        call_api featurestudios/d/"$ONSHAPE_STD_DOC_ID"/"$wvm_id"/e/"$id" | jq -r .contents > "$git_repo_dir/$name$suffix"
        ;;
      BLOB)
        call_api blobelements/d/"$ONSHAPE_STD_DOC_ID"/"$wvm_id"/e/"$id" > "$git_repo_dir/$name$suffix"
        ;;
      *)
        echo "ERROR: unknown type: $elementType" >&2
        exit 1
    esac
  done

}

fetch_versions() {
  call_api documents/d/"$ONSHAPE_STD_DOC_ID"/versions | jq -c '.[] | {"id", "name", "microversion", "createdAt", "parent"}'
}

# No longer used, but I'm keeping it around for reference and/or future use.
#
# Arguments: the git repo to consult and the Onshape version ID to look for. Outputs nothing (but succeeds) if no matching commit is found.
# find_commit_for_version() {
#   local git_repo_dir="$1" version_id="$2"

#   if [ -z "$version_id" ]; then
#     echo "can't look for empty version" >&2
#     exit 1
#   fi

#   git -C "$git_repo_dir" rev-parse -q --verify :/"version id: $version_id" || true
# }

# Arguments: the git repo to download into and the JSON blob from fetch_versions corresponding to the version to download
download_version_if_needed() {
  local git_repo_dir="$1"
  local version_spec="$2"
  local post_process_function="$3"
  local commit_message_prefix="$4"

  local id="$(json_field "$version_spec" id)"
  local name="$(json_field "$version_spec" name)"

  if git -C "$git_repo_dir" log | grep -E "version id: $id" >/dev/null; then
    echo "[download_version_if_needed] [$(git -C "$git_repo_dir" rev-parse --abbrev-ref HEAD)] Version $name is already downloaded" >&2
    return
  fi

  local microversion="$(json_field "$version_spec" microversion)"
  local createdAt="$(json_field "$version_spec" createdAt)"
  local parent="$(json_field "$version_spec" parent)"

  # Note that this doesn't handle Onshape document branches. That's fine because Onshape never branches their canonical
  # standard library document, but if we ever generalize this script to run on arbitrary Onshape documents, it'll need
  # to be modified to track branches correctly.

  echo "[download_version_if_needed] Downloading version $name ..."

  git -C "$git_repo_dir" reset # in case a previous invocation crashed midway between doing git add --all and git commit

  download_elements "$git_repo_dir" "v/$id" # this will rm all files beforehand, so no need to do that here

  # Pull in the contents of the latest README branch commit - that way we don't blow away the
  # onshape-download-std-library README whenever we commit a new version. Note that this only works correctly if
  # merge_readme_branch is always run before download_all_versions; otherwise we'll pull in changes that we haven't yet
  # merged and we'll get a merge conflict when we try to do so. (Really the only reason for doing those merges in the
  # first place is so that changes to the README show up as having come from their corresponding readme branch commits,
  # not Onshape standard library version bump commits.)
  git -C "$git_repo_dir" checkout origin/readme -- .

  echo "[download_version_if_needed] Post-processing version using $post_process_function ..."
  "$post_process_function" "$git_repo_dir"

  echo "[download_version_if_needed] Committing version $name ..."

  local commit_message="${commit_message_prefix}Version $name

version id: $id
parent: $parent
microversion: $microversion
created at: $createdAt
"

  git -C "$git_repo_dir" add --all
  GIT_AUTHOR_DATE="$createdAt" GIT_COMMITTER_DATE="$createdAt" git -C "$git_repo_dir" commit -m "$commit_message" --allow-empty

  echo "[download_version_if_needed] Committed version $name as $(git -C "$git_repo_dir" rev-parse HEAD)."
}

post_process_main() {
  # do nothing; main is good to go as is
  :
}

post_process_without_versions() {
  # replace version numbers with sparkles to cut down on diff noise
  # NOTE: Onshape's standard library does not use directories, so we don't worry about recursion here. We'll need to
  # change that if we ever generalize this to mirroring arbitrary Onshape documents.
  sed_i_arg=()
  if [ "$(uname)" == Darwin ]; then
    # On macOS (and technically any BSD-derived system), sed's -i flag takes a mandatory argument.
    # On Linux, the argument is optional, and cannot be the empty string - which means there's no portable way to use
    # sed -i, so we have to manually detect macOS and add the empty argument to the -i flag.
    sed_i_arg=('')
  fi

  (cd "$1" && sed -Ei "${sed_i_arg[@]}" 's/^(FeatureScript )[0-9\.]+(\;)/\1✨\2/g;s/(path *: *"onshape.+", *version *: *")[0-9\.]+(")/\1✨\2/g' *.fs)
}

download_all_versions() {
  local git_repo_dir="$1"
  local post_process_function="$2"
  local commit_message_prefix="$3"

  fetch_versions | while read v; do
    download_version_if_needed "$git_repo_dir" "$v" "$post_process_function" "$commit_message_prefix"
  done
}

clone_fresh_repo() {
  if [ -d "$2" ]; then
    rm -rf "$2"
  fi

  git clone -b main "$1" "$2" # `without-versions` is now the default branch in github, so add `-b main` to checkout main specifically
}

checkout() {
  git -C "$1" checkout "$2"
}

merge_readme_branch() {
  echo "[merge_readme_branch] Merging changes from the readme branch, if any ..." >&2
  git -C "$1" merge origin/readme --no-edit
}

# Arguments: the path to the git repo to push and the branch to push
push_changes() {
  git -C "$1" push origin "$2" -f
}

clone_fresh_repo "$REPO_REMOTE" "$repo_dir"
merge_readme_branch "$repo_dir"
download_all_versions "$repo_dir" post_process_main ""
checkout "$repo_dir" without-versions
merge_readme_branch "$repo_dir"
# TODO: reuse files from the main branch instead of redownloading them
download_all_versions "$repo_dir" post_process_without_versions "[✨] " # update git-filter-repo-specs/without-versions/replace-message.txt when changing this prefix
push_changes "$repo_dir" main
push_changes "$repo_dir" without-versions


# experiments
# call_api documents/d/"$ONSHAPE_STD_DOC_ID"/"$ONSHAPE_STD_VWM_ID"/elements?withThumbnails=false | jq -c '.[] | {"name", "id", "elementType", "dataType"}'
# call_api documents/d/"$ONSHAPE_STD_DOC_ID"/versions | jq # -c '.[] | {"name", "id", "elementType", "dataType"}'