HFtoMS.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/fireicewolf/HFtoMS/blob/main/HFtoMS.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "Mh-qYDMKX_Zu"
      },
      "outputs": [],
      "source": [
        "# @title Install dependencies\n",
        "!pip install --no-cache-dir -U huggingface-hub modelscope\n",
        "!pip cache purge"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "G0U7KXTOX_Zw"
      },
      "outputs": [],
      "source": [
        "# @title Remove useless python package for more storage space.\n",
        "!pip uninstall tensorboard tensorboard-data-server tensorflow tensorflow-datasets tensorflow-estimator tensorflow-gcs-config tensorflow-hub tensorflow-io-gcs-filesystem tensorflow-metadata tensorflow-probability tensorstore -y\n",
        "!pip uninstall torch torchaudio torchsummary torchtext torchvision triton -y\n",
        "!pip uninstall opencv-python-headless opencv-python opencv-contrib-python -y\n",
        "!pip uninstall Sphinx sphinxcontrib-applehelp sphinxcontrib-devhelp sphinxcontrib-htmlhelp sphinxcontrib-jsmath sphinxcontrib-qthelp sphinxcontrib-serializinghtml -y"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "22OBweJZX_Zx"
      },
      "outputs": [],
      "source": [
        "# @title Cleanup workspace\n",
        "%cd /content\n",
        "!rm -rf /content/*"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "cellView": "form",
        "id": "_-9nOJhrX_Zy"
      },
      "outputs": [],
      "source": [
        "#@title HF to MS\n",
        "import math\n",
        "import os\n",
        "\n",
        "from huggingface_hub import HfApi,login, list_repo_tree, hf_hub_download\n",
        "from huggingface_hub.hf_api import RepoFile, RepoFolder\n",
        "\n",
        "from modelscope.hub.api import HubApi\n",
        "from modelscope.hub.constants import ModelVisibility\n",
        "\n",
        "WORKSPACE = \"/content\"\n",
        "HF_TOKEN = \"\" # @param {type:\"string\"}\n",
        "HF_REPO_ID = \"\" # @param {type:\"string\"}\n",
        "HF_RECURSIVE = False # @param {type:\"boolean\"}\n",
        "HF_REPO_TYPE = \"model\" # @param {type:\"string\"}\n",
        "HF_REVISION = \"main\" # @param {type:\"string\"}\n",
        "MS_TOKEN = \"\" # @param {type:\"string\"}\n",
        "MS_REPO_ID = \"\" # @param {type:\"string\"}\n",
        "MS_REPO_PRIVATE = True # @param {type:\"boolean\"}\n",
        "MS_REVISION = \"master\" # @param {type:\"string\"}\n",
        "MS_COMMIT = \"Upload to ModelScope\" # @param {type:\"string\"}\n",
        "\n",
        "def login_hf(hf_token: str):\n",
        "    login(token=hf_token)\n",
        "\n",
        "def list_hf_repo_tree(\n",
        "        hf_repo_id: str,\n",
        "        hf_recursive: bool = True,\n",
        "        hf_repo_type: str = \"model\",\n",
        "        hf_revision: str = \"main\"\n",
        "    ):\n",
        "    hf_repo_tree = list_repo_tree(hf_repo_id, recursive=hf_recursive, repo_type=hf_repo_type, revision=hf_revision)\n",
        "    return list(hf_repo_tree)\n",
        "\n",
        "def hf_file_download(hf_repo_id: str, hf_path: str, hf_repo_type: str, hf_revision: str,local_dir: str):\n",
        "    hf_filename = os.path.basename(hf_path)\n",
        "    hf_subfolder = os.path.dirname(hf_path)\n",
        "    hf_hub_download(hf_repo_id, hf_filename, subfolder=hf_subfolder, repo_type=hf_repo_type, revision=hf_revision, local_dir=local_dir, resume_download=True)\n",
        "\n",
        "def ms_create_repo(\n",
        "        ms_token: str,\n",
        "        ms_repo_id: str,\n",
        "        ms_repo_private: bool,\n",
        "    ):\n",
        "    api = HubApi()\n",
        "    ms_git_token = (api.login(ms_token))[0]\n",
        "    print(f'Creating repo \"{ms_repo_id}\" on ModelScope...')\n",
        "    try:\n",
        "        api.get_model(ms_repo_id)\n",
        "        print(f'\"{ms_repo_id}\" already exist on ModelScope!')\n",
        "    except Exception:\n",
        "        model_visibility = ModelVisibility.PRIVATE if ms_repo_private else ModelVisibility.PUBLIC\n",
        "        api.create_model(ms_repo_id, visibility=model_visibility)\n",
        "        print(f'\"{ms_repo_id}\" created on ModelScope')\n",
        "\n",
        "    return ms_git_token\n",
        "\n",
        "\n",
        "def ms_git_clone_repo(ms_git_token, ms_repo_id, ms_revision):\n",
        "    os.chdir(WORKSPACE)\n",
        "    repo_on_local_path = os.path.join(WORKSPACE, os.path.basename(ms_repo_id))\n",
        "    print('Disabling Git LFS...')\n",
        "    os.environ[\"GIT_LFS_SKIP_SMUDGE\"] = \"1\"\n",
        "    !git lfs uninstall\n",
        "    if os.path.exists(repo_on_local_path):\n",
        "        print(f'\"{repo_on_local_path}\" already exist, will delete it!!!')\n",
        "        !rm -rf \"{repo_on_local_path}\"\n",
        "\n",
        "    !git clone -b {ms_revision} http://outh2:{ms_git_token}@www.modelscope.cn/{ms_repo_id}.git {repo_on_local_path}\n",
        "    os.chdir(repo_on_local_path)\n",
        "    !git switch {ms_revision}\n",
        "    print('Enabling Git LFS...')\n",
        "    os.environ[\"GIT_LFS_SKIP_SMUDGE\"] = \"0\"\n",
        "    !git lfs install\n",
        "    return repo_on_local_path\n",
        "\n",
        "def format_file_size(size_bytes):\n",
        "    if size_bytes == 0:\n",
        "        return \"0B\"\n",
        "    size_name = (\"B\", \"KB\", \"MB\", \"GB\", \"TB\")\n",
        "    i = int(math.floor(math.log(size_bytes, 1024)))\n",
        "    p = math.pow(1024, i)\n",
        "    s = round(size_bytes / p, 2)\n",
        "    return f\"{s} {size_name[i]}\"\n",
        "\n",
        "def main():\n",
        "    os.chdir(WORKSPACE)\n",
        "    !git config --global user.email \"you@example.com\"\n",
        "    !git config --global user.name \"Your Name\"\n",
        "    ms_git_token = ms_create_repo(MS_TOKEN, MS_REPO_ID, MS_REPO_PRIVATE)\n",
        "    repo_on_local = ms_git_clone_repo(ms_git_token, MS_REPO_ID, MS_REVISION)\n",
        "    login_hf(HF_TOKEN)\n",
        "    hf_repo_tree = list_hf_repo_tree(HF_REPO_ID)\n",
        "\n",
        "    file_use_git_lfs = []\n",
        "    total = len(hf_repo_tree)\n",
        "    i = 0\n",
        "    for file in hf_repo_tree:\n",
        "        if type(file) == RepoFolder:\n",
        "            path = file.path\n",
        "            path_on_local = os.path.join(repo_on_local, path)\n",
        "            os.makedirs(path_on_local, exist_ok=True)\n",
        "        elif type(file) == RepoFile:\n",
        "            path = file.path\n",
        "            size = file.size\n",
        "            file_on_local = os.path.join(repo_on_local, path)\n",
        "\n",
        "            if size < 25 * 1024 * 1024:\n",
        "                i+=1\n",
        "                print(f'Processing {i}/{total}: {path}...')\n",
        "                if os.path.exists(file_on_local):\n",
        "                    print(f'\"{file_on_local}\" already exist, will delete it!!!')\n",
        "                    !rm -rf \"{file_on_local}\"\n",
        "                hf_file_download(HF_REPO_ID, path, HF_REPO_TYPE, HF_REVISION, repo_on_local)\n",
        "\n",
        "            else:\n",
        "                print(f'\"{path}\" is {format_file_size(size)},  will add it to Git LFS list.')\n",
        "                file_use_git_lfs.append(path)\n",
        "\n",
        "    os.chdir(repo_on_local)\n",
        "    !git add -A .\n",
        "    git_commit_cmd = f'git commit -m \"{MS_COMMIT}\"'\n",
        "    !{git_commit_cmd}\n",
        "    !git push --set-upstream origin {MS_REVISION}\n",
        "\n",
        "\n",
        "    totol_non_lfs = i\n",
        "    print(f'Total non Git-LFS files: {totol_non_lfs}.')\n",
        "\n",
        "    for file in file_use_git_lfs:\n",
        "        i+=1\n",
        "        repo_on_local = ms_git_clone_repo(ms_git_token, MS_REPO_ID, MS_REVISION)\n",
        "        file_on_local = os.path.join(repo_on_local, file)\n",
        "\n",
        "        if os.path.exists(file_on_local):\n",
        "            print(f'\"{file_on_local}\" already exist, will delete it!!!')\n",
        "            !rm -rf \"{file_on_local}\"\n",
        "        print(f'Processing {i}/{total}: {file}...')\n",
        "        os.chdir(repo_on_local)\n",
        "        hf_file_download(HF_REPO_ID, file, HF_REPO_TYPE, HF_REVISION, repo_on_local)\n",
        "        if os.path.exists(os.path.join(repo_on_local, \".huggingface\")):\n",
        "            !rm -rf {os.path.join(repo_on_local, \".huggingface\")}\n",
        "        !git lfs track {file_on_local}\n",
        "        !git add {file_on_local}\n",
        "        !{git_commit_cmd}\n",
        "        !git push --set-upstream origin {MS_REVISION}\n",
        "        os.chdir(WORKSPACE)\n",
        "\n",
        "    total_lfs = len(file_use_git_lfs)\n",
        "    print(f'Total Git-LFS files: {total_lfs}.')\n",
        "    print(f'Total files: {total}.')\n",
        "\n",
        "main()"
      ]
    }
  ],
  "metadata": {
    "language_info": {
      "name": "python"
    },
    "colab": {
      "provenance": [],
      "include_colab_link": true
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}