notebook/bge-embedding-m3-deploy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7060c891-cebd-4011-b350-b7d1e70b40b2",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 1. 安装HuggingFace 并下载模型到本地"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f413314-c410-43d3-bb3a-ba0aa18ec1be",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install huggingface-hub -Uqq\n",
    "!pip install -Uqq sagemaker"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "df3b0a4b-f166-4f1a-a7cc-9c7277c68173",
   "metadata": {},
   "source": [
    "### 如果是外海region，执行下面cell通过huggingface_hub下载, 否则跳过"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "be112a00-cbef-4387-b0d7-80e5e7b7030d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from huggingface_hub import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./bge-m3-model\")\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"BAAI/bge-m3\"\n",
    "commit_hash = \"4277867103fc67328e2033176de4387b85e9960f\"\n",
    "snapshot_download(repo_id=model_name, revision=commit_hash, cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "67712dfc-a411-433b-bc67-9734a1686480",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 如果是中国区，执行下面cell通过modelscope进行下载"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b3fa6b9-492d-477b-8a45-8b8e3bbfb6e8",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!pip install modelscope -i https://pypi.tuna.tsinghua.edu.cn/simple -Uqq"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dda7d753-6faf-46fc-a8c1-da9a3f1cd1db",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from modelscope.hub.snapshot_download import snapshot_download\n",
    "from pathlib import Path\n",
    "\n",
    "local_model_path = Path(\"./bge-zh-model\")\n",
    "\n",
    "local_model_path.mkdir(exist_ok=True)\n",
    "model_name = \"Xorbits/bge-large-zh-v1.5\"\n",
    "commit_hash = \"v0.0.1\"\n",
    "\n",
    "snapshot_download(model_name, revision=commit_hash, cache_dir=local_model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6b61ad8-a8c2-48c2-8539-e7c1e2afe773",
   "metadata": {},
   "source": [
    "### 2. 把模型拷贝到S3为后续部署做准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e1873f4-1bfe-4146-8297-584e9ad76fc9",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import sagemaker\n",
    "from sagemaker import image_uris\n",
    "import boto3\n",
    "import os\n",
    "import time\n",
    "import json\n",
    "\n",
    "role = sagemaker.get_execution_role()  # execution role for the endpoint\n",
    "sess = sagemaker.session.Session()  # sagemaker session for interacting with different AWS APIs\n",
    "bucket = sess.default_bucket()  # bucket to house artifacts\n",
    "\n",
    "region = sess._region_name\n",
    "account_id = sess.account_id()\n",
    "\n",
    "s3_client = boto3.client(\"s3\")\n",
    "sm_client = boto3.client(\"sagemaker\")\n",
    "smr_client = boto3.client(\"sagemaker-runtime\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "68394e44-4d51-48ae-adc1-d02f520a5d4d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_model_prefix = \"LLM-RAG/workshop/bge-m3-model\"  # folder where model checkpoint will go\n",
    "if region in ['cn-north-1', 'cn-northwest-1']:\n",
    "    model_snapshot_path = f'{local_model_path}/{model_name}'\n",
    "else:\n",
    "    model_snapshot_path = list(local_model_path.glob(\"**/snapshots/*\"))[0]\n",
    "s3_code_prefix = \"LLM-RAG/workshop/bge_m3_deploy_code\"\n",
    "print(f\"s3_code_prefix: {s3_code_prefix}\")\n",
    "print(f\"model_snapshot_path: {model_snapshot_path}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0b9e177a-886d-4838-891e-2e612a3cbc9d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!aws s3 cp --recursive {model_snapshot_path} s3://{bucket}/{s3_model_prefix}"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59f35a6f-5988-42ec-87b0-de36eaebe41b",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 3. 模型部署准备（entrypoint脚本，容器镜像，服务配置）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "86daea77-a7ae-46b8-8800-212d07ce5605",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "inference_image_uri = (\n",
    "    f\"763104351884.dkr.ecr.{region}.amazonaws.com/djl-inference:0.23.0-deepspeed0.9.5-cu118\"\n",
    "    \n",
    ")\n",
    "\n",
    "#中国区需要替换为下面的image_uri\n",
    "if region in ['cn-north-1', 'cn-northwest-1']:\n",
    "    inference_image_uri = (\n",
    "        f\"727897471807.dkr.ecr.{region}.amazonaws.com.cn/djl-inference:0.23.0-deepspeed0.9.5-cu118\"\n",
    "    )\n",
    "\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49435172-e6c5-492a-8dcb-43e3fffb0f5c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!mkdir -p bge_m3_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70990dd3-431e-4dd0-a494-d26ceb454945",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile bge_m3_deploy_code/model.py\n",
    "from djl_python import Input, Output\n",
    "import torch\n",
    "import logging\n",
    "import math\n",
    "import os\n",
    "from FlagEmbedding import BGEM3FlagModel\n",
    "\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else 'cpu')\n",
    "print(f'--device={device}')\n",
    "\n",
    "def load_model(properties):\n",
    "    tensor_parallel = properties[\"tensor_parallel_degree\"]\n",
    "    model_location = properties['model_dir']\n",
    "    if \"model_id\" in properties:\n",
    "        model_location = properties['model_id']\n",
    "    logging.info(f\"Loading model in {model_location}\")\n",
    "\n",
    "    model =  BGEM3FlagModel(model_location,use_fp16=True)\n",
    "    \n",
    "    return model\n",
    "\n",
    "model = None\n",
    "\n",
    "def handle(inputs: Input):\n",
    "    global model\n",
    "    if not model:\n",
    "        model = load_model(inputs.get_properties())\n",
    "\n",
    "    if inputs.is_empty():\n",
    "        return None\n",
    "    data = inputs.get_as_json()\n",
    "    \n",
    "    input_sentences = None\n",
    "    inputs = data[\"inputs\"]\n",
    "    if isinstance(inputs, list):\n",
    "        input_sentences = inputs\n",
    "    else:\n",
    "        input_sentences =  [inputs]\n",
    "        \n",
    "    is_query = data.get(\"is_query\")\n",
    "    max_length = data.get(\"max_length\", 2048)\n",
    "    instruction = data.get(\"instruction\",'')\n",
    "    logging.info(f\"inputs: {input_sentences}\")\n",
    "    logging.info(f\"is_query: {is_query}\")\n",
    "    logging.info(f\"instruction: {instruction}\")\n",
    "    \n",
    "    if is_query and instruction:\n",
    "        input_sentences = [ instruction + sent for sent in input_sentences ]\n",
    "        \n",
    "    sentence_embeddings =  model.encode(input_sentences, max_length=max_length)\n",
    "        \n",
    "    result = {\"embeddings\": sentence_embeddings['dense_vecs']}\n",
    "    return Output().add_as_json(result)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f4c6d5f",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "print(f\"option.s3url ==> s3://{bucket}/{s3_model_prefix}/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a1e1ecec-79cf-4ed4-bba1-95e2fe79daea",
   "metadata": {},
   "source": [
    "#### Note: option.s3url 为SageMaker部署时使用的模型S3路径"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7b126565-66e2-4987-ac6b-e02f09070a65",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "\n",
    "if not os.path.exists(\"bge_m3_deploy_code\"):\n",
    "    os.mkdir(\"bge_m3_deploy_code\")\n",
    "\n",
    "with open('bge_m3_deploy_code/serving.properties', 'w') as f:\n",
    "    f.write(\"engine=Python\")\n",
    "    f.write(\"\\n\")\n",
    "    f.write(\"option.tensor_parallel_degree=1\")\n",
    "    f.write(\"\\n\")\n",
    "    f.write(f\"option.s3url=s3://{bucket}/{s3_model_prefix}/\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "5e11b8f7-aebf-42a0-9a7f-31691059cc65",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 【注意】下面这两个Cell，根据region，仅挑选一个执行，否则会产生问题"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e1434f9a-f114-4f83-a103-04fde82cb307",
   "metadata": {},
   "source": [
    "#### 如果是中国region，执行下面这个cell，在requirements.txt中添加国内的pip镜像"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38bf548e-fb01-4951-b49f-15a91c61fb2e",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile bge_m3_deploy_code/requirements.txt\n",
    "-i https://pypi.tuna.tsinghua.edu.cn/simple\n",
    "FlagEmbedding"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2a35bbdf-c246-4e9e-ab7f-aac5b389ddf2",
   "metadata": {
    "tags": []
   },
   "source": [
    "#### 如果是海外region，执行下面这个cell，无需额外添加pip镜像"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "928a7806-afc4-4ae7-9253-1c9dfabfed99",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%writefile bge_m3_deploy_code/requirements.txt\n",
    "FlagEmbedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ffe41472-c2cf-4bb5-99aa-84df76c629b3",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "!rm s2e_model.tar.gz\n",
    "!cd bge_m3_deploy_code && rm -rf \".ipynb_checkpoints\"\n",
    "!tar czvf s2e_model.tar.gz bge_m3_deploy_code"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1fabd7ce-b855-4569-857c-ad872662800b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "s3_code_artifact = sess.upload_data(\"s2e_model.tar.gz\", bucket, s3_code_prefix)\n",
    "print(f\"S3 Code or Model tar ball uploaded to --- > {s3_code_artifact}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "18fb01ed-6bd3-4880-a647-cfd71e692820",
   "metadata": {
    "tags": []
   },
   "source": [
    "### 4. 创建模型 & 创建endpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e6209d24-8473-4256-93d3-02e4e144386b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sagemaker.utils import name_from_base\n",
    "import boto3\n",
    "\n",
    "model_name = name_from_base(\"bge-m3\") #Note: Need to specify model_name\n",
    "print(model_name)\n",
    "print(f\"Image going to be used is ---- > {inference_image_uri}\")\n",
    "\n",
    "create_model_response = sm_client.create_model(\n",
    "    ModelName=model_name,\n",
    "    ExecutionRoleArn=role,\n",
    "    PrimaryContainer={\n",
    "        \"Image\": inference_image_uri,\n",
    "        \"ModelDataUrl\": s3_code_artifact\n",
    "    },\n",
    "    \n",
    ")\n",
    "model_arn = create_model_response[\"ModelArn\"]\n",
    "\n",
    "print(f\"Created Model: {model_arn}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686abae8-5db7-4ebd-9fbf-5bd54f36c0ab",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "endpoint_config_name = f\"{model_name}-config\"\n",
    "endpoint_name = f\"{model_name}-endpoint\"\n",
    "\n",
    "endpoint_config_response = sm_client.create_endpoint_config(\n",
    "    EndpointConfigName=endpoint_config_name,\n",
    "    ProductionVariants=[\n",
    "        {\n",
    "            \"VariantName\": \"variant1\",\n",
    "            \"ModelName\": model_name,\n",
    "            \"InstanceType\": \"ml.g4dn.xlarge\",\n",
    "            \"InitialInstanceCount\": 1,\n",
    "            # \"VolumeSizeInGB\" : 400,\n",
    "            # \"ModelDataDownloadTimeoutInSeconds\": 2400,\n",
    "            \"ContainerStartupHealthCheckTimeoutInSeconds\": 5*60,\n",
    "        },\n",
    "    ],\n",
    ")\n",
    "endpoint_config_response"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f4c1df06-ae4a-42e2-9695-da0afa9ad734",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "create_endpoint_response = sm_client.create_endpoint(\n",
    "    EndpointName=f\"{endpoint_name}\", EndpointConfigName=endpoint_config_name\n",
    ")\n",
    "print(f\"Created Endpoint: {create_endpoint_response['EndpointArn']}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9c71240-6878-4fed-bf7d-2c1cf75f4ac5",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "status = resp[\"EndpointStatus\"]\n",
    "print(\"Status: \" + status)\n",
    "\n",
    "while status == \"Creating\":\n",
    "    time.sleep(60)\n",
    "    resp = sm_client.describe_endpoint(EndpointName=endpoint_name)\n",
    "    status = resp[\"EndpointStatus\"]\n",
    "    print(\"Status: \" + status)\n",
    "\n",
    "print(\"Arn: \" + resp[\"EndpointArn\"])\n",
    "print(\"Status: \" + status)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dddba20e-fc18-480d-9940-ae39695ac450",
   "metadata": {},
   "source": [
    "### 5. 模型测试"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1f28db25-6996-440c-b004-14f96cfd982d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_vector_by_sm_endpoint(questions, sm_client, endpoint_name):\n",
    "    parameters = {\n",
    "    }\n",
    "\n",
    "    response_model = sm_client.invoke_endpoint(\n",
    "        EndpointName=endpoint_name,\n",
    "        Body=json.dumps(\n",
    "            {\n",
    "                \"inputs\": questions,\n",
    "                # \"is_query\": True,\n",
    "                \"instruction\" :  \"Represent this sentence for searching relevant passages:\"\n",
    "            }\n",
    "        ),\n",
    "        ContentType=\"application/json\",\n",
    "    )\n",
    "    # 中文instruction => 为这个句子生成表示以用于检索相关文章：\n",
    "    json_str = response_model['Body'].read().decode('utf8')\n",
    "    json_obj = json.loads(json_str)\n",
    "    embeddings = json_obj['embeddings']\n",
    "    return embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52d4f56a-092e-4a6a-a920-48550ec9f20c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "prompts1 = [\"How to compete with OCI\",\"如何与OCI竞争\",\"如何與OCI競爭\"]\n",
    "\n",
    "emb = get_vector_by_sm_endpoint(prompts1, smr_client, endpoint_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2ea82819-1c9f-493c-8bda-5b1e16f056cf",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "def cos_sim(vector1, vector2):\n",
    "    dot_product = np.dot(vector1, vector2)\n",
    "    norm_v1 = np.linalg.norm(vector1)\n",
    "    norm_v2 = np.linalg.norm(vector2)\n",
    "    cos_sim = dot_product / (norm_v1 * norm_v2)\n",
    "    return cos_sim"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f59bd2e-5f8a-42ae-bfc0-6e7d05ae209b",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "cos_sim(emb[1],emb[2])"
   ]
  }
 ],
 "metadata": {
  "availableInstances": [
   {
    "_defaultOrder": 0,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.t3.medium",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 1,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.t3.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 2,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.t3.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 3,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.t3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 4,
    "_isFastLaunch": true,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 5,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 6,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 7,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 8,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 9,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 10,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 11,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 12,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.m5d.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 13,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.m5d.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 14,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.m5d.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 15,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.m5d.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 16,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.m5d.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 17,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.m5d.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 18,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.m5d.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 19,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.m5d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 20,
    "_isFastLaunch": false,
    "category": "General purpose",
    "gpuNum": 0,
    "hideHardwareSpecs": true,
    "memoryGiB": 0,
    "name": "ml.geospatial.interactive",
    "supportedImageNames": [
     "sagemaker-geospatial-v1-0"
    ],
    "vcpuNum": 0
   },
   {
    "_defaultOrder": 21,
    "_isFastLaunch": true,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 4,
    "name": "ml.c5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 22,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 8,
    "name": "ml.c5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 23,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.c5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 24,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.c5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 25,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 72,
    "name": "ml.c5.9xlarge",
    "vcpuNum": 36
   },
   {
    "_defaultOrder": 26,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 96,
    "name": "ml.c5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 27,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 144,
    "name": "ml.c5.18xlarge",
    "vcpuNum": 72
   },
   {
    "_defaultOrder": 28,
    "_isFastLaunch": false,
    "category": "Compute optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.c5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 29,
    "_isFastLaunch": true,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g4dn.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 30,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g4dn.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 31,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g4dn.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 32,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g4dn.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 33,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g4dn.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 34,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g4dn.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 35,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 61,
    "name": "ml.p3.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 36,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 244,
    "name": "ml.p3.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 37,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 488,
    "name": "ml.p3.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 38,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.p3dn.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 39,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.r5.large",
    "vcpuNum": 2
   },
   {
    "_defaultOrder": 40,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.r5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 41,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.r5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 42,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.r5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 43,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.r5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 44,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.r5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 45,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 512,
    "name": "ml.r5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 46,
    "_isFastLaunch": false,
    "category": "Memory Optimized",
    "gpuNum": 0,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.r5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 47,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 16,
    "name": "ml.g5.xlarge",
    "vcpuNum": 4
   },
   {
    "_defaultOrder": 48,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 32,
    "name": "ml.g5.2xlarge",
    "vcpuNum": 8
   },
   {
    "_defaultOrder": 49,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 64,
    "name": "ml.g5.4xlarge",
    "vcpuNum": 16
   },
   {
    "_defaultOrder": 50,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 128,
    "name": "ml.g5.8xlarge",
    "vcpuNum": 32
   },
   {
    "_defaultOrder": 51,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 1,
    "hideHardwareSpecs": false,
    "memoryGiB": 256,
    "name": "ml.g5.16xlarge",
    "vcpuNum": 64
   },
   {
    "_defaultOrder": 52,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 192,
    "name": "ml.g5.12xlarge",
    "vcpuNum": 48
   },
   {
    "_defaultOrder": 53,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 4,
    "hideHardwareSpecs": false,
    "memoryGiB": 384,
    "name": "ml.g5.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 54,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 768,
    "name": "ml.g5.48xlarge",
    "vcpuNum": 192
   },
   {
    "_defaultOrder": 55,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4d.24xlarge",
    "vcpuNum": 96
   },
   {
    "_defaultOrder": 56,
    "_isFastLaunch": false,
    "category": "Accelerated computing",
    "gpuNum": 8,
    "hideHardwareSpecs": false,
    "memoryGiB": 1152,
    "name": "ml.p4de.24xlarge",
    "vcpuNum": 96
   }
  ],
  "instance_type": "ml.m5.large",
  "kernelspec": {
   "display_name": "conda_pytorch_p310",
   "language": "python",
   "name": "conda_pytorch_p310"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}