1-2.Word2Vec/Word2Vec.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import mindspore\n",
    "import mindspore.nn as nn\n",
    "import mindspore.ops as ops\n",
    "import matplotlib.pyplot as plt\n",
    "from mindspore import Tensor, ms_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def random_batch():\n",
    "    random_inputs = []\n",
    "    random_labels = []\n",
    "    random_index = np.random.choice(range(len(skip_grams)), batch_size, replace=False)\n",
    "\n",
    "    for i in random_index:\n",
    "        random_inputs.append(np.eye(voc_size)[skip_grams[i][0]])  # target\n",
    "        random_labels.append(skip_grams[i][1])  # context word\n",
    "\n",
    "    return random_inputs, random_labels"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Word2Vec(nn.Cell):\n",
    "    def __init__(self, voc_size, embed_size):\n",
    "        super(Word2Vec, self).__init__()\n",
    "        # W and WT is not Traspose relationship\n",
    "        self.W = nn.Dense(voc_size, embed_size, has_bias=False) # voc_size > embedding_size Weight\n",
    "        self.WT = nn.Dense(embed_size, voc_size, has_bias=False) # embedding_size > voc_size Weight\n",
    "        \n",
    "    def construct(self, X):\n",
    "        # X : [batch_size, voc_size]\n",
    "        hidden_layer = self.W(X) # hidden_layer : [batch_size, embedding_size]\n",
    "        output_layer = self.WT(hidden_layer) # output_layer : [batch_size, voc_size]\n",
    "        return output_layer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "batch_size = 2 # mini-batch size\n",
    "embed_size = 2 # embedding size\n",
    "\n",
    "sentences = [\"apple banana fruit\", \"banana orange fruit\", \"orange banana fruit\",\n",
    "             \"dog cat animal\", \"cat monkey animal\", \"monkey dog animal\"]\n",
    "\n",
    "word_sequence = \" \".join(sentences).split()\n",
    "word_list = \" \".join(sentences).split()\n",
    "word_list = list(set(word_list))\n",
    "word_dict = {w: i for i, w in enumerate(word_list)}\n",
    "voc_size = len(word_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Make skip gram of one size window\n",
    "skip_grams = []\n",
    "for i in range(1, len(word_sequence) - 1):\n",
    "    target = word_dict[word_sequence[i]]\n",
    "    context = [word_dict[word_sequence[i - 1]], word_dict[word_sequence[i + 1]]]\n",
    "    for w in context:\n",
    "        skip_grams.append([target, w])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Word2Vec(voc_size, embed_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = nn.Adam(model.trainable_params(), learning_rate=0.001)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def forward(inputs, targets):\n",
    "    logits = model(inputs)\n",
    "    loss = criterion(logits, targets)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "grad_fn = ops.value_and_grad(forward, None, optimizer.parameters)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "@ms_function\n",
    "def train_step(inputs, targets):\n",
    "    loss, grads = grad_fn(inputs, targets)\n",
    "    optimizer(grads)\n",
    "    return loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch: 1000 cost =  1.538420\n",
      "Epoch: 2000 cost =  1.268426\n",
      "Epoch: 3000 cost =  1.192565\n",
      "Epoch: 4000 cost =  1.204078\n",
      "Epoch: 5000 cost =  1.086928\n"
     ]
    }
   ],
   "source": [
    "model.set_train()\n",
    "\n",
    "epoch = 5000\n",
    "for step in range(epoch):\n",
    "    input_batch, target_batch = random_batch()\n",
    "    input_batch = Tensor(input_batch, mindspore.float32)\n",
    "    target_batch = Tensor(target_batch, mindspore.int32)\n",
    "    loss = train_step(input_batch, target_batch)\n",
    "    if (step + 1) % 1000 == 0:\n",
    "        print('Epoch:', '%04d' % (step + 1), 'cost = ', '{:.6f}'.format(loss.asnumpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAD8CAYAAAB0IB+mAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAe7klEQVR4nO3de3RU5b3/8fc3IQQFDVZAqFy1QCEXIASkcjGCVRArQvGuWD2IglS0rb/SxakGz7LLVls5VD0WDipaLFIUhUqLyqWCghI0plwr0silCOF+EZCQ7++PDDlccnUmMxP257XWrOz97Gee5zuzyIedfZkxd0dERM58CbEuQEREokOBLyISEAp8EZGAUOCLiASEAl9EJCAU+CIiARF24JtZCzNbaGarzWyVmY0po4+Z2UQzW29m+WaWGe68IiJSPXUiMEYR8FN3/9jMzgFWmNk77r76hD4DgLahxyXA/4R+iohIlIS9h+/uW93949DyfmANcOEp3QYBL3mJZUBDM2sW7twi8cbMWpvZyir0yzazS6NRk8hxkdjDL2VmrYEuwIenbLoQ2HTC+uZQ29aKxmvUqJG3bt06ghWK1Ky0tDTWr19PVlZWhbewN2vWjISEBLKysqJVmgTEihUrdrh747K2RSzwzawB8BrwgLvvC2OcEcAIgJYtW5KbmxuhCkVOVlBQQP/+/enRowcffPAB3bp148477+SRRx5h+/btTJs2je985zvcddddbNiwgbPPPptJkyaRkZFBTk4OGzduZMOGDWzcuJEHHniA+++/n9/+9reMGzeOo0ePctFFF/Hpp59y9tlnU1xczLZt27jwwgtJTk7m6NGjJCUlkZ+fz7x587j88svZt28fnTp14p///CdJSUmxfnukljKzL8rbFpHAN7MkSsJ+mru/XkaXLUCLE9abh9pO4+6TgElApXtJIuFav349f/7zn3n++efp1q0br7zyCkuWLGH27Nn86le/okWLFnTp0oU33niDBQsWMGzYMPLy8gBYu3YtCxcuZP/+/bRv354+ffrw9NNP07p1a2bMmMHQoUOZOnUqvXr14oorruDWW29l165d3HDDDdx0003cd999rFq1ir179wIwffp0hgwZorCXGhN24JuZAVOANe7+u3K6zQZGm9l0Sk7W7nX3Cg/niERDmzZtSE9PByA1NZV+/fphZqSnp1NQUMAXX3zBa6+9BkDfvn3ZuXMn+/aV/AE7cOBAkpOTSU5OpkmTJsyZM4err76amTNnMmjQIF5//XWOHTvGFVdcwcKFC3n//fepW7cu8+fPZ//+/QAMHz6c3/zmN1x33XW88MILTJ48OTZvhARCJK7D7wncDvQ1s7zQ42ozu9fM7g31mQtsANYDk4FREZhXJGzJycmlywkJCaXrCQkJFBUVVfm5iYmJHDt2DICUlBRatmzJkiVL+PGPf8zdd9/NBRdcwN/+9jcyMzPJy8tj9OjRAPTs2ZOCggIWLVrEsWPHSEtLi/RLFCkV9h6+uy8BrJI+DtwX7lwi0da7d2+mTZvGL3/5SxYtWkSjRo0499xzy+x76aWXcs8995CcnMysWbPo27cv27Zto23btrRp04bx48cD4O7s37+f4x9NPmzYMG655RZ++ctfRu11STDpTluRCuTk5LBixQoyMjIYO3YsU6dOLbdvu3btGD16NAUFBVx66aVcfPHF1KlTh4EDB7J//34KCgrIzc0lNTWVo0ePMmvWLDp37szFF1/M7t27ufnmm6P4yiSILJ6/ACUrK8t1lY6cKf754ZcsffNzDuw6QoNvJfO9QRfT7pKmzJw5kzfffJOXX3451iXKGcDMVrh7mdf7ag9fJAr++eGXLJy2lgO7jgBwYNcR5k/JY/C3v8tPhw3jgZ49Y1yhBIECXyQKlr75OUVfF5/UVpxQl4FX/hd/a9mK+s/9gb1z5sSoOgkKBb5IFBzfsz/VkeRvAeCHD7P9qQlRrEiCSIEvEgUNvpVcZnvykV2ly0VbdWuK1CwFvkgUfG/QxdSpe/KvW8KxI1y8YXbpep1m+jxBqVkR/fA0ESlbu0uaAoSu0jlMvSO7uejzN2m6veQqNKtXjyYPPhDDCiUIFPgiUdLukqalwb93zhy2P/Vvisyo06wZTR58gJQf/CDGFcqZToEvEgMpP/iBAl6iTsfwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAkKBLyISEBEJfDN73sy2m9nKcrZnm9leM8sLPR6OxLwiIlJ1kfosnReBp4GXKuiz2N2vidB8IiJSTRHZw3f394BdlXYUEZGYieYx/O+Z2adm9lczSy2vk5mNMLNcM8stLCyMYnkiIme2aAX+x0Ard+8E/B54o7yO7j7J3bPcPatx48ZRKk9E5MwXlcB3933ufiC0PBdIMrNG0ZhbRERKRCXwzaypmVlouXto3p3RmFtEREpE5CodM/sTkA00MrPNwCNAEoC7PwcMBUaaWRFwCLjJ3T0Sc4uISNVEJPDd/eZKtj9NyWWbIiISI7rTVkQkIBT4IiIBocAXEQkIBb6ISEAo8EVEAkKBLyISEAp8EZGAUOCLiASEAl9EJCAU+CIiAaHAFxEJCAW+iEhAKPBFRAJCgS8iEhAKfBGRgFDgi4gEhAJfRCQgFPgiIgGhwBcRCYiIBL6ZPW9m281sZTnbzcwmmtl6M8s3s8xIzCsiIlUXqT38F4H+FWwfALQNPUYA/xOheUVEpIoiEvju/h6wq4Iug4CXvMQyoKGZNYvE3CIiUjXROoZ/IbDphPXNobbTmNkIM8s1s9zCwsKoFCciEgRxd9LW3Se5e5a7ZzVu3DjW5YiInDGiFfhbgBYnrDcPtYmISJREK/BnA8NCV+v0APa6+9YozS0iIkCdSAxiZn8CsoFGZrYZeARIAnD354C5wNXAeuAr4M5IzCsiIlUXkcB395sr2e7AfZGYS0REvpm4O2krIiI1Q4EvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAkKBLyISEAp8EZGAUOCLiASEAl9EJCAU+CIiAaHAFxEJCAW+iEhAKPBFRAJCgS8iEhAKfBGRgIhI4JtZfzNbZ2brzWxsGdt/ZGaFZpYXegyPxLwiIlJ1YX+JuZklAs8A3wc2A8vNbLa7rz6l66vuPjrc+URE5JuJxB5+d2C9u29w96+B6cCgCIwrIiIRFInAvxDYdML65lDbqX5oZvlmNtPMWkRgXhERqYZonbSdA7R29wzgHWBqeR3NbISZ5ZpZbmFhYZTKExE580Ui8LcAJ+6xNw+1lXL3ne5+JLT6v0DX8gZz90nunuXuWY0bN45AeSIiApEJ/OVAWzNrY2Z1gZuA2Sd2MLNmJ6xeC6yJwLwiIlINYV+l4+5FZjYamAckAs+7+yozexTIdffZwP1mdi1QBOwCfhTuvCIiUj3m7rGuoVxZWVmem5sb6zJERGoNM1vh7lllbdOdtiIiEfbcc8/x0ksvRWSs1q1bs2PHjoiMFfYhHREROdm9994b6xLKpD18EZEquO666+jatSupqalMmjQJgAYNGjBu3Dg6depEjx492LZtGwA5OTk8+eSTAGRnZ/Pggw+SlZVFhw4dWL58OUOGDKFt27b853/+Z4XjR5oCX0SkCp5//nlWrFhBbm4uEydOZOfOnRw8eJAePXrw6aef0qdPHyZPnlzmc+vWrUtubi733nsvgwYN4plnnmHlypW8+OKL7Ny5s9zxI02BLyJSBRMnTizdk9+0aROfffYZdevW5ZprrgGga9euFBQUlPnca6+9FoD09HRSU1Np1qwZycnJXHTRRWzatKnc8SNNx/BFRCqxaNEi3n33XZYuXcrZZ59NdnY2hw8fJikpCTMDIDExkaKiojKfn5ycDEBCQkLp8vH1oqKicsePNO3hi4hUYu/evZx33nmcffbZrF27lmXLltWq8Y9T4IuIVKJ///4UFRXRoUMHxo4dS48ePWrV+MfpxisRkThx8JPt7JtXwLE9R0hsmMy5V7Wmfpcm1Rqj1t54tX37djp06MCtt95a5edcffXV7Nmzhz179vDss8/WYHUiIpFz8JPt7Hn9M47tKfmcyWN7jrDn9c84+Mn2iM0R13v49erV8/Xr19O8efPStqKiIurUqfxcc0FBAddccw0rV66syRJFRCJi6+MflYb9iRIbJtNsbPcqj1Mr9/DN7Lmvv/6aAQMGkJKSwu23307Pnj25/fbbefHFFxk9+v++LfGaa65h0aJFwP/dhjx27Fg+//xzOnfuzEMPPRSjVyGxcuq/EZF4V1bYV9T+TcTtZZnufm9ycvI9Cxcu5Omnn2bOnDksWbKEs846ixdffLHS5z/++OOsXLmSvLy8Gq9VRCRciQ2Ty93Dj5S43cM/1bXXXstZZ50V6zIkCsq7hf3BBx8kNTWVfv36cfzb0LKzsxkzZgydO3cmLS2Njz766LTxCgsL+eEPf0i3bt3o1q0b77//flRfj0hVnHtVayzp5Ei2pATOvap1xOaoNYFfv3790uU6depQXFxcul4TNyhI7JR3C3tWVharVq3isssuY/z48aX9v/rqK/Ly8nj22We56667ThtvzJgxPPjggyxfvpzXXnuN4cOHR/PliFRJ/S5NaDikbekefWLDZBoOaVvtq3QqEreHdCrSunVrnn32WYqLi9myZUuZe3XnnHMO+/fvj0F1Eq6JEycya9YsgNJbzBMSErjxxhsBuO222xgyZEhp/5tvvhmAPn36sG/fPvbs2XPSeO+++y6rV68uXd+3bx8HDhygQYMGNfxKak5OTg4NGjTgZz/7WaxLkQiq36VJRAP+VLUy8Hv27EmbNm3o2LEjHTp0IDMz87Q+559/Pj179iQtLY0BAwbwxBNPxKBSqa6q3mJ+/Hb2U5fLWi8uLmbZsmXUq1evZooWqSXi+pBOeno6jRo1Iicn56Q9GTNj2rRprF27llmzZrFo0SKys7MBeGbBM9yy6BYypmawY8gOfj371wr7WqS8W8yLi4uZOXMmAK+88gq9evUqfc6rr74KwJIlS0hJSSElJeWkMa+88kp+//vfl67X1hP5jz32GO3ataNXr16sW7cOKHktPXr0ICMjg8GDB7N7924Ali9fTkZGRulVamlpabEsXeJEXAd+db214S1yPshh68GtOM7Wg1vJ+SCHtza8FevSpIrKu8W8fv36fPTRR6SlpbFgwQIefvjh0ufUq1ePLl26cO+99zJlypTTxpw4cSK5ublkZGTQsWNHnnvuuai9nkhZsWIF06dPJy8vj7lz57J8+XIAhg0bxq9//Wvy8/NJT08vPbdx55138oc//IG8vDwSExNjWbrEkbi+8aq6H61w5cwr2Xpw62ntzeo34+2hb0eyNImyBg0acODAgdPas7OzefLJJ8nKKvM+E8ifAfMfhb2bIaU59HsYMm6o4Wojb8KECezatYtHH30UgJ/85CekpKQwZcoUNm7cCMDnn3/O9ddfz4IFC+jUqRNffPEFAPn5+dxyyy26CTEgavzGKzPrb2brzGy9mY0tY3uymb0a2v6hmbWOxLyn+vLgl9VqlzNc/gyYcz/s3QR4yc8595e0iwRQ2IFvZonAM8AAoCNws5l1PKXbfwC73f07wFPAr8OdtyxN6zetVrvUHmXt3UPJSd5y9+7nPwpHD53cdvRQSXst06dPH9544w0OHTrE/v37mTNnDvXr1+e8885j8eLFALz88stcdtllNGzYkHPOOYcPP/wQgOnTp8eydIkjkdjD7w6sd/cN7v41MB0YdEqfQcDU0PJMoJ+deilFBIzJHEO9xJOvxKiXWI8xmWMiPZXUBns3V689jmVmZnLjjTfSqVMnBgwYQLdu3QCYOnUqDz30EBkZGeTl5ZWe25gyZQp33303nTt35uDBg6edyJZgisRlmRcCm05Y3wxcUl4fdy8ys73A+cCOUwczsxHACICWLVtWq5CBFw0E4L8//m++PPglTes3ZUzmmNJ2CZiU5qHDOWW010Ljxo1j3Lhxp7WX9WUZqamp5OfnAyUfM1LuX0ESKHF3Hb67TwImQclJ2+o+f+BFAxXwUqLfwyXH7E88rJN0Vkn7GWzN4oX87r/GM3f5J5CQwHfatWfm7DmxLkviQCQCfwvQ4oT15qG2svpsNrM6QAoQ+a9kFznR8atxzoCrdKpqzeKFvD3pab57XgO+e2VvAOrUTWbH2pU0bnx5jKuTWItE4C8H2ppZG0qC/SbgllP6zAbuAJYCQ4EFHs/Xg8qZI+OGMzrgT7V4+ksUfX3yJy4WfX2ExdNfokNvBX7QhR34oWPyo4F5QCLwvLuvMrNHgVx3nw1MAV42s/XALkr+UxCRCNu/87TTYhW2S7BE5Bi+u88F5p7S9vAJy4eB6yMxl4iU75zzG7F/R2GZ7SJn1EcriARd75uGUafuyV+YUaduMr1vGhajiiSexN1VOiLyzR0/Tr94+kvs37mDc85vRO+bhun4vQAKfJEzTofelyvgpUw6pCMiEhAKfBGRgFDgi4gEhAJfRCQgFPgiIgGhwBcRCQgFvohIQCjwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAiKswDezb5nZO2b2WejneeX0O2ZmeaHH7HDmFBGRbybcPfyxwHx3bwvMD62X5ZC7dw49rg1zThER+QbCDfxBwNTQ8lTgujDHExGRGhJu4F/g7ltDy18CF5TTr56Z5ZrZMjO7rqIBzWxEqG9uYWFhmOWJiMhxlX6JuZm9CzQtY9O4E1fc3c3MyxmmlbtvMbOLgAVm9g93/7ysju4+CZgEkJWVVd54IiJSTZUGvrtfUd42M9tmZs3cfauZNQO2lzPGltDPDWa2COgClBn4IiJSM8I9pDMbuCO0fAfw5qkdzOw8M0sOLTcCegKrw5xXRESqKdzAfxz4vpl9BlwRWsfMsszsf0N9OgC5ZvYpsBB43N0V+CIiUVbpIZ2KuPtOoF8Z7bnA8NDyB0B6OPOIiEj4dKetiEhAKPBFRAJCgS8iEhAKfBGRgFDgi4gEhAJfRCQgFPgiIgGhwBcRCQgFvohIQCjwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQkIBb6ISEAo8EVEAkKBLyISEGEFvpldb2arzKzYzLIq6NffzNaZ2XozGxvOnCIi8s2Eu4e/EhgCvFdeBzNLBJ4BBgAdgZvNrGOY84qISDXVCefJ7r4GwMwq6tYdWO/uG0J9pwODgNXhzC0iItUTjWP4FwKbTljfHGoTEZEoqnQP38zeBZqWsWmcu78Z6YLMbAQwAqBly5aRHl5EJLAqDXx3vyLMObYALU5Ybx5qK2++ScAkgKysLA9zbhERCYnGIZ3lQFsza2NmdYGbgNlRmFdERE4Q7mWZg81sM/A94C0zmxdq/7aZzQVw9yJgNDAPWAPMcPdV4ZUtIiLVFe5VOrOAWWW0/xu4+oT1ucDccOYSEZHw6E5bEZGAUOCLiASEAl9EJCAU+CIiAaHAFxEJCAW+iEhAKPBFRAJCgS8iEhAKfBGRgFDgi4gEhAJfRCQgFPgiIgGhwBcRCQgFvohIQCjwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBEVbgm9n1ZrbKzIrNLKuCfgVm9g8zyzOz3HDmlG+uoKCAtLS0WJchIjFSJ8znrwSGAH+oQt/L3X1HmPOJiMg3FNYevruvcfd1kSommtyd4uLiWJcRdUVFRdx666106NCBoUOH8tVXX/Hoo4/SrVs30tLSGDFiBO4OQHZ2Nj//+c/p3r077dq1Y/HixUDJXwq9e/cmMzOTzMxMPvjgAwAWLVpEdnY2Q4cO5bvf/S633npr6VjlzSEi0ROtY/gOvG1mK8xsREUdzWyEmeWaWW5hYWFYk/7ud78jLS2NtLQ0JkyYQEFBAe3bt2fYsGGkpaWxadMmRo4cSVZWFqmpqTzyyCOlz23dujWPPPIImZmZpKens3btWgAKCwv5/ve/T2pqKsOHD6dVq1bs2FHyh8sf//hHunfvTufOnbnnnns4duxYWPXXhHXr1jFq1CjWrFnDueeey7PPPsvo0aNZvnw5K1eu5NChQ/zlL38p7V9UVMRHH33EhAkTGD9+PABNmjThnXfe4eOPP+bVV1/l/vvvL+3/ySefMGHCBFavXs2GDRt4//33ASqcQ0Sio9LAN7N3zWxlGY9B1Zinl7tnAgOA+8ysT3kd3X2Su2e5e1bjxo2rMcXJVqxYwQsvvMCHH37IsmXLmDx5Mrt37+azzz5j1KhRrFq1ilatWvHYY4+Rm5tLfn4+f//738nPzy8do1GjRnz88ceMHDmSJ598EoDx48fTt29fVq1axdChQ9m4cSMAa9as4dVXX+X9998nLy+PxMREpk2b9o3rryktWrSgZ8+eANx2220sWbKEhQsXcskll5Cens6CBQtYtWpVaf8hQ4YA0LVrVwoKCgA4evQod999N+np6Vx//fWsXr26tH/37t1p3rw5CQkJdO7cufQ5Fc0hItFR6TF8d78i3EncfUvo53YzmwV0B94Ld9yKLFmyhMGDB1O/fn2gJLgWL15Mq1at6NGjR2m/GTNmMGnSJIqKiti6dSurV68mIyOj9DlQEnavv/566bizZs0CoH///px33nkAzJ8/nxUrVtCtWzcADh06RJMmTWryJX4jZnba+qhRo8jNzaVFixbk5ORw+PDh0u3JyckAJCYmUlRUBMBTTz3FBRdcwKeffkpxcTH16tU7rf+Jzzl8+HCFc4hIdNT4IR0zq29m5xxfBq6k5GRvTBz/DwDgX//6F08++STz588nPz+fgQMHVhp25XF37rjjDvLy8sjLy2PdunXk5OTUyGsIx8aNG1m6dCkAr7zyCr169QJK/po5cOAAM2fOrHSMvXv30qxZMxISEnj55ZcrPXR1/D2tzhwiEnnhXpY52Mw2A98D3jKzeaH2b5vZ3FC3C4AlZvYp8BHwlrv/LZx5q6J379688cYbfPXVVxw8eJBZs2bRu3fvk/rs27eP+vXrk5KSwrZt2/jrX/9a6bg9e/ZkxowZALz99tvs3r0bgH79+jFz5ky2b98OwK5du/jiiy8i/KrC1759e5555hk6dOjA7t27GTlyJHfffTdpaWlcddVVpX+hVGTUqFFMnTqVTp06sXbt2pP+Ey1Lw4YNqz2HiNQAd4/bR9euXT0cv/3tbz01NdVTU1P9qaee8n/961+empp6Up877rjD27Zt63379vXBgwf7Cy+84O7urVq18sLCQnd3X758uV922WXu7r5t2zbv27evp6am+vDhw71p06Z++PBhd3efPn26d+rUydPT0z0zM9OXLl0aVv0iItUF5Ho5mWoex5fHZWVleW5ufN2ndeTIERITE6lTpw5Lly5l5MiR5OXlkZ+fz/z589m7dy8pKSn069ev9FxA0L3xyRaemLeOf+85xLcbnsVDV7Xnui4XxroskTOSma1w9zJvhA33xqvA2bhxIzfccAPFxcXUrVuXyZMnk5+fz5w5czh69ChQcox7zpw5AIEP/Tc+2cIvXv8Hh46WHOffsucQv3j9HwAKfZEoU+BXU9u2bfnkk09OanvqqadKw/64o0ePMn/+/MAH/hPz1pWG/XGHjh7jiXnrFPgiUaYPT4uAvXv3Vqs9SP6951C12kWk5ijwIyAlJaVa7UHy7YZnVatdRGqOAj8C+vXrR1JS0kltSUlJ9OvXL0YVxY+HrmrPWUmJJ7WdlZTIQ1e1j1FFIsGlY/gRcPw4va7SOd3x4/S6Skck9nRZpojIGaSiyzJ1SEdEJCAU+CIiAaHAFxEJCAW+iEhAKPBFRAIirq/SMbP9QK38ztwoagToy+Erp/epavQ+VS7e36NW7l7m1wXG+3X468q7vEhKmFmu3qPK6X2qGr1PlavN75EO6YiIBIQCX0QkIOI98CfFuoBaQO9R1eh9qhq9T5Wrte9RXJ+0FRGRyIn3PXwREYmQuA58M3vCzNaaWb6ZzTKzhrGuKR6Z2fVmtsrMis2sVl49UJPMrL+ZrTOz9WY2Ntb1xCMze97MtpvZyljXEq/MrIWZLTSz1aHftzGxrqm64jrwgXeANHfPAP4J/CLG9cSrlcAQ4L1YFxJvzCwReAYYAHQEbjazjrGtKi69CPSPdRFxrgj4qbt3BHoA99W2f0txHfju/ra7F4VWlwHNY1lPvHL3Ne6uG9TK1h1Y7+4b3P1rYDowKMY1xR13fw/YFes64pm7b3X3j0PL+4E1QK36Yoe4DvxT3AX8NdZFSK1zIbDphPXN1LJfUok/ZtYa6AJ8GONSqiXmd9qa2btA0zI2jXP3N0N9xlHy59S0aNYWT6ryPolIzTOzBsBrwAPuvi/W9VRHzAPf3a+oaLuZ/Qi4BujnAb6GtLL3Scq1BWhxwnrzUJtItZlZEiVhP83dX491PdUV14d0zKw/8P+Aa939q1jXI7XScqCtmbUxs7rATcDsGNcktZCZGTAFWOPuv4t1Pd9EXAc+8DRwDvCOmeWZ2XOxLigemdlgM9sMfA94y8zmxbqmeBE66T8amEfJSbYZ7r4qtlXFHzP7E7AUaG9mm83sP2JdUxzqCdwO9A3lUZ6ZXR3roqpDd9qKiAREvO/hi4hIhCjwRUQCQoEvIhIQCnwRkYBQ4IuIBIQCX0QkIBT4IiIBocAXEQmI/w81nm1VM/4ougAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "for i, label in enumerate(word_list):\n",
    "    W, WT = model.get_parameters()\n",
    "    x, y = W[0][i].asnumpy(), W[1][i].asnumpy()\n",
    "    plt.scatter(x, y)\n",
    "    plt.annotate(label, xy=(x, y), xytext=(5, 2), textcoords='offset points', ha='right', va='bottom')\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.7.13 ('ms1.8')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "vscode": {
   "interpreter": {
    "hash": "bd0943702584cdb580f8947884f31a9fb49482f77f8c89ed6532de3aa180e7ba"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}