Arcface_Training_Code.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 5,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.12"
    },
    "colab": {
      "name": "Arcface_Training_Code.ipynb",
      "provenance": [],
      "collapsed_sections": []
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "incomplete-estimate"
      },
      "source": [
        "import tensorflow as tf\n",
        "import math\n",
        "import time\n",
        "import numpy as np"
      ],
      "id": "incomplete-estimate",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Oe6aDoIVoe_S"
      },
      "source": [
        "from IPython.display import clear_output\n",
        "import matplotlib.pyplot as plt"
      ],
      "id": "Oe6aDoIVoe_S",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5JGHxhN11ArQ",
        "outputId": "af5c6ca0-77a7-4f87-a863-811910f5a07a"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "id": "5JGHxhN11ArQ",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GhoR8pEMuYZX"
      },
      "source": [
        "!cp /content/drive/MyDrive/Arcface/backbone_0.py .\n",
        "!cp /content/drive/MyDrive/Arcface/hypar.py .\n",
        "!cp /content/drive/MyDrive/Arcface/network_16.py .\n",
        "import hypar\n",
        "import backbone_0 as nn\n",
        "import network_16 as net"
      ],
      "id": "GhoR8pEMuYZX",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "damaged-exploration"
      },
      "source": [
        "physical_devices = tf.config.experimental.list_physical_devices('GPU')\n",
        "assert len(physical_devices) > 0, \"Not enough GPU hardware devices available\"\n",
        "config = tf.config.experimental.set_memory_growth(physical_devices[0], True)"
      ],
      "id": "damaged-exploration",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "H89tHgtiE8X3",
        "outputId": "ea043f75-3cb6-4442-cdcb-b2144076e4fe"
      },
      "source": [
        "physical_devices"
      ],
      "id": "H89tHgtiE8X3",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "assured-short"
      },
      "source": [
        "X = np.load('/content/drive/MyDrive/Arcface/x_train.npy', allow_pickle=True)\n",
        "Y = np.load('/content/drive/MyDrive/Arcface/y_train.npy', allow_pickle=True)"
      ],
      "id": "assured-short",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "configured-period",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3e856770-61e6-4641-9e75-2e4752117ce6"
      },
      "source": [
        "X = np.array(X, dtype='float32')\n",
        "Y = np.array(Y, dtype='int32')\n",
        "Y = np.reshape(Y, Y.shape[0])\n",
        "print(\"X shape:\",X.shape,\"Y shape:\",Y.shape) \n",
        "X = net.Resnet_preprocess(X)\n",
        "images = X\n",
        "labels = Y\n",
        "\n",
        "# Prepare the training dataset.\n",
        "train_dataset = tf.data.Dataset.from_tensor_slices((images, labels))\n",
        "train_dataset = train_dataset.shuffle(buffer_size=1024).batch(hypar.batch_size)\n",
        "print(\"Training dataset ready!\")"
      ],
      "id": "configured-period",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "X shape: (1476, 112, 112, 3) Y shape: (1476,)\n",
            "Training dataset ready!\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bigger-century"
      },
      "source": [
        "num_classes = hypar.num_classes #number of people in the dataset\n",
        "\n",
        "#feature vector dimension = (512) [comes from the resnet model]\n",
        "\n",
        "class Arcface_Layer(tf.keras.layers.Layer):                                     # Arcface layer definition\n",
        "    def __init__(self, num_outputs = num_classes, s=64., m=0.5):                 # s is scale factor, m is the margin to be added to the angle 'theta'\n",
        "        super(Arcface_Layer, self).__init__()\n",
        "        self.num_outputs = num_outputs\n",
        "        self.s = s\n",
        "        self.m = m\n",
        "\n",
        "    def build(self, input_shape):\n",
        "\n",
        "        self.kernel = self.add_weight(name='weight',\n",
        "                                          shape=(input_shape[-1],self.num_outputs),\n",
        "                                          initializer='glorot_uniform',\n",
        "                                          regularizer=tf.keras.regularizers.l2(l=5e-4),\n",
        "                                          trainable=True)\n",
        "        super(Arcface_Layer, self).build(input_shape)\n",
        "\n",
        "\n",
        "    def call(self, feature_vec, ground_truth_vec):                              # inputs is the 512 dimension feature vector\n",
        "        mask = tf.one_hot(ground_truth_vec, num_classes)                        # dims of mask is (num_classes), it is a one-hot vector\n",
        "        inv_mask = tf.subtract(1., mask)\n",
        "        \n",
        "        # feature vector and weights norm\n",
        "        x = feature_vec\n",
        "        norm_x = tf.norm(feature_vec, axis=1, keepdims=True)\n",
        "        norm_W = tf.norm(self.kernel, axis=0, keepdims=True)\n",
        "        \n",
        "        x = tf.math.divide(x, norm_x)\n",
        "        W = tf.math.divide(self.kernel,norm_W)\n",
        "        #cos_theta = tf.matmul(tf.transpose(W),tf.transpose(x))  # logit of  W.t*x\n",
        "        cos_theta = tf.matmul(x, W)\n",
        "        theta = tf.math.acos(cos_theta)                                         # all angle between each class' weight and x\n",
        "        theta_class = tf.multiply(theta,mask)                                   # increasing angle theta of the class x belongs to alone\n",
        "        theta_class_added_margin = theta_class + self.m\n",
        "        theta_class_added_margin = theta_class_added_margin*mask\n",
        "        cos_theta_margin = tf.math.cos(theta_class_added_margin)\n",
        "        s_cos_t = tf.multiply(self.s, cos_theta_margin)\n",
        "        s_cos_j = tf.multiply(self.s,tf.multiply(cos_theta,inv_mask))\n",
        "\n",
        "        output = tf.add(s_cos_t,s_cos_j)\n",
        "\n",
        "        return output"
      ],
      "id": "bigger-century",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "portable-shade"
      },
      "source": [
        "class train_model(tf.keras.Model):\n",
        "    def __init__(self):\n",
        "        super(train_model, self).__init__()\n",
        "        #self.resnet = net.Resnet_nn()\n",
        "        self.resnet = net.Resnet()\n",
        "        self.arcface = Arcface_Layer()\n",
        "\n",
        "    def call(self, x, y):\n",
        "        x = self.resnet(x)\n",
        "        return self.arcface(x, y)\n",
        "\n",
        "# Instantiate a loss function.\n",
        "def loss_fxn(logits,labels):\n",
        "    loss_fn = tf.reduce_mean(tf.nn.sparse_softmax_cross_entropy_with_logits(logits=logits, labels=labels))\n",
        "    return loss_fn\n",
        "    \n",
        "# Instantiate an optimizer to train the model.\n",
        "learning_rate = 0.0005\n",
        "#optimizer = tf.keras.optimizers.SGD(learning_rate=learning_rate, momentum=0.9, nesterov=False)\n",
        "optimizer = tf.keras.optimizers.Adam(\n",
        "    learning_rate=0.00001, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False,\n",
        "    name='Adam'\n",
        ")\n",
        "\n",
        "model = train_model()\n",
        "\n",
        "@tf.function\n",
        "def train_step(images, labels):\n",
        "    with tf.GradientTape() as tape:\n",
        "        logits = model(images,labels)\n",
        "        pred = tf.nn.softmax(logits)\n",
        "        #inf_loss = loss_fxn(pred,labels)\n",
        "        inf_loss = loss_fxn(logits,labels)\n",
        "        loss = inf_loss\n",
        "    gradients = tape.gradient(loss, model.trainable_variables)\n",
        "    optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "    train_loss = tf.reduce_mean(loss)\n",
        "    accuracy = tf.reduce_mean(tf.cast(tf.equal(tf.argmax(pred, axis=1, output_type=tf.dtypes.int32), tf.cast(labels,dtype = tf.int32)), dtype=tf.float32))\n",
        "    inference_loss = tf.reduce_mean(inf_loss)\n",
        "    regularization_loss = 0\n",
        "    return accuracy, train_loss, inference_loss, regularization_loss"
      ],
      "id": "portable-shade",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "aHdyzzFC_rq4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7fae1658-59f1-4fd9-afc3-0e11fbedbf29"
      },
      "source": [
        "epochs = 20\n",
        "reg_coef = 1.0\n",
        "\n",
        "file_name = '/content/drive/MyDrive/Arcface/checkpoint/try4_16/model_weights_'\n",
        "file_number = 10\n",
        "file_name = file_name + str(file_number*epochs)+ '_epochs'\n",
        "model = tf.keras.models.load_model(file_name)"
      ],
      "id": "aHdyzzFC_rq4",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "WARNING:tensorflow:No training configuration found in save file, so the model was *not* compiled. Compile it manually.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K_UtYDB612eO",
        "outputId": "28016cb5-5601-4482-e0f5-87fd7c785c18"
      },
      "source": [
        "print(file_name)"
      ],
      "id": "K_UtYDB612eO",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/content/drive/MyDrive/Arcface/checkpoint/try4_16/model_weights_200_epochs\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "angry-currency",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 851
        },
        "outputId": "fe392d9d-b97c-409c-ee3a-b83645862692"
      },
      "source": [
        "random_out = model(X[0:3,:,:,:],Y[0:3])\n",
        "loss_log = []\n",
        "\n",
        "epochs = 50\n",
        "reg_coef = 1.0\n",
        "file_number = 5\n",
        "\n",
        "for save_wt in range (0, 5):\n",
        "  for epoch in range(epochs):\n",
        "      print(\"\\nStart of epoch %d\" % (epoch,))\n",
        "      start_time = time.time()\n",
        "\n",
        "      # Iterate over the batches of the dataset.\n",
        "      for step, (x_batch_train, y_batch_train) in enumerate(train_dataset):\n",
        "          accuracy, train_loss, inference_loss, regularization_loss = train_step(x_batch_train, y_batch_train)\n",
        "          if step % 20 == 0:\n",
        "            loss_log.append(train_loss)\n",
        "            print(\"Training loss (for one batch) at step %d: %.4f\"% (step, float(train_loss)))\n",
        "      \n",
        "  file_number += 1\n",
        "  file_name = '/content/drive/MyDrive/Arcface/checkpoint/try4_16/model_weights_'\n",
        "  file_name = file_name + str(file_number*epochs)+ '_epochs'\n",
        "  model.save(file_name)"
      ],
      "id": "angry-currency",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Start of epoch 0\n",
            "Training loss (for one batch) at step 0: 36.7603\n",
            "Training loss (for one batch) at step 20: 31.7190\n",
            "Training loss (for one batch) at step 40: 35.1816\n",
            "Training loss (for one batch) at step 60: 34.8949\n",
            "Training loss (for one batch) at step 80: 34.2491\n",
            "\n",
            "Start of epoch 1\n",
            "Training loss (for one batch) at step 0: 35.5965\n",
            "Training loss (for one batch) at step 20: 35.9811\n",
            "Training loss (for one batch) at step 40: 34.2415\n",
            "Training loss (for one batch) at step 60: 35.7493\n",
            "Training loss (for one batch) at step 80: 33.2042\n",
            "\n",
            "Start of epoch 2\n",
            "Training loss (for one batch) at step 0: 33.7470\n",
            "Training loss (for one batch) at step 20: 34.9552\n",
            "Training loss (for one batch) at step 40: 33.3988\n",
            "Training loss (for one batch) at step 60: nan\n",
            "Training loss (for one batch) at step 80: nan\n",
            "\n",
            "Start of epoch 3\n",
            "Training loss (for one batch) at step 0: nan\n",
            "Training loss (for one batch) at step 20: nan\n",
            "Training loss (for one batch) at step 40: nan\n",
            "Training loss (for one batch) at step 60: nan\n",
            "Training loss (for one batch) at step 80: nan\n"
          ]
        },
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-17-5f06e1708409>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     13\u001b[0m       \u001b[0;31m# Iterate over the batches of the dataset.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m       \u001b[0;32mfor\u001b[0m \u001b[0mstep\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mx_batch_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_batch_train\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 15\u001b[0;31m           \u001b[0maccuracy\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minference_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mregularization_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_step\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx_batch_train\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_batch_train\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     16\u001b[0m           \u001b[0;32mif\u001b[0m \u001b[0mstep\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0;36m20\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     17\u001b[0m             \u001b[0mloss_log\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loss\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    883\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    884\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 885\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    886\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    887\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    915\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3038\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3039\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3040\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3042\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1962\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1963\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1964\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1965\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1966\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    594\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    597\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dkn-YrGqdcJt"
      },
      "source": [
        "loss_log_data = np.array(loss_log)\n",
        "np.save('/content/drive/MyDrive/Arcface/checkpoint/try4_16/loss_log/loss_log_1.npy', loss_log_data)"
      ],
      "id": "dkn-YrGqdcJt",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V0C7irVSe2Wc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "994f4cd2-3b01-4ae1-9d8b-8bcf53fdc165"
      },
      "source": [
        "test_model = model.resnet\n",
        "result = test_model.predict(X)\n",
        "result.shape"
      ],
      "id": "V0C7irVSe2Wc",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Exception ignored in: <function IteratorResourceDeleter.__del__ at 0x7f3305652ef0>\n",
            "Traceback (most recent call last):\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/tensorflow/python/data/ops/iterator_ops.py\", line 546, in __del__\n",
            "    handle=self._handle, deleter=self._deleter)\n",
            "  File \"/usr/local/lib/python3.7/dist-packages/tensorflow/python/ops/gen_dataset_ops.py\", line 1264, in delete_iterator\n",
            "    _ctx, \"DeleteIterator\", name, handle, deleter)\n",
            "KeyboardInterrupt: \n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(1476, 512)"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1-DPkdau5L3q"
      },
      "source": [
        "index = []\n",
        "for i in range (0,1470):\n",
        "    if Y[i] == 25:\n",
        "        index.append(i)"
      ],
      "id": "1-DPkdau5L3q",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YNkfgK3KPkM8",
        "outputId": "8b4f002e-4916-41bc-d660-594821655cc7"
      },
      "source": [
        "index"
      ],
      "id": "YNkfgK3KPkM8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0,\n",
              " 6,\n",
              " 26,\n",
              " 44,\n",
              " 162,\n",
              " 163,\n",
              " 165,\n",
              " 251,\n",
              " 253,\n",
              " 364,\n",
              " 398,\n",
              " 399,\n",
              " 461,\n",
              " 494,\n",
              " 507,\n",
              " 518,\n",
              " 524,\n",
              " 583,\n",
              " 730,\n",
              " 789,\n",
              " 863,\n",
              " 877,\n",
              " 991,\n",
              " 1002,\n",
              " 1041,\n",
              " 1051,\n",
              " 1068,\n",
              " 1158,\n",
              " 1178,\n",
              " 1184,\n",
              " 1308,\n",
              " 1415]"
            ]
          },
          "metadata": {},
          "execution_count": 58
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4mJP43S5kg8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "10aa4360-686a-4959-e3b7-d95ff7eefc69"
      },
      "source": [
        "a = np.dot(result[877,:], np.array(result[1308,:]).T)\n",
        "print(a)"
      ],
      "id": "L4mJP43S5kg8",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "1440517.4\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "o9hXk2dAPN79"
      },
      "source": [
        "from sklearn.preprocessing import normalize\n",
        "results = normalize(result, axis = 0)\n",
        "results.shape"
      ],
      "id": "o9hXk2dAPN79",
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OXbmaykCSnPV",
        "outputId": "9b96a2f9-c598-4216-aa6b-34f548181f83"
      },
      "source": [
        "model.arcface.get_weights()"
      ],
      "id": "OXbmaykCSnPV",
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[array([[-0.11767528,  0.14456384, -0.02135386, ..., -0.03348209,\n",
              "         -0.00698351, -0.05040066],\n",
              "        [-0.03981043,  0.02772743, -0.06435528, ...,  0.09751327,\n",
              "          0.06037353,  0.1113415 ],\n",
              "        [ 0.04373817, -0.03364272, -0.10783768, ...,  0.06325577,\n",
              "          0.03565495,  0.04472327],\n",
              "        ...,\n",
              "        [-0.01150434, -0.03776852, -0.04783383, ...,  0.00264406,\n",
              "         -0.0351619 ,  0.05422439],\n",
              "        [-0.10210086, -0.06197488, -0.05334978, ...,  0.0618397 ,\n",
              "          0.06696489,  0.05739159],\n",
              "        [-0.00232715, -0.08407147, -0.12328175, ...,  0.00945917,\n",
              "          0.03173999,  0.11961782]], dtype=float32)]"
            ]
          },
          "metadata": {},
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sWhKJMGvFVlg"
      },
      "source": [
        ""
      ],
      "id": "sWhKJMGvFVlg",
      "execution_count": null,
      "outputs": []
    }
  ]
}