Thyroid_Experiment.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "lAnvkkKKj7yU"
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import, division, print_function, unicode_literals"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "gg5B1yY7kA6w",
    "outputId": "1069c0f6-918b-4add-e4fa-97c82b033222"
   },
   "outputs": [],
   "source": [
    "try:\n",
    "  # %tensorflow_version only exists in Colab.\n",
    "  %tensorflow_version 2.x\n",
    "except Exception:\n",
    "  pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IOSmuaY0kBXR"
   },
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "ejbzd6zdDLMy",
    "outputId": "81059252-57fe-4f18-ef9e-e93ec4b905d4"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.0'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 85
    },
    "colab_type": "code",
    "id": "22aE3_QdkEpc",
    "outputId": "5bb26a9f-43e9-4ea1-a7f6-b098435b47e2"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: imageio in /usr/local/lib/python3.6/dist-packages (2.6.1)\n",
      "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from imageio) (1.17.2)\n",
      "Requirement already satisfied: pillow in /usr/local/lib/python3.6/dist-packages (from imageio) (6.2.1)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 19.3.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# To generate GIFs\n",
    "!pip install imageio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "colab_type": "code",
    "id": "qDwo4ClNlTts",
    "outputId": "0281d884-156a-4b51-ecce-15efc8a879a9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tensorflow_privacy in /usr/local/lib/python3.6/dist-packages (0.2.2)\n",
      "Requirement already satisfied: mpmath in /usr/local/lib/python3.6/dist-packages (from tensorflow_privacy) (1.1.0)\n",
      "Requirement already satisfied: scipy>=0.17 in /usr/local/lib/python3.6/dist-packages (from tensorflow_privacy) (1.3.3)\n",
      "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.6/dist-packages (from scipy>=0.17->tensorflow_privacy) (1.17.2)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 19.3.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# To use Differential Privacy\n",
    "!pip install tensorflow_privacy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gjGtjZWokG36"
   },
   "outputs": [],
   "source": [
    "import glob\n",
    "import imageio\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "import PIL\n",
    "from tensorflow.keras import layers, models\n",
    "import time\n",
    "\n",
    "from IPython import display\n",
    "\n",
    "from tensorflow_privacy.privacy.analysis import compute_dp_sgd_privacy\n",
    "from tensorflow_privacy.privacy.optimizers.dp_optimizer import DPGradientDescentGaussianOptimizer\n",
    "from tensorflow_privacy.privacy.optimizers.dp_optimizer import DPAdamGaussianOptimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "JuCo5voGT-Jb"
   },
   "source": [
    "## Modified Optimizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hOBQELw7mo9Z"
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "from absl import logging\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "import collections\n",
    "\n",
    "from tensorflow_privacy.privacy.analysis import privacy_ledger\n",
    "from tensorflow_privacy.privacy.dp_query import gaussian_query\n",
    "from tensorflow_privacy.privacy.dp_query import no_privacy_query\n",
    "\n",
    "\n",
    "def make_optimizer_class(cls):\n",
    "  \"\"\"Constructs a DP optimizer class from an existing one.\"\"\"\n",
    "  parent_code = tf.compat.v1.train.Optimizer.compute_gradients.__code__\n",
    "  child_code = cls.compute_gradients.__code__\n",
    "  GATE_OP = tf.compat.v1.train.Optimizer.GATE_OP  # pylint: disable=invalid-name\n",
    "  if child_code is not parent_code:\n",
    "    logging.warning(\n",
    "        'WARNING: Calling make_optimizer_class() on class %s that overrides '\n",
    "        'method compute_gradients(). Check to ensure that '\n",
    "        'make_optimizer_class() does not interfere with overridden version.',\n",
    "        cls.__name__)\n",
    "\n",
    "  class DPOptimizerClass(cls):\n",
    "    \"\"\"Differentially private subclass of given class cls.\"\"\"\n",
    "\n",
    "    _GlobalState = collections.namedtuple(\n",
    "      '_GlobalState', ['l2_norm_clip', 'stddev'])\n",
    "    \n",
    "    def __init__(\n",
    "        self,\n",
    "        dp_sum_query,\n",
    "        num_microbatches=None,\n",
    "        unroll_microbatches=False,\n",
    "        *args,  # pylint: disable=keyword-arg-before-vararg, g-doc-args\n",
    "        **kwargs):\n",
    "      \"\"\"Initialize the DPOptimizerClass.\n",
    "\n",
    "      Args:\n",
    "        dp_sum_query: DPQuery object, specifying differential privacy\n",
    "          mechanism to use.\n",
    "        num_microbatches: How many microbatches into which the minibatch is\n",
    "          split. If None, will default to the size of the minibatch, and\n",
    "          per-example gradients will be computed.\n",
    "        unroll_microbatches: If true, processes microbatches within a Python\n",
    "          loop instead of a tf.while_loop. Can be used if using a tf.while_loop\n",
    "          raises an exception.\n",
    "      \"\"\"\n",
    "      super(DPOptimizerClass, self).__init__(*args, **kwargs)\n",
    "      self._dp_sum_query = dp_sum_query\n",
    "      self._num_microbatches = num_microbatches\n",
    "      self._global_state = self._dp_sum_query.initial_global_state()\n",
    "      # TODO(b/122613513): Set unroll_microbatches=True to avoid this bug.\n",
    "      # Beware: When num_microbatches is large (>100), enabling this parameter\n",
    "      # may cause an OOM error.\n",
    "      self._unroll_microbatches = unroll_microbatches\n",
    "\n",
    "    def compute_gradients(self,\n",
    "                          loss,\n",
    "                          var_list,\n",
    "                          gate_gradients=GATE_OP,\n",
    "                          aggregation_method=None,\n",
    "                          colocate_gradients_with_ops=False,\n",
    "                          grad_loss=None,\n",
    "                          gradient_tape=None,\n",
    "                          SET_noise=0,\n",
    "                          SET_clip=10):\n",
    "\n",
    "      self._dp_sum_query = gaussian_query.GaussianSumQuery(SET_clip, SET_clip*SET_noise)\n",
    "      self._global_state = self._dp_sum_query.make_global_state(SET_clip, SET_clip*SET_noise)\n",
    "      \n",
    "\n",
    "\n",
    "\n",
    "      # TF is running in Eager mode, check we received a vanilla tape.\n",
    "      if not gradient_tape:\n",
    "        raise ValueError('When in Eager mode, a tape needs to be passed.')\n",
    "\n",
    "      vector_loss = loss()\n",
    "      if self._num_microbatches is None:\n",
    "        self._num_microbatches = tf.shape(input=vector_loss)[0]\n",
    "      sample_state = self._dp_sum_query.initial_sample_state(var_list)\n",
    "      microbatches_losses = tf.reshape(vector_loss, [self._num_microbatches, -1])\n",
    "      sample_params = (self._dp_sum_query.derive_sample_params(self._global_state))\n",
    "\n",
    "      def process_microbatch(i, sample_state):\n",
    "        \"\"\"Process one microbatch (record) with privacy helper.\"\"\"\n",
    "        microbatch_loss = tf.reduce_mean(input_tensor=tf.gather(microbatches_losses, [i]))\n",
    "        grads = gradient_tape.gradient(microbatch_loss, var_list)\n",
    "        sample_state = self._dp_sum_query.accumulate_record(sample_params, sample_state, grads)\n",
    "        return sample_state\n",
    "    \n",
    "      for idx in range(self._num_microbatches):\n",
    "        sample_state = process_microbatch(idx, sample_state)\n",
    "\n",
    "      if SET_noise > 0:\n",
    "        grad_sums, self._global_state = (self._dp_sum_query.get_noised_result(sample_state, self._global_state))\n",
    "      else:\n",
    "        grad_sums = sample_state\n",
    "\n",
    "      def normalize(v):\n",
    "        return v / tf.cast(self._num_microbatches, tf.float32)\n",
    "\n",
    "      final_grads = tf.nest.map_structure(normalize, grad_sums)\n",
    "      grads_and_vars = final_grads#list(zip(final_grads, var_list))\n",
    "    \n",
    "      return grads_and_vars\n",
    "\n",
    "  return DPOptimizerClass\n",
    "\n",
    "\n",
    "def make_gaussian_optimizer_class(cls):\n",
    "  \"\"\"Constructs a DP optimizer with Gaussian averaging of updates.\"\"\"\n",
    "\n",
    "  class DPGaussianOptimizerClass(make_optimizer_class(cls)):\n",
    "    \"\"\"DP subclass of given class cls using Gaussian averaging.\"\"\"\n",
    "\n",
    "    def __init__(\n",
    "        self,\n",
    "        l2_norm_clip,\n",
    "        noise_multiplier,\n",
    "        num_microbatches=None,\n",
    "        ledger=None,\n",
    "        unroll_microbatches=False,\n",
    "        *args,  # pylint: disable=keyword-arg-before-vararg\n",
    "        **kwargs):\n",
    "      dp_sum_query = gaussian_query.GaussianSumQuery(\n",
    "          l2_norm_clip, l2_norm_clip * noise_multiplier)\n",
    "\n",
    "      if ledger:\n",
    "        dp_sum_query = privacy_ledger.QueryWithLedger(dp_sum_query,\n",
    "                                                      ledger=ledger)\n",
    "\n",
    "      super(DPGaussianOptimizerClass, self).__init__(\n",
    "          dp_sum_query,\n",
    "          num_microbatches,\n",
    "          unroll_microbatches,\n",
    "          *args,\n",
    "          **kwargs)\n",
    "\n",
    "    @property\n",
    "    def ledger(self):\n",
    "      return self._dp_sum_query.ledger\n",
    "\n",
    "  return DPGaussianOptimizerClass\n",
    "\n",
    "\n",
    "GradientDescentOptimizer = tf.compat.v1.train.GradientDescentOptimizer\n",
    "DPGradientDescentGaussianOptimizer_NEW = make_gaussian_optimizer_class(GradientDescentOptimizer)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Dnmy7SGjT8qx"
   },
   "source": [
    "## START"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "IuUBjz5ZEn5D"
   },
   "outputs": [],
   "source": [
    "base_dir = '/storage/TF2'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5BjbuiWuEv__"
   },
   "outputs": [],
   "source": [
    "name_trial = '/touseexp'\n",
    "\n",
    "result_dir = base_dir + '/GANPoster' + name_trial"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XOcY7Xxnplya"
   },
   "outputs": [],
   "source": [
    "# Define checkpoint dir and prefix\n",
    "checkpoint_dir = result_dir + '/training_checkpoints'\n",
    "\n",
    "def checkpoint_name(title):  \n",
    "  checkpoint_prefix = os.path.join(checkpoint_dir, \"ckpt__\" + str(title))\n",
    "  return(checkpoint_prefix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "gubUb2LBF2Hj"
   },
   "outputs": [],
   "source": [
    "if not os.path.exists(result_dir):\n",
    "  os.makedirs(result_dir)\n",
    "\n",
    "if not os.path.exists(checkpoint_dir):\n",
    "  os.makedirs(checkpoint_dir)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "ehOkmDl__ZyH"
   },
   "source": [
    "## C-GAN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "PVBOYFJskMTZ"
   },
   "outputs": [],
   "source": [
    "def make_generator_model_FCC():\n",
    "    # INPUT: label input\n",
    "    in_label = layers.Input(shape=(COND_num_classes,))\n",
    "\n",
    "    # INPUT: image generator input\n",
    "    in_lat = layers.Input(shape=(Z_DIM,))\n",
    "\n",
    "    # MERGE\n",
    "    merge = layers.concatenate([in_lat, in_label], axis=1)\n",
    "\n",
    "    ge1 = layers.Dense(128, use_bias=True)(merge)\n",
    "    ge1 = layers.ReLU()(ge1)\n",
    "\n",
    "    out_layer = layers.Dense(21, use_bias=True, activation=\"tanh\")(ge1)\n",
    "\n",
    "    model = models.Model([in_lat, in_label], out_layer)\n",
    "\n",
    "    return model\n",
    "\n",
    "def make_discriminator_model_FCC():\n",
    "    # INPUT: Label\n",
    "    in_label = layers.Input(shape=(COND_num_classes,))\n",
    "\n",
    "    # INPUT: Image\n",
    "    in_image = layers.Input(shape=(21,))\n",
    "\n",
    "    # MERGE\n",
    "    merge = layers.concatenate([in_image, in_label], axis=1)\n",
    "\n",
    "    ge1 = layers.Dense(128, use_bias=True)(merge)\n",
    "    ge1 = layers.ReLU()(ge1)\n",
    "\n",
    "    out_layer = layers.Dense(1, use_bias=True)(ge1)\n",
    "\n",
    "    model = models.Model([in_image, in_label], out_layer)\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "RqMlVrs5G9bI"
   },
   "source": [
    "## SETUP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zAn0CETqHzAz"
   },
   "outputs": [],
   "source": [
    "# SETUP\n",
    "COND_num_classes = 3\n",
    "\n",
    "Z_DIM = 100 # Does NOT affect EPSILON\n",
    "NORM_CLIP = 1.1 # Does NOT affect EPSILON, but increases NOISE on gradients\n",
    "\n",
    "DP_DELTA = 1e-5\n",
    "\n",
    "NOISE_MULT = 1.15\n",
    "EPOCHS = 50\n",
    "\n",
    "BUFFER_SIZE = 3772\n",
    "\n",
    "BATCH_SIZE = 32\n",
    "\n",
    "N_DISC = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 105
    },
    "colab_type": "code",
    "id": "VocXhKB5lSEV",
    "outputId": "96502b09-fd36-4152-de2d-4d51efb3c336"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "DP-SGD with sampling rate = 0.848% and noise_multiplier = 1.15 iterated over 5894 steps satisfies differential privacy with eps = 3.72 and delta = 1e-05.\n",
      "The optimal RDP order is 7.0.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(3.718745314966683, 7.0)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Obtain DP_EPSILON\n",
    "compute_dp_sgd_privacy.compute_dp_sgd_privacy(n = BUFFER_SIZE, \n",
    "                                              batch_size = BATCH_SIZE, \n",
    "                                              noise_multiplier = NOISE_MULT, \n",
    "                                              epochs = EPOCHS, \n",
    "                                              delta = DP_DELTA)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "id": "R1WvIihNKwXI",
    "outputId": "4b017caf-fa4e-49cf-9906-11362202657a"
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.265"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# SD of noise\n",
    "NOISE_MULT*NORM_CLIP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "14pCCe5KpYJz"
   },
   "outputs": [],
   "source": [
    "generator = make_generator_model_FCC()\n",
    "discriminator = make_discriminator_model_FCC()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "zl2cQPbrLI5P"
   },
   "outputs": [],
   "source": [
    "#generator.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "ZrlnUgnYLKRW"
   },
   "outputs": [],
   "source": [
    "#discriminator.summary()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "NMMbWnKoNa99"
   },
   "source": [
    "### OPTIMIZERS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "UovjSm3wKG3h"
   },
   "outputs": [],
   "source": [
    "import tensorflow.keras.backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "Yj0E2qo3kHb0"
   },
   "outputs": [],
   "source": [
    "###################\n",
    "## OPTIMIZERS\n",
    "generator_optimizer = tf.keras.optimizers.Adam()\n",
    "\n",
    "lr_disc = tf.compat.v1.train.polynomial_decay(learning_rate=0.150,\n",
    "                                              global_step=tf.compat.v1.train.get_or_create_global_step(),\n",
    "                                              decay_steps=10000,\n",
    "                                              end_learning_rate=0.052,\n",
    "                                              power=1)\n",
    "\n",
    "discriminator_optimizer = DPGradientDescentGaussianOptimizer_NEW(\n",
    "   learning_rate = lr_disc,\n",
    "   l2_norm_clip = NORM_CLIP,\n",
    "   noise_multiplier = NOISE_MULT,\n",
    "   num_microbatches = BATCH_SIZE)\n",
    "\n",
    "# DP\n",
    "cross_entropy_DISC = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.losses.Reduction.NONE)\n",
    "\n",
    "# Not-DP\n",
    "cross_entropy_GEN = tf.keras.losses.BinaryCrossentropy(from_logits=True)\n",
    "\n",
    "###################\n",
    "## LOSS AND UPDATES\n",
    "\n",
    "# Notice the use of `tf.function`: This annotation causes the function to be \"compiled\".\n",
    "@tf.function\n",
    "def train_step_DISC_BOTH(images, labels, noise):    \n",
    "    with tf.GradientTape(persistent=True) as disc_tape_real:\n",
    "      # This dummy call is needed to obtain the var list.\n",
    "      dummy = discriminator([images, labels], training=True)\n",
    "      var_list = discriminator.trainable_variables\n",
    "\n",
    "      # In Eager mode, the optimizer takes a function that returns the loss.\n",
    "      def loss_fn_real():\n",
    "        real_output = discriminator([images, labels], training=True)\n",
    "        disc_real_loss = cross_entropy_DISC(tf.ones_like(real_output), real_output)\n",
    "        return disc_real_loss\n",
    "      \n",
    "      grads_and_vars_real = discriminator_optimizer.compute_gradients(loss_fn_real, var_list, \n",
    "                                                                      gradient_tape=disc_tape_real, \n",
    "                                                                      SET_noise=NOISE_MULT,\n",
    "                                                                      SET_clip=NORM_CLIP)\n",
    "      \n",
    "      # In Eager mode, the optimizer takes a function that returns the loss.\n",
    "      def loss_fn_fake():\n",
    "        generated_images = generator([noise, labels], training=True)\n",
    "        fake_output = discriminator([generated_images, labels], training=True)\n",
    "        disc_fake_loss = cross_entropy_DISC(tf.zeros_like(fake_output), fake_output)\n",
    "        return disc_fake_loss\n",
    "      \n",
    "      grads_and_vars_fake = discriminator_optimizer.compute_gradients(loss_fn_fake, var_list, \n",
    "                                                                      gradient_tape=disc_tape_real,\n",
    "                                                                      SET_noise=0,\n",
    "                                                                      SET_clip=NORM_CLIP)\n",
    "    \n",
    "    norm_r = 0 #tf.linalg.global_norm(grads_and_vars_real)\n",
    "    norm_f = 0 #tf.linalg.global_norm(grads_and_vars_fake)\n",
    "        \n",
    "    s_grads_and_vars = [(grads_and_vars_real[idx] + grads_and_vars_fake[idx]) for idx in \n",
    "                        range(len(grads_and_vars_real))]\n",
    "    sanitized_grads_and_vars = list(zip(s_grads_and_vars, var_list))\n",
    "    \n",
    "    discriminator_optimizer.apply_gradients(sanitized_grads_and_vars)\n",
    "    \n",
    "    return(norm_r, norm_f)\n",
    "\n",
    "\n",
    "################################################################################################\n",
    "################################################################################################\n",
    "\n",
    "# Notice the use of `tf.function`: This annotation causes the function to be \"compiled\".\n",
    "@tf.function\n",
    "def train_step_GEN(labels, noise):\n",
    "    with tf.GradientTape() as gen_tape:\n",
    "      generated_images = generator([noise, labels], training=True)\n",
    "      fake_output = discriminator([generated_images, labels], training=True)\n",
    "      # if the generator is performing well, the discriminator will classify the fake images as real (or 1)\n",
    "      gen_loss = cross_entropy_GEN(tf.ones_like(fake_output), fake_output)\n",
    "\n",
    "    gradients_of_generator = gen_tape.gradient(gen_loss, generator.trainable_variables)\n",
    "    generator_optimizer.apply_gradients(zip(gradients_of_generator, generator.trainable_variables))\n",
    "\n",
    "    return(gen_loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "n6hVijBPlfg8"
   },
   "source": [
    "## TRAIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "tGI0EgGjoTrQ"
   },
   "outputs": [],
   "source": [
    "# CREATE CHECKPOINT STRUCTURE\n",
    "checkpoint = tf.train.Checkpoint(generator_optimizer=generator_optimizer,\n",
    "                                 discriminator_optimizer=discriminator_optimizer,\n",
    "                                 generator=generator,\n",
    "                                 discriminator=discriminator)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "AUflcdXYnV_M"
   },
   "outputs": [],
   "source": [
    "def train_main_loop(dataset, title):\n",
    "  for epoch in range(EPOCHS):\n",
    "    start = time.time()\n",
    "\n",
    "    i_gen = 0\n",
    "    for image_batch, label_batch in dataset:      \n",
    "      noise = tf.random.normal([BATCH_SIZE, Z_DIM])\n",
    "      #print(str(i_gen) + \" ------------------------------------------------------ \")\n",
    "      d_r, d_f = train_step_DISC_BOTH(image_batch, label_batch, noise)\n",
    "      #print(\"------ NORM REAL: \" + str(d_r))\n",
    "      #print(\"------ NORM FAKE: \" + str(d_f))\n",
    "\n",
    "      #if (i_gen + 1) % N_DISC == 0:\n",
    "      g_f = train_step_GEN(label_batch, noise)\n",
    "      \n",
    "      i_gen = i_gen + 1      \n",
    "\n",
    "    print ('Time for epoch {} is {} sec'.format(epoch + 1, time.time()-start))\n",
    "\n",
    "    # Save the model\n",
    "    checkpoint.save(file_prefix = checkpoint_name(title + \"__epoch=\" + str(epoch) + \"__\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "9Sb7NTLhqyh8"
   },
   "source": [
    "## Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(\"/storage/TF2/ann-train.data\", sep=\" \",  header=None)\n",
    "train.drop(train.columns[[22, 23]], axis=1, inplace=True)\n",
    "trainXpre = train.loc[:,range(21)].values\n",
    "trainYpre = train.loc[:,21]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.73 , 0.   , 1.   , ..., 0.12 , 0.082, 0.146],\n",
       "       [0.24 , 0.   , 0.   , ..., 0.143, 0.133, 0.108],\n",
       "       [0.47 , 0.   , 0.   , ..., 0.102, 0.131, 0.078],\n",
       "       ...,\n",
       "       [0.88 , 0.   , 0.   , ..., 0.123, 0.099, 0.124],\n",
       "       [0.64 , 1.   , 0.   , ..., 0.106, 0.088, 0.121],\n",
       "       [0.46 , 0.   , 0.   , ..., 0.093, 0.091, 0.102]])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trainXpre"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " ...\n",
      " [0. 1. 0.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "trainYpre = trainYpre.values\n",
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "trainYpre = trainYpre.reshape(len(trainYpre), 1)\n",
    "trainY = onehot_encoder.fit_transform(trainYpre)\n",
    "print(trainY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.77659574, 0.        , 1.        , ..., 0.27906977, 0.35344828,\n",
       "        0.23856209],\n",
       "       [0.25531915, 0.        , 0.        , ..., 0.33255814, 0.57327586,\n",
       "        0.17647059],\n",
       "       [0.5       , 0.        , 0.        , ..., 0.2372093 , 0.56465517,\n",
       "        0.12745098],\n",
       "       ...,\n",
       "       [0.93617021, 0.        , 0.        , ..., 0.28604651, 0.42672414,\n",
       "        0.20261438],\n",
       "       [0.68085106, 1.        , 0.        , ..., 0.24651163, 0.37931034,\n",
       "        0.19771242],\n",
       "       [0.4893617 , 0.        , 0.        , ..., 0.21627907, 0.39224138,\n",
       "        0.16666667]])"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn import preprocessing\n",
    "\n",
    "trainX = preprocessing.maxabs_scale(trainXpre)\n",
    "trainX"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "457S-cC38uPO",
    "outputId": "cbc04237-e522-4ae6-f223-82bf42775bb6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3772, 21)\n",
      "(3772, 3)\n"
     ]
    }
   ],
   "source": [
    "print(trainX.shape)\n",
    "print(trainY.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "7UpjHhpSq9_W"
   },
   "source": [
    "# Train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "nTJgxxIqqw5n"
   },
   "outputs": [],
   "source": [
    "tf.random.set_seed(1)\n",
    "train_dataset = tf.data.Dataset.from_tensor_slices((trainX, trainY)).shuffle(BUFFER_SIZE, reshuffle_each_iteration=True).batch(BATCH_SIZE, drop_remainder=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "BVDi7vlNqBlB"
   },
   "outputs": [],
   "source": [
    "training_title = 'TD_first'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 1000
    },
    "colab_type": "code",
    "id": "IL6rZttmrBOp",
    "outputId": "37984a96-9103-463e-debf-a76d3aacd62b"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:Layer concatenate_1 is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "WARNING:tensorflow:Calling GradientTape.gradient on a persistent tape inside its context is significantly less efficient than calling it outside the context (it causes the gradient ops to be recorded on the tape, leading to increased CPU and memory usage). Only call GradientTape.gradient inside the context if you actually want to trace the gradient in order to compute higher order derivatives.\n",
      "WARNING:tensorflow:Layer concatenate is casting an input tensor from dtype float64 to the layer's dtype of float32, which is new behavior in TensorFlow 2.  The layer has dtype float32 because it's dtype defaults to floatx.\n",
      "\n",
      "If you intended to run this layer in float32, you can safely ignore this warning. If in doubt, this warning is likely only an issue if you are porting a TensorFlow 1.X model to TensorFlow 2.\n",
      "\n",
      "To change all layers to have dtype float64 by default, call `tf.keras.backend.set_floatx('float64')`. To change just this layer, pass dtype='float64' to the layer constructor. If you are the author of this layer, you can disable autocasting by passing autocast=False to the base Layer constructor.\n",
      "\n",
      "Time for epoch 1 is 11.149208545684814 sec\n",
      "Time for epoch 2 is 2.570085048675537 sec\n",
      "Time for epoch 3 is 2.548340082168579 sec\n",
      "Time for epoch 4 is 2.5091614723205566 sec\n",
      "Time for epoch 5 is 2.6506736278533936 sec\n",
      "Time for epoch 6 is 2.530118942260742 sec\n",
      "Time for epoch 7 is 2.3571605682373047 sec\n",
      "Time for epoch 8 is 2.460496664047241 sec\n",
      "Time for epoch 9 is 2.551248788833618 sec\n",
      "Time for epoch 10 is 2.4475300312042236 sec\n",
      "Time for epoch 11 is 2.443773031234741 sec\n",
      "Time for epoch 12 is 2.404229164123535 sec\n",
      "Time for epoch 13 is 2.3403360843658447 sec\n",
      "Time for epoch 14 is 2.644965648651123 sec\n",
      "Time for epoch 15 is 2.452285051345825 sec\n",
      "Time for epoch 16 is 2.6184136867523193 sec\n",
      "Time for epoch 17 is 2.4492416381835938 sec\n",
      "Time for epoch 18 is 2.475661277770996 sec\n",
      "Time for epoch 19 is 2.5186734199523926 sec\n",
      "Time for epoch 20 is 2.3564367294311523 sec\n",
      "Time for epoch 21 is 2.4804272651672363 sec\n",
      "Time for epoch 22 is 2.506974935531616 sec\n",
      "Time for epoch 23 is 2.4650402069091797 sec\n",
      "Time for epoch 24 is 2.5112011432647705 sec\n",
      "Time for epoch 25 is 2.4993865489959717 sec\n",
      "Time for epoch 26 is 2.5195834636688232 sec\n",
      "Time for epoch 27 is 2.5085761547088623 sec\n",
      "Time for epoch 28 is 2.6689889430999756 sec\n",
      "Time for epoch 29 is 2.543809175491333 sec\n",
      "Time for epoch 30 is 2.547311782836914 sec\n",
      "Time for epoch 31 is 2.5014424324035645 sec\n",
      "Time for epoch 32 is 2.525139570236206 sec\n",
      "Time for epoch 33 is 2.5257749557495117 sec\n",
      "Time for epoch 34 is 2.629650354385376 sec\n",
      "Time for epoch 35 is 2.4586377143859863 sec\n",
      "Time for epoch 36 is 2.716996908187866 sec\n",
      "Time for epoch 37 is 2.671062707901001 sec\n",
      "Time for epoch 38 is 2.4842448234558105 sec\n",
      "Time for epoch 39 is 2.499656915664673 sec\n",
      "Time for epoch 40 is 2.483309745788574 sec\n",
      "Time for epoch 41 is 2.5402612686157227 sec\n",
      "Time for epoch 42 is 2.5382139682769775 sec\n",
      "Time for epoch 43 is 2.6620380878448486 sec\n",
      "Time for epoch 44 is 2.5516626834869385 sec\n",
      "Time for epoch 45 is 2.4619081020355225 sec\n",
      "Time for epoch 46 is 2.5988872051239014 sec\n",
      "Time for epoch 47 is 2.4243249893188477 sec\n",
      "Time for epoch 48 is 2.599740743637085 sec\n",
      "Time for epoch 49 is 2.5589399337768555 sec\n",
      "Time for epoch 50 is 2.5565991401672363 sec\n"
     ]
    }
   ],
   "source": [
    "train_main_loop(train_dataset, training_title)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "HDXQ-UIbrXf_"
   },
   "source": [
    "## Classification"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'/storage/TF2/GANPoster/touseexp/training_checkpoints'"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint_dir"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tensorflow.python.training.tracking.util.CheckpointLoadStatus at 0x7f097be1a2e8>"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "checkpoint.restore('/storage/TF2/GANPoster/touseexp/training_checkpoints/ckpt__TD_first__epoch=49__-50')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "9DqLFm1Ix7cK"
   },
   "outputs": [],
   "source": [
    "tf.random.set_seed(100)\n",
    "N_GEN = 3772\n",
    "\n",
    "noise_GEN = tf.random.normal([N_GEN, Z_DIM])\n",
    "labels_GEN = tf.Variable(np.array([1,0,0]*93 + \n",
    "                                   [0,1,0]*191 +\n",
    "                                   [0,0,1]*3488, \n",
    "                                   dtype='float32').reshape((N_GEN,COND_num_classes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jkV2uEM4ykxF"
   },
   "outputs": [],
   "source": [
    "images_GEN = generator([noise_GEN, labels_GEN], training=False)\n",
    "images_tt = layers.Flatten()(images_GEN)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "wqAmbPcSy_aP"
   },
   "outputs": [],
   "source": [
    "labels_tt = tf.Variable(np.array([0]*93 + \n",
    "                                   [1]*191 +\n",
    "                                   [2]*3488, \n",
    "                                   dtype='float32').reshape((N_GEN,1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 51
    },
    "colab_type": "code",
    "id": "VHlJsKCWy1cb",
    "outputId": "e47a1844-d438-4478-8edc-705e5a8a2ce4"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3772, 1)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "TensorShape([3772, 21])"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(labels_tt.shape)\n",
    "images_tt.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "2npDQqY3zWHt"
   },
   "outputs": [],
   "source": [
    "Y_train = labels_tt[:images_tt.shape[0]]\n",
    "X_train = images_tt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "Qtb06W4a-nWD"
   },
   "source": [
    "### Cross Validation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 122
    },
    "colab_type": "code",
    "id": "YJNN15kNDLOf",
    "outputId": "7789a8dc-50b3-4813-f190-0f0b588984b1"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: sklearn in /usr/local/lib/python3.6/dist-packages (0.0)\n",
      "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sklearn) (0.22)\n",
      "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn) (0.14.0)\n",
      "Requirement already satisfied: numpy>=1.11.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn) (1.17.2)\n",
      "Requirement already satisfied: scipy>=0.17.0 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sklearn) (1.3.3)\n",
      "\u001b[33mWARNING: You are using pip version 19.2.3, however version 19.3.1 is available.\n",
      "You should consider upgrading via the 'pip install --upgrade pip' command.\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "!pip install sklearn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9915164369034994\n",
      "{'alpha': 1e-08, 'hidden_layer_sizes': 16, 'max_iter': 300, 'random_state': 1, 'solver': 'lbfgs'}\n",
      "112.29233694076538\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/usr/local/lib/python3.6/dist-packages/sklearn/neural_network/_multilayer_perceptron.py:470: ConvergenceWarning: lbfgs failed to converge (status=1):\n",
      "STOP: TOTAL NO. of ITERATIONS REACHED LIMIT.\n",
      "\n",
      "Increase the number of iterations (max_iter) or scale the data as shown in:\n",
      "    https://scikit-learn.org/stable/modules/preprocessing.html.\n",
      "  self.n_iter_ = _check_optimize_result(\"lbfgs\", opt_res, self.max_iter)\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "parameters = {'solver': ['lbfgs', 'adam'], \n",
    "              'max_iter': [300], \n",
    "              'alpha': 10.0 ** -np.arange(1, 10), \n",
    "              'hidden_layer_sizes': np.arange(15, 20), \n",
    "              'random_state':[1]}\n",
    "clf = GridSearchCV(MLPClassifier(), parameters, n_jobs=-1)\n",
    "\n",
    "clf.fit(trainX, trainY)\n",
    "print(clf.score(trainX, trainY))\n",
    "print(clf.best_params_)\n",
    "print(time.time()-start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## TEST RESULTS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv(\"/storage/TF2/ann-test.data\", sep=\" \",  header=None)\n",
    "test.drop(test.columns[[22, 23]], axis=1, inplace=True)\n",
    "testXpre = test.loc[:,range(21)]\n",
    "testX = preprocessing.maxabs_scale(testXpre)\n",
    "\n",
    "testYpre = test.loc[:,21]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[0. 1. 0.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " ...\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]\n",
      " [0. 0. 1.]]\n"
     ]
    }
   ],
   "source": [
    "from sklearn.preprocessing import OneHotEncoder\n",
    "\n",
    "testYpre = testYpre.values\n",
    "onehot_encoder = OneHotEncoder(sparse=False)\n",
    "testYpre = testYpre.reshape(len(testYpre), 1)\n",
    "testY = onehot_encoder.fit_transform(testYpre)\n",
    "print(testY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.956242707117853"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Accuracy\n",
    "clf.score(testX, testY)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn import metrics\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "def multiclass_roc_auc_score(y_test, y_pred, average=\"micro\"):\n",
    "    lb = LabelBinarizer()\n",
    "    lb.fit(y_test)\n",
    "    y_test = lb.transform(y_test)\n",
    "    y_pred = lb.transform(y_pred)\n",
    "    return metrics.roc_auc_score(y_test, y_pred, average=average)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9746207701283548"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# AuROC\n",
    "multiclass_roc_auc_score(testY, clf.predict(testX))"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [
    "nqaSIxRXegHt"
   ],
   "name": "CUR__opt_DP_C_DCGAN_MNIST.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}