frameworks/tensorflow/tf_estimator_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "XeahyCWWxmqn"
   },
   "outputs": [],
   "source": [
    "!pip install tensorflow==1.14.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 34
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 804,
     "status": "ok",
     "timestamp": 1572228045573,
     "user": {
      "displayName": "王DevOps",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAzLB0C3whTHAdHpq24UrEWqGtbhJElQxTU5_b_4g=s64",
      "userId": "04300517850278510646"
     },
     "user_tz": -480
    },
    "id": "XENhFdT4waO9",
    "outputId": "9e6fbac3-4d0f-46eb-f472-bea7a743d4a4"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:516: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:517: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:518: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:519: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:520: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/python/framework/dtypes.py:525: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:541: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint8 = np.dtype([(\"qint8\", np.int8, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:542: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint8 = np.dtype([(\"quint8\", np.uint8, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:543: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint16 = np.dtype([(\"qint16\", np.int16, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:544: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_quint16 = np.dtype([(\"quint16\", np.uint16, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:545: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  _np_qint32 = np.dtype([(\"qint32\", np.int32, 1)])\n",
      "/Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorboard/compat/tensorflow_stub/dtypes.py:550: FutureWarning: Passing (type, 1) or '1type' as a synonym of type is deprecated; in a future version of numpy, it will be understood as (type, (1,)) / '(1,)type'.\n",
      "  np_resource = np.dtype([(\"resource\", np.ubyte, 1)])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow_transform/beam/common.py:51: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
      "\n",
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow_transform/beam/impl.py:283: The name tf.SparseTensorValue is deprecated. Please use tf.compat.v1.SparseTensorValue instead.\n",
      "\n",
      "TF version: 1.14.0\n"
     ]
    }
   ],
   "source": [
    "import tensorflow as tf\n",
    "import tensorflow_model_analysis as tfma\n",
    "import numpy as np\n",
    "import os\n",
    "\n",
    "print(\"TF version: {}\".format(tf.__version__))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "nkkQ9BJJwaPP"
   },
   "source": [
    "# Brief"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "vW1-z8HQwaPR"
   },
   "source": [
    "In this tutorial, you'll learn how to complete a model training, evaluation, and prediction via `tf.estimator` API."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "y0EJik0NwaPU"
   },
   "source": [
    "# Downloading and Processing Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "-S65N0oqwaPW"
   },
   "outputs": [],
   "source": [
    "from tensorflow.examples.tutorials.mnist import input_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 530
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 3391,
     "status": "ok",
     "timestamp": 1572228056973,
     "user": {
      "displayName": "王DevOps",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAzLB0C3whTHAdHpq24UrEWqGtbhJElQxTU5_b_4g=s64",
      "userId": "04300517850278510646"
     },
     "user_tz": -480
    },
    "id": "idlzuMT8waPd",
    "outputId": "369a4e81-a241-41f4-fad3-430fedda9239"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-4-bc4a15cc9771>:2: read_data_sets (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n",
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:260: maybe_download (from tensorflow.contrib.learn.python.learn.datasets.base) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please write your own downloading logic.\n",
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:262: extract_images (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /Users/jiankaiwang/devops/tmp/MNIST_data/train-images-idx3-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:267: extract_labels (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.data to implement this functionality.\n",
      "Extracting /Users/jiankaiwang/devops/tmp/MNIST_data/train-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:110: dense_to_one_hot (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use tf.one_hot on tensors.\n",
      "Extracting /Users/jiankaiwang/devops/tmp/MNIST_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /Users/jiankaiwang/devops/tmp/MNIST_data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From /Users/jiankaiwang/devops/pyenv/tfx/lib/python3.7/site-packages/tensorflow/contrib/learn/python/learn/datasets/mnist.py:290: DataSet.__init__ (from tensorflow.contrib.learn.python.learn.datasets.mnist) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use alternatives such as official/mnist/dataset.py from tensorflow/models.\n"
     ]
    }
   ],
   "source": [
    "if os.path.exists(\"/Users/jiankaiwang/devops/tmp/MNIST_data/\"):\n",
    "  mnist = input_data.read_data_sets(\"/Users/jiankaiwang/devops/tmp/MNIST_data/\", one_hot=True)\n",
    "else:\n",
    "  mnist = input_data.read_data_sets(\"/tmp/MNIST_data/\", one_hot=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "LOcsLJsGwaPj"
   },
   "source": [
    "# Building Model and Training"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "5nCQafQfwaPl"
   },
   "source": [
    "In `tf.Estimator` API, you have to complete both `model_fn` and `params`.\n",
    "* `model_fn`: define the neural network\n",
    "* `params`: define the hyperparameters used in training neural network\n",
    "\n",
    "The following is basic `model_fn` example.\n",
    "```python\n",
    "def model_fn(features, labels, mode, params):\n",
    "   # Logic to do the following:\n",
    "   # 1. Configure the model via TensorFlow operations\n",
    "   # 2. Define the loss function for training/evaluation\n",
    "   # 3. Define the training operation/optimizer\n",
    "   # 4. Generate predictions\n",
    "   # 5. Return predictions/loss/train_op/eval_metric_ops in EstimatorSpec object\n",
    "   return EstimatorSpec(mode, predictions, loss, train_op, eval_metric_ops)\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "hiQINEprwaPo"
   },
   "outputs": [],
   "source": [
    "def layer(input, weight_shape, bias_shape):\n",
    "    weight_std = (2.0 / weight_shape[0]) ** 0.5\n",
    "    w_init = tf.random_normal_initializer(stddev=weight_std)\n",
    "    b_init = tf.constant_initializer(value=0)\n",
    "    W = tf.get_variable(name=\"W\", shape=weight_shape, initializer=w_init)\n",
    "    b = tf.get_variable(name=\"b\", shape=bias_shape, initializer=b_init)\n",
    "    return tf.nn.relu(tf.matmul(input, W) + b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "5x4GYXEVwaPu"
   },
   "outputs": [],
   "source": [
    "def evaluate(output, y):\n",
    "\n",
    "  compare = tf.equal(tf.argmax(output, axis=1), tf.argmax(y, axis=1))\n",
    "  compare_to_float = tf.cast(compare, tf.float32)\n",
    "    \n",
    "  accuracy, updated_op = tf.metrics.mean(compare_to_float)\n",
    "\n",
    "  return accuracy, updated_op"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "jxz19hJCwaP0"
   },
   "outputs": [],
   "source": [
    "def model_fn(features, labels, mode, params):\n",
    "    \n",
    "    # build a mlp model\n",
    "    with tf.variable_scope(\"hidden1\"):\n",
    "        hidden1 = layer(features[\"image\"], [784, 1024], [1024])\n",
    "    with tf.variable_scope(\"hidden2\"):\n",
    "        hidden2 = layer(hidden1, [1024, 512], [512])\n",
    "    with tf.variable_scope(\"output\"):\n",
    "        output = layer(hidden2, [512, 10], [10])\n",
    "    \n",
    "    # define the result\n",
    "    result = tf.identity(output, name=\"class\")\n",
    "    predict_dict = {\"class\": result}\n",
    "    if mode == tf.estimator.ModeKeys.PREDICT:\n",
    "        return tf.estimator.EstimatorSpec(\n",
    "            mode=mode,\n",
    "            predictions=predict_dict)\n",
    "    \n",
    "    # define loss function\n",
    "    cross_entropy = tf.nn.softmax_cross_entropy_with_logits_v2(logits=output, labels=labels)\n",
    "    loss = tf.reduce_mean(cross_entropy)\n",
    "    \n",
    "    # define optimizer\n",
    "    optimizer = tf.train.GradientDescentOptimizer(learning_rate=params[\"learning_rate\"])\n",
    "    train_opt = optimizer.minimize(loss, global_step=tf.train.get_global_step())\n",
    "    \n",
    "    # metrics\n",
    "    eval_metric_ops = {\n",
    "        #\"accuracy\": tf.metrics.accuracy(labels, output)\n",
    "        \"accuracy\": evaluate(output, labels)\n",
    "    }\n",
    "    \n",
    "    # return an estimator spec\n",
    "    return tf.estimator.EstimatorSpec(\n",
    "        mode=mode,\n",
    "        loss=loss,\n",
    "        train_op=train_opt,\n",
    "        eval_metric_ops=eval_metric_ops)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "meUZR0aqwaP8"
   },
   "source": [
    "Start training a model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "MiQVvtyTwaP_"
   },
   "outputs": [],
   "source": [
    "batch_size = 32\n",
    "epochs = mnist.train.num_examples // 32\n",
    "\n",
    "model_params = {\"learning_rate\": 1e-4}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 190
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 1032,
     "status": "ok",
     "timestamp": 1572228062406,
     "user": {
      "displayName": "王DevOps",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAzLB0C3whTHAdHpq24UrEWqGtbhJElQxTU5_b_4g=s64",
      "userId": "04300517850278510646"
     },
     "user_tz": -480
    },
    "id": "szShnu5uwaQE",
    "outputId": "c6634155-7f83-457a-ced8-216fa4961bf6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Using default config.\n",
      "WARNING:tensorflow:Using temporary folder as model directory: /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmp0ypx7mmn\n",
      "INFO:tensorflow:Using config: {'_model_dir': '/var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmp0ypx7mmn', '_tf_random_seed': None, '_save_summary_steps': 100, '_save_checkpoints_steps': None, '_save_checkpoints_secs': 600, '_session_config': allow_soft_placement: true\n",
      "graph_options {\n",
      "  rewrite_options {\n",
      "    meta_optimizer_iterations: ONE\n",
      "  }\n",
      "}\n",
      ", '_keep_checkpoint_max': 5, '_keep_checkpoint_every_n_hours': 10000, '_log_step_count_steps': 100, '_train_distribute': None, '_device_fn': None, '_protocol': None, '_eval_distribute': None, '_experimental_distribute': None, '_experimental_max_worker_delay_secs': None, '_service': None, '_cluster_spec': <tensorflow.python.training.server_lib.ClusterSpec object at 0x12bd356d8>, '_task_type': 'worker', '_task_id': 0, '_global_id_in_cluster': 0, '_master': '', '_evaluation_master': '', '_is_chief': True, '_num_ps_replicas': 0, '_num_worker_replicas': 1}\n"
     ]
    }
   ],
   "source": [
    "nn = tf.estimator.Estimator(model_fn=model_fn, params=model_params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 615
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 5923,
     "status": "ok",
     "timestamp": 1572228067518,
     "user": {
      "displayName": "王DevOps",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAzLB0C3whTHAdHpq24UrEWqGtbhJElQxTU5_b_4g=s64",
      "userId": "04300517850278510646"
     },
     "user_tz": -480
    },
    "id": "r0N-m6JOwaQL",
    "outputId": "a99f0c63-ef07-4733-a421-88bbc6e4ca10",
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "<dtype: 'float32'>\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Create CheckpointSaverHook.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Saving checkpoints for 0 into /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmp0ypx7mmn/model.ckpt.\n",
      "INFO:tensorflow:loss = 2.376337, step = 1\n",
      "INFO:tensorflow:global_step/sec: 78.5194\n",
      "INFO:tensorflow:loss = 2.4521444, step = 101 (1.275 sec)\n",
      "INFO:tensorflow:Saving checkpoints for 200 into /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmp0ypx7mmn/model.ckpt.\n",
      "INFO:tensorflow:Loss for final step: 2.40667.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<tensorflow_estimator.python.estimator.estimator.Estimator at 0x12bd350f0>"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"image\": mnist.train.images.astype(\"float32\")},\n",
    "    y=mnist.train.labels.astype(\"float32\"),\n",
    "    num_epochs=epochs,\n",
    "    shuffle=True)\n",
    "\n",
    "# Train\n",
    "nn.train(input_fn=train_input_fn, steps=200)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "YTtPoDAnwaQQ"
   },
   "source": [
    "# Evaluating the model."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 221
    },
    "colab_type": "code",
    "executionInfo": {
     "elapsed": 7081,
     "status": "ok",
     "timestamp": 1572187944327,
     "user": {
      "displayName": "王DevOps",
      "photoUrl": "https://lh3.googleusercontent.com/a-/AAuE7mAzLB0C3whTHAdHpq24UrEWqGtbhJElQxTU5_b_4g=s64",
      "userId": "04300517850278510646"
     },
     "user_tz": -480
    },
    "id": "_SVkD_yuwaQS",
    "outputId": "7dd3ec16-7d83-479e-f9dd-27cee836f4cb"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Starting evaluation at 2019-11-05T16:34:19Z\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmpz84lwrvd/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "INFO:tensorflow:Finished evaluation at 2019-11-05-16:34:20\n",
      "INFO:tensorflow:Saving dict for global step 2000: accuracy = 0.3318, global_step = 2000, loss = 2.1445124\n",
      "INFO:tensorflow:Saving 'checkpoint_path' summary for global step 2000: /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmpz84lwrvd/model.ckpt-2000\n",
      "Loss: 2.1445124\n",
      "Accuracy: 0.3318\n"
     ]
    }
   ],
   "source": [
    "# Score accuracy\n",
    "test_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"image\": mnist.test.images.astype(\"float32\")},\n",
    "    y=mnist.test.labels.astype(\"float32\"),\n",
    "    num_epochs=1,\n",
    "    shuffle=False)\n",
    "\n",
    "ev = nn.evaluate(input_fn=test_input_fn)\n",
    "print(\"Loss: %s\" % ev[\"loss\"])\n",
    "print(\"Accuracy: %s\" % ev[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The following is an example for training and evaluating.\n",
    "\n",
    "```python\n",
    "train_spec = tf.estimator.TrainSpec(\n",
    "                input_fn=generate_input_fn(data_train),\n",
    "                max_steps=3000)\n",
    "\n",
    "exporter = tf.estimator.LatestExporter('Servo', serving_input_fn)\n",
    "\n",
    "eval_spec=tf.estimator.EvalSpec(\n",
    "            input_fn=generate_input_fn(data_test),\n",
    "            steps=1,\n",
    "            exporters=exporter)\n",
    "\n",
    "tf.estimator.train_and_evaluate(generate_estimator(output_dir), train_spec, eval_spec)\n",
    "```"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab_type": "text",
    "id": "iOW1cILqwaQX"
   },
   "source": [
    "# Prediction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "llMPJVohwaQZ",
    "outputId": "377f6eee-ed43-4ea9-94e5-94387c374931"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Graph was finalized.\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmpz84lwrvd/model.ckpt-2000\n",
      "INFO:tensorflow:Running local_init_op.\n",
      "INFO:tensorflow:Done running local_init_op.\n",
      "Prediction 1: [0.24514045 0.7282403  0.40821418 0.1869801  0.         0.\n",
      " 0.         0.         0.3044557  0.        ]\n",
      "Prediction 2: [0.887169   0.         0.37492734 0.         0.0809705  0.5858083\n",
      " 0.3675366  0.         0.7051594  0.        ]\n"
     ]
    }
   ],
   "source": [
    "# Print out predictions\n",
    "predict_input_fn = tf.estimator.inputs.numpy_input_fn(\n",
    "    x={\"image\": mnist.test.images[2:4].astype('float32')},\n",
    "    num_epochs=1,\n",
    "    shuffle=False)\n",
    "\n",
    "predictions = nn.predict(input_fn=predict_input_fn)\n",
    "for i, p in enumerate(predictions):\n",
    "  print(\"Prediction %s: %s\" % (i + 1, p[\"class\"]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "colab": {},
    "colab_type": "code",
    "id": "8ja8Fr8CwaQg"
   },
   "source": [
    "# Exporting the SavedModel\n",
    "\n",
    "A simple example for a `serving_input_fn`.\n",
    "\n",
    "```python\n",
    "def serving_input_fn():\n",
    "  #feature_placeholders are what the caller of the predict() method will have to provide\n",
    "  feature_placeholders = {\n",
    "      column.name: tf.placeholder(column.dtype, [None])\n",
    "      for column in feature_cols\n",
    "  }\n",
    "  \n",
    "  #features are what we actually pass to the estimator\n",
    "  features = {\n",
    "    # Inputs are rank 1 so that we can provide scalars to the server\n",
    "    # but Estimator expects rank 2, so we expand dimension\n",
    "    key: tf.expand_dims(tensor, -1)\n",
    "    for key, tensor in feature_placeholders.items()\n",
    "  }\n",
    "  return tf.estimator.export.ServingInputReceiver(\n",
    "    features, feature_placeholders\n",
    "  )\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "ServingInputReceiver(features={'image': <tf.Tensor 'Placeholder:0' shape=(?, 784) dtype=float32>}, receiver_tensors={'image_content': <tf.Tensor 'Placeholder:0' shape=(?, 784) dtype=float32>}, receiver_tensors_alternatives=None)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def serving_input_receiver_fn():\n",
    "    \"\"\"Serving input_fn that builds features from placeholders\n",
    "\n",
    "    Returns\n",
    "    -------\n",
    "    tf.estimator.export.ServingInputReceiver\n",
    "    \"\"\"\n",
    "    \n",
    "    content = tf.placeholder(dtype=tf.float32, shape=[None, 784])\n",
    "    \n",
    "    # receiver_tensors here can be replaced by `content`\n",
    "    receiver_tensors = {'image_content': content}\n",
    "    \n",
    "    # definition to features must be the same with data input in trainer (tf.estimator.inputs.numpy_input_fn)\n",
    "    features = {'image': content}\n",
    "    return tf.estimator.export.ServingInputReceiver(features, receiver_tensors)\n",
    "\n",
    "serving_input_receiver_fn()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO:tensorflow:Calling model_fn.\n",
      "INFO:tensorflow:Done calling model_fn.\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Classify: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Regress: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Predict: ['serving_default']\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Train: None\n",
      "INFO:tensorflow:Signatures INCLUDED in export for Eval: None\n",
      "INFO:tensorflow:Restoring parameters from /var/folders/4b/wh1nthj1563b1gl58ydgwnfw0000gp/T/tmpp77omimo/model.ckpt-2000\n",
      "INFO:tensorflow:Assets added to graph.\n",
      "INFO:tensorflow:No assets to write.\n",
      "INFO:tensorflow:SavedModel written to: /Users/jiankaiwang/Desktop/TFE_SavedModel/temp-b'1572937612'/saved_model.pb\n"
     ]
    }
   ],
   "source": [
    "if os.path.exists(\"/Users/jiankaiwang/Desktop/TFE_SavedModel/\"):\n",
    "    nn.export_saved_model(\"/Users/jiankaiwang/Desktop/TFE_SavedModel/\", serving_input_receiver_fn)\n",
    "else:\n",
    "    nn.export_saved_model(\"/tmp/TFE_SavedModel/\", serving_input_receiver_fn)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Checking SavedModel Format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "saved_model_path = \"/Users/jiankaiwang/Desktop/TFE_SavedModel/1572937612\"\n",
    "assert os.path.exists(saved_model_path), \"Exported dir was not found.\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#!saved_model_cli show --dir \"/Users/jiankaiwang/Desktop/TFE_SavedModel/1572426418\" --tag_set serve --signature_def serving_default"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predicting via SavedModel format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def show_graph_info(graph):\n",
    "    \"\"\"List operations in the frozen model.\n",
    "    \"\"\"\n",
    "    print(graph.signature_def)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From <ipython-input-20-3a8d4a34e7c0>:8: load (from tensorflow.python.saved_model.loader_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This function will only be available through the v1 compatibility library as tf.compat.v1.saved_model.loader.load or tf.compat.v1.saved_model.load. There will be a new function for importing SavedModels in Tensorflow 2.0.\n",
      "INFO:tensorflow:Restoring parameters from /Users/jiankaiwang/Desktop/TFE_SavedModel/1572937612/variables/variables\n",
      "{'serving_default': inputs {\n",
      "  key: \"image_content\"\n",
      "  value {\n",
      "    name: \"Placeholder:0\"\n",
      "    dtype: DT_FLOAT\n",
      "    tensor_shape {\n",
      "      dim {\n",
      "        size: -1\n",
      "      }\n",
      "      dim {\n",
      "        size: 784\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "outputs {\n",
      "  key: \"class\"\n",
      "  value {\n",
      "    name: \"class:0\"\n",
      "    dtype: DT_FLOAT\n",
      "    tensor_shape {\n",
      "      dim {\n",
      "        size: -1\n",
      "      }\n",
      "      dim {\n",
      "        size: 10\n",
      "      }\n",
      "    }\n",
      "  }\n",
      "}\n",
      "method_name: \"tensorflow/serving/predict\"\n",
      "}\n",
      "[[0.         0.47255427 0.24939585 0.27586827 0.         0.\n",
      "  0.         0.29299074 0.07623385 0.1681642 ]\n",
      " [0.         0.03338228 0.07360079 0.25867754 0.2637101  0.\n",
      "  0.         0.45219374 0.31700423 0.        ]]\n"
     ]
    }
   ],
   "source": [
    "# you have to reset the graph if you are going to use the same scope\n",
    "tf.reset_default_graph()\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    imported = tf.saved_model.loader.load(\n",
    "        export_dir=saved_model_path, \n",
    "        sess=sess, \n",
    "        tags=['serve'])\n",
    "    \n",
    "    show_graph_info(imported)\n",
    "    \n",
    "    graph = tf.get_default_graph()\n",
    "    x = graph.get_tensor_by_name(\"Placeholder:0\")\n",
    "    y = graph.get_tensor_by_name(\"class:0\")\n",
    "    data = mnist.test.images[2:4].astype(\"float32\")\n",
    "    res = sess.run(y, feed_dict={x: data})\n",
    "    print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "colab": {
   "collapsed_sections": [],
   "name": "tf_estimator_example.ipynb",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}