bayesian_logistic_regression.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "cannot import name 'set_shapes_for_outputs'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-0f70b3cdec8f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexamples\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtutorials\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmnist\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0minput_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mCategorical\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNormal\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0medward\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0med\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mpandas\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/edward/edward/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcriticisms\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0minferences\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mmodels\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/edward/edward/criticisms/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0m__future__\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mprint_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcriticisms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mevaluate\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcriticisms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mppc\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcriticisms\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mppc_plots\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/edward/edward/criticisms/evaluate.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmodels\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mRandomVariable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcheck_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mget_session\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/edward/edward/util/__init__.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraphs\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprogbar\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrandom_variables\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensorflow\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/src/edward/edward/util/random_variables.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0medward\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraphs\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mrandom_variables\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mattr_value_pb2\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mframework\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mops\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mset_shapes_for_outputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0mtensorflow\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpython\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mutil\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: cannot import name 'set_shapes_for_outputs'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'\n",
    "\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow as tf\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "import time\n",
    "from edward.models import Categorical, Normal\n",
    "import edward as ed\n",
    "import pandas as pd\n",
    "\n",
    "learning_rate = 0.001\n",
    "batch_size = 128\n",
    "n_epochs = 25\n",
    "\n",
    "mnist = input_data.read_data_sets('/data/mnist', one_hot=True) \n",
    "#create placeholders for features and labels\n",
    "X = tf.placeholder(dtype=tf.float32, shape=(batch_size,784), name=\"X\")\n",
    "Y_plh = tf.placeholder(dtype=tf.int32, shape=(batch_size,10), name=\"Y\")\n",
    "\n",
    "#weights, biases\n",
    "W = Normal(loc=tf.zeros([784, 10]), scale=tf.ones(784, 10))\n",
    "b = Normal(loc=tf.zeros(10),scale=tf.ones(10))\n",
    "\n",
    "#create logits\n",
    "Y = Categorical(tf.matmul(X,W)+b)\n",
    "\n",
    "#create loss\n",
    "entropy = tf.nn.softmax_cross_entropy_with_logits(logits=logits, labels=Y, name=\"loss\")\n",
    "loss = tf.reduce_mean(entropy) # compute mean of all examples in batch\n",
    "\n",
    "#create prior over weights\n",
    "qw = Normal(loc=tf.Variable(tf.random_normal([784,10])),scale=tf.nn.softplus(tf.Variable(tf.random_normal([784,10]))))\n",
    "qb = Normal(loc=tf.Variable(tf.random_normal([10])),scale=tf.nn.softplus(tf.Variable(tf.random_normal([10]))))\n",
    "\n",
    "inference = ed.KLqp({W:qw,b:qb}, data={Y:Y_ph})\n",
    "\n",
    "inference.initialize(n_iter=5000, n_print=100, scale={y:float(55000/100)})\n",
    "\n",
    "with ed.get_session() as sess:\n",
    "    sess.run(tf.global_variables_initializer())\n",
    "    writer = tf.summary.FileWriter('./graphs/logistic_reg', sess.graph)\n",
    "    start_time = time.time()\n",
    "    n_batches = int(mnist.train.num_examples/batch_size)\n",
    "    for i in range(n_epochs):\n",
    "        total_loss = 0\n",
    "        for j in range(n_batches):\n",
    "            X_batch, Y_batch = mnist.train.next_batch(batch_size)\n",
    "            Y_batch = np.argmax(Y_batch,axis=1)\n",
    "            info_dict = inference.update(feed_dict={x: X_batch, y_ph: Y_batch})\n",
    "            inference.print_progress(info_dict)\n",
    "\n",
    "    print('Total time: {0} seconds'.format(time.time() - start_time))\n",
    "    \n",
    "    X_test = mnist.test.images\n",
    "    Y_test = np.argmax(mnist.test.labels,axis=1)\n",
    "\n",
    "    n_samples = 100\n",
    "    prob_lst = []\n",
    "    samples = []\n",
    "    w_samples = []\n",
    "    b_samples = []\n",
    "    for _ in range(n_samples):\n",
    "        w_samp = qw.sample()\n",
    "        b_samp = qb.sample()\n",
    "        w_samples.append(w_samp)\n",
    "        b_samples.append(b_samp)\n",
    "        # Also compue the probabiliy of each class for each (w,b) sample.\n",
    "        prob = tf.nn.softmax(tf.matmul( X_test,w_samp ) + b_samp)\n",
    "        prob_lst.append(prob.eval())\n",
    "        sample = tf.concat([tf.reshape(w_samp,[-1]),b_samp],0)\n",
    "        samples.append(sample.eval())\n",
    "\n",
    "    accy_test = []\n",
    "    for prob in prob_lst:\n",
    "        y_trn_prd = np.argmax(prob,axis=1).astype(np.float32)\n",
    "        acc = (y_trn_prd == Y_test).mean()*100\n",
    "        accy_test.append(acc)\n",
    "\n",
    "    # Load the first image from the test data and its label.\n",
    "    test_image = X_test[0:1]\n",
    "    test_label = Y_test[0]\n",
    "    print('truth = ',test_label)\n",
    "    pixels = test_image.reshape((28, 28))\n",
    "    plt.imshow(pixels,cmap='Blues')\n",
    "\n",
    "    # Now the check what the model perdicts for each (w,b) sample from the posterior. This may take a few seconds...\n",
    "    sing_img_probs = []\n",
    "    for w_samp,b_samp in zip(w_samples,b_samples):\n",
    "        prob = tf.nn.softmax(tf.matmul( X_test[0:1],w_samp ) + b_samp)\n",
    "        sing_img_probs.append(prob.eval())\n",
    "\n",
    "    plt.hist(np.argmax(sing_img_probs,axis=2),bins=range(10))\n",
    "    plt.xticks(np.arange(0,10))\n",
    "    plt.xlim(0,10)\n",
    "    plt.xlabel(\"Accuracy of the prediction of the test digit\")\n",
    "    plt.ylabel(\"Frequency\")\n",
    "        "
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}