grad_check.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "import tensorflow as tf\n",
    "import numpy as np\n",
    "import math\n",
    "import timeit\n",
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "from __future__ import division\n",
    "from scipy.misc import toimage\n",
    "import scipy.io as spio\n",
    "\n",
    "from lasso_codes.lasso import lasso_fista\n",
    "from lasso_codes.lasso import lasso_fista_col\n",
    "from lasso_codes.lasso import col_to_image\n",
    "from lasso_codes.lasso import compute_lasso_grad\n",
    "from lasso_codes.lasso import compute_lasso_grad_col2\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train data shape:  (490, 32, 32, 3)\n",
      "Train labels shape:  (490,)\n",
      "Validation data shape:  (10, 32, 32, 3)\n",
      "Validation labels shape:  (10,)\n",
      "Test data shape:  (100, 32, 32, 3)\n",
      "Test labels shape:  (100,)\n"
     ]
    }
   ],
   "source": [
    "from cs231n.data_utils import load_CIFAR10\n",
    "\n",
    "def get_CIFAR10_data(num_training=490, num_validation=10, num_test=100):\n",
    "    \"\"\"\n",
    "    Load the CIFAR-10 dataset from disk and perform preprocessing to prepare\n",
    "    it for the two-layer neural net classifier. These are the same steps as\n",
    "    we used for the SVM, but condensed to a single function.  \n",
    "    \"\"\"\n",
    "    # Load the raw CIFAR-10 data\n",
    "    cifar10_dir = 'cs231n/datasets/cifar-10-batches-py'\n",
    "    X_train, y_train, X_test, y_test = load_CIFAR10(cifar10_dir)\n",
    "\n",
    "    # Subsample the data\n",
    "    mask = range(num_training, num_training + num_validation)\n",
    "    X_val = X_train[mask]\n",
    "    y_val = y_train[mask]\n",
    "    mask = range(num_training)\n",
    "    X_train = X_train[mask]\n",
    "    y_train = y_train[mask]\n",
    "    mask = range(num_test)\n",
    "    X_test = X_test[mask]\n",
    "    y_test = y_test[mask]\n",
    "\n",
    "    # Normalize the data: subtract the mean image\n",
    "    mean_image = np.mean(X_train, axis=0)\n",
    "    X_train -= mean_image\n",
    "    X_val -= mean_image\n",
    "    X_test -= mean_image\n",
    "\n",
    "    return X_train, y_train, X_val, y_val, X_test, y_test\n",
    "\n",
    "\n",
    "# Invoke the above function to get our data.\n",
    "X_train, y_train, X_val, y_val, X_test, y_test = get_CIFAR10_data()\n",
    "print('Train data shape: ', X_train.shape)\n",
    "print('Train labels shape: ', y_train.shape)\n",
    "print('Validation data shape: ', X_val.shape)\n",
    "print('Validation labels shape: ', y_val.shape)\n",
    "print('Test data shape: ', X_test.shape)\n",
    "print('Test labels shape: ', y_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 0.09448719024658203 seconds ---Lasso\n",
      "--- 0.1074981689453125 seconds ---Grad\n",
      "47 3298\n",
      "emirical grad=-0.004807104070465065,  computed grad=-0.00448663185017942,  error=7.142824082453503\n",
      "--- 0.09133291244506836 seconds ---Lasso\n",
      "--- 0.037648916244506836 seconds ---Grad\n",
      "30 7342\n",
      "emirical grad=0.029985313249631863,  computed grad=0.02979911267157966,  error=0.6248527602292997\n",
      "--- 0.09419083595275879 seconds ---Lasso\n",
      "--- 0.051375389099121094 seconds ---Grad\n",
      "66 3348\n",
      "emirical grad=-0.020667609180359536,  computed grad=-0.025604504394698524,  error=19.281354320457712\n",
      "--- 0.09280729293823242 seconds ---Lasso\n",
      "--- 0.04837846755981445 seconds ---Grad\n",
      "9 2422\n",
      "emirical grad=-0.017495348185638245,  computed grad=-0.01725870584139773,  error=1.3711476770922992\n",
      "--- 0.0942695140838623 seconds ---Lasso\n",
      "--- 0.05732154846191406 seconds ---Grad\n",
      "5 9254\n",
      "emirical grad=-0.020488441345252864,  computed grad=-0.020520467709115205,  error=0.15607034067802728\n",
      "--- 0.09312200546264648 seconds ---Lasso\n",
      "--- 0.05838894844055176 seconds ---Grad\n",
      "70 2024\n",
      "emirical grad=-0.1287585035066058,  computed grad=-0.12826358118755699,  error=0.385863480862203\n",
      "--- 0.09792351722717285 seconds ---Lasso\n",
      "--- 0.04725980758666992 seconds ---Grad\n",
      "42 3930\n",
      "emirical grad=-0.03529626473978931,  computed grad=-0.03551241618508515,  error=0.6086644292781829\n",
      "--- 0.09344053268432617 seconds ---Lasso\n",
      "--- 0.047716379165649414 seconds ---Grad\n",
      "74 8657\n",
      "emirical grad=-0.05914143872672543,  computed grad=-0.05912605402132166,  error=0.02602017952733071\n",
      "--- 0.09426379203796387 seconds ---Lasso\n",
      "--- 0.045267343521118164 seconds ---Grad\n",
      "9 5496\n",
      "emirical grad=0.04165779490155434,  computed grad=0.04161408503869724,  error=0.1050362222705381\n",
      "--- 0.09381842613220215 seconds ---Lasso\n",
      "--- 0.0476222038269043 seconds ---Grad\n",
      "34 9420\n",
      "emirical grad=-0.01931487799208003,  computed grad=-0.019168236184787718,  error=0.7650250439249495\n",
      "--- 0.0903315544128418 seconds ---Lasso\n",
      "--- 0.05254673957824707 seconds ---Grad\n",
      "62 852\n",
      "emirical grad=0.007168934669304492,  computed grad=0.007041898781042837,  error=1.8040004863978165\n",
      "--- 0.09385871887207031 seconds ---Lasso\n",
      "--- 0.05414128303527832 seconds ---Grad\n",
      "15 1698\n",
      "emirical grad=-0.07524032269925766,  computed grad=-0.07508314295934178,  error=0.20934091690993917\n",
      "--- 0.09516239166259766 seconds ---Lasso\n",
      "--- 0.04630327224731445 seconds ---Grad\n",
      "32 1333\n",
      "emirical grad=0.13119700108566823,  computed grad=0.13107795974346137,  error=0.09081720713370878\n",
      "--- 0.09346485137939453 seconds ---Lasso\n",
      "--- 0.04400014877319336 seconds ---Grad\n",
      "57 7901\n",
      "emirical grad=-0.040110123999220704,  computed grad=-0.04004937503813957,  error=0.15168516618118133\n",
      "--- 0.09340262413024902 seconds ---Lasso\n",
      "--- 0.045775413513183594 seconds ---Grad\n",
      "39 2941\n",
      "emirical grad=0.05174057204803188,  computed grad=0.05202187740001992,  error=0.5407443292077985\n",
      "--- 0.09277486801147461 seconds ---Lasso\n",
      "--- 0.05951952934265137 seconds ---Grad\n",
      "58 2024\n",
      "emirical grad=-0.04264568788267213,  computed grad=-0.05437092302313721,  error=21.56526777276795\n",
      "--- 0.0914616584777832 seconds ---Lasso\n",
      "--- 0.047210693359375 seconds ---Grad\n",
      "48 3790\n",
      "emirical grad=-0.02511874898059061,  computed grad=-0.025027478950232843,  error=0.3646792812782129\n",
      "--- 0.09288954734802246 seconds ---Lasso\n",
      "--- 0.04545736312866211 seconds ---Grad\n",
      "16 6413\n",
      "emirical grad=0.07106936191714379,  computed grad=0.07670370012334482,  error=7.345588540240733\n",
      "--- 0.09512042999267578 seconds ---Lasso\n",
      "--- 0.053148508071899414 seconds ---Grad\n",
      "9 7910\n",
      "emirical grad=-0.028452816416720313,  computed grad=-0.028438995522660732,  error=0.048598390363574495\n",
      "--- 0.09409546852111816 seconds ---Lasso\n",
      "--- 0.05515027046203613 seconds ---Grad\n",
      "39 4281\n",
      "emirical grad=-0.06960200989840327,  computed grad=-0.06921205023765126,  error=0.563427408107442\n",
      "--- 0.09166836738586426 seconds ---Lasso\n",
      "--- 0.04439687728881836 seconds ---Grad\n",
      "47 5413\n",
      "emirical grad=0.0005275895761047769,  computed grad=0.00044521893603667096,  error=18.501153792192127\n",
      "--- 0.0910179615020752 seconds ---Lasso\n",
      "--- 0.04614138603210449 seconds ---Grad\n",
      "36 7888\n",
      "emirical grad=-0.009233572895154932,  computed grad=-0.009785766116919304,  error=5.642820553514416\n",
      "--- 0.09193897247314453 seconds ---Lasso\n",
      "--- 0.04331088066101074 seconds ---Grad\n",
      "2 2712\n",
      "emirical grad=0.03551450598974792,  computed grad=0.03471802272496658,  error=2.2941492696488375\n",
      "--- 0.0913081169128418 seconds ---Lasso\n",
      "--- 0.0418698787689209 seconds ---Grad\n",
      "2 1258\n",
      "emirical grad=-0.067151946288231,  computed grad=-0.06729694786990681,  error=0.21546531643027445\n",
      "--- 0.09428071975708008 seconds ---Lasso\n",
      "--- 0.048027753829956055 seconds ---Grad\n",
      "42 7521\n",
      "emirical grad=0.03478333029837859,  computed grad=0.03476033344023538,  error=0.06615833585930098\n",
      "--- 0.09363746643066406 seconds ---Lasso\n",
      "--- 0.050212860107421875 seconds ---Grad\n",
      "57 6273\n",
      "emirical grad=-0.012270971862815545,  computed grad=-0.01231208737278492,  error=0.33394426732430427\n",
      "--- 0.09161043167114258 seconds ---Lasso\n",
      "--- 0.05307769775390625 seconds ---Grad\n",
      "10 1391\n",
      "emirical grad=-0.12173014039897463,  computed grad=-0.12133208615551895,  error=0.3280700563785599\n",
      "--- 0.09102296829223633 seconds ---Lasso\n",
      "--- 0.041320085525512695 seconds ---Grad\n",
      "41 4384\n",
      "emirical grad=0.09714620905592229,  computed grad=0.09805415792087385,  error=0.9259667149293604\n",
      "--- 0.09242701530456543 seconds ---Lasso\n",
      "--- 0.04854917526245117 seconds ---Grad\n",
      "44 5783\n",
      "emirical grad=0.0770323389894512,  computed grad=0.0767032806496763,  error=0.4290016502394245\n",
      "--- 0.09561514854431152 seconds ---Lasso\n",
      "--- 0.04533863067626953 seconds ---Grad\n",
      "2 8210\n",
      "emirical grad=0.08457981674002892,  computed grad=0.08425776891036944,  error=0.38221737155426094\n",
      "--- 0.09150886535644531 seconds ---Lasso\n",
      "--- 0.037076473236083984 seconds ---Grad\n",
      "72 9128\n",
      "emirical grad=0.008778562649156465,  computed grad=0.00867578647849837,  error=1.1846323202260534\n",
      "--- 0.09428644180297852 seconds ---Lasso\n",
      "--- 0.0409696102142334 seconds ---Grad\n",
      "52 7544\n",
      "emirical grad=0.023363023696100053,  computed grad=0.023398782209191935,  error=0.15282211173295313\n",
      "--- 0.09189391136169434 seconds ---Lasso\n",
      "--- 0.047652244567871094 seconds ---Grad\n",
      "3 8676\n",
      "emirical grad=0.07665921724421297,  computed grad=0.07658108387687383,  error=0.10202698027199641\n",
      "--- 0.09383916854858398 seconds ---Lasso\n",
      "--- 0.04441571235656738 seconds ---Grad\n",
      "33 567\n",
      "emirical grad=0.00013172454314185652,  computed grad=0.00045288348216182946,  error=70.91425315114778\n",
      "--- 0.09110879898071289 seconds ---Lasso\n",
      "--- 0.049237728118896484 seconds ---Grad\n",
      "48 7690\n",
      "emirical grad=0.03860109001396774,  computed grad=0.03869677881648527,  error=0.2472784697954311\n",
      "--- 0.09365320205688477 seconds ---Lasso\n",
      "--- 0.04820108413696289 seconds ---Grad\n",
      "3 5060\n",
      "emirical grad=-0.04063454428552661,  computed grad=-0.04033239387680166,  error=0.7491506942233439\n",
      "--- 0.09161686897277832 seconds ---Lasso\n",
      "--- 0.0544431209564209 seconds ---Grad\n",
      "19 9177\n",
      "emirical grad=0.03613779844921125,  computed grad=0.03978129088045875,  error=9.158808954179133\n",
      "--- 0.0929250717163086 seconds ---Lasso\n",
      "--- 0.0531156063079834 seconds ---Grad\n",
      "9 2921\n",
      "emirical grad=0.03591558859311868,  computed grad=0.03582868175991419,  error=0.24256218463980517\n",
      "--- 0.09303498268127441 seconds ---Lasso\n",
      "--- 0.04246997833251953 seconds ---Grad\n",
      "59 2755\n",
      "emirical grad=0.0075292073253002845,  computed grad=0.007700025430220088,  error=2.2184096204331705\n",
      "--- 0.09354948997497559 seconds ---Lasso\n",
      "--- 0.04442119598388672 seconds ---Grad\n",
      "3 2941\n",
      "emirical grad=0.023953759546670672,  computed grad=0.023729929171605987,  error=0.9432408055077834\n",
      "--- 0.09418606758117676 seconds ---Lasso\n",
      "--- 0.04041099548339844 seconds ---Grad\n",
      "20 4991\n",
      "emirical grad=0.05915093238459691,  computed grad=0.059249227583199324,  error=0.1659012321542641\n",
      "--- 0.09213805198669434 seconds ---Lasso\n",
      "--- 0.04550671577453613 seconds ---Grad\n",
      "14 916\n",
      "emirical grad=0.04544702716006701,  computed grad=0.04534816325239041,  error=0.21801083127968024\n",
      "--- 0.09269976615905762 seconds ---Lasso\n",
      "--- 0.045760154724121094 seconds ---Grad\n",
      "35 2623\n",
      "emirical grad=0.0750036238893237,  computed grad=0.07492546221045404,  error=0.10431924817508875\n",
      "--- 0.09456396102905273 seconds ---Lasso\n",
      "--- 0.045734405517578125 seconds ---Grad\n",
      "65 9963\n",
      "emirical grad=-0.034830148504385726,  computed grad=-0.0347683903033374,  error=0.17762743834131012\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 0.09293913841247559 seconds ---Lasso\n",
      "--- 0.046123504638671875 seconds ---Grad\n",
      "16 6021\n",
      "emirical grad=0.022889108823248867,  computed grad=0.022717123039706934,  error=0.7570755471162371\n",
      "--- 0.0926048755645752 seconds ---Lasso\n",
      "--- 0.041129350662231445 seconds ---Grad\n",
      "68 1279\n",
      "emirical grad=0.03246828995082052,  computed grad=0.03220051717646185,  error=0.831579110643621\n",
      "--- 0.09366488456726074 seconds ---Lasso\n",
      "--- 0.051856040954589844 seconds ---Grad\n",
      "3 8144\n",
      "emirical grad=0.01499340518619037,  computed grad=0.015362461702351657,  error=2.4023266798757437\n",
      "--- 0.09422945976257324 seconds ---Lasso\n",
      "--- 0.04607105255126953 seconds ---Grad\n",
      "61 2852\n",
      "emirical grad=0.07402568813154566,  computed grad=0.07407190972347022,  error=0.062400972375520336\n",
      "--- 0.09380578994750977 seconds ---Lasso\n",
      "--- 0.04254603385925293 seconds ---Grad\n",
      "51 6650\n",
      "emirical grad=0.05621190884841393,  computed grad=0.05624744646041459,  error=0.0631808450640925\n",
      "--- 0.09334278106689453 seconds ---Lasso\n",
      "--- 0.04326891899108887 seconds ---Grad\n",
      "54 2391\n",
      "emirical grad=-0.009295402306764166,  computed grad=-0.00969261421254944,  error=4.098088473086936\n",
      "--- 0.09233617782592773 seconds ---Lasso\n",
      "--- 0.04378914833068848 seconds ---Grad\n",
      "22 7478\n",
      "emirical grad=-0.025743636041219778,  computed grad=-0.025739768541941932,  error=0.015025384830269495\n",
      "--- 0.09446096420288086 seconds ---Lasso\n",
      "--- 0.04769325256347656 seconds ---Grad\n",
      "9 1025\n",
      "emirical grad=0.04336004334009181,  computed grad=0.04359953521794321,  error=0.5492991534295832\n",
      "--- 0.0944979190826416 seconds ---Lasso\n",
      "--- 0.04166603088378906 seconds ---Grad\n",
      "59 2636\n",
      "emirical grad=0.08788991325597467,  computed grad=0.08790549771153365,  error=0.017728647200332404\n",
      "--- 0.0911252498626709 seconds ---Lasso\n",
      "--- 0.03911089897155762 seconds ---Grad\n",
      "54 5310\n",
      "emirical grad=-0.029172763054324067,  computed grad=-0.029602077305114386,  error=1.4502842025756912\n",
      "--- 0.09380435943603516 seconds ---Lasso\n",
      "--- 0.04762554168701172 seconds ---Grad\n",
      "61 7032\n",
      "emirical grad=-0.01821243653925129,  computed grad=-0.01843154891858063,  error=1.1887898314853707\n",
      "--- 0.09393835067749023 seconds ---Lasso\n",
      "--- 0.06062436103820801 seconds ---Grad\n",
      "73 6302\n",
      "emirical grad=-0.02815136262038833,  computed grad=-0.02817561683000558,  error=0.08608226667612406\n",
      "--- 0.0912466049194336 seconds ---Lasso\n",
      "--- 0.04470109939575195 seconds ---Grad\n",
      "38 6765\n",
      "emirical grad=0.03911987100350643,  computed grad=0.03907464420620194,  error=0.11574461706118644\n",
      "--- 0.09121179580688477 seconds ---Lasso\n",
      "--- 0.03841400146484375 seconds ---Grad\n",
      "38 1836\n",
      "emirical grad=0.019427881364182116,  computed grad=0.03248669486241888,  error=40.19742098585537\n",
      "--- 0.0937337875366211 seconds ---Lasso\n",
      "--- 0.048688650131225586 seconds ---Grad\n",
      "7 2250\n",
      "emirical grad=-0.03509754218601999,  computed grad=-0.03506631795869688,  error=0.08904335881481096\n",
      "--- 0.09117245674133301 seconds ---Lasso\n",
      "--- 0.04282569885253906 seconds ---Grad\n",
      "23 8621\n",
      "emirical grad=0.029221776538120726,  computed grad=0.029093227441795166,  error=0.4418523059455781\n",
      "--- 0.0950322151184082 seconds ---Lasso\n",
      "--- 0.04400372505187988 seconds ---Grad\n",
      "61 2216\n",
      "emirical grad=0.12200994670297648,  computed grad=0.12213361309630055,  error=0.10125500277026189\n",
      "--- 0.09244179725646973 seconds ---Lasso\n",
      "--- 0.04683089256286621 seconds ---Grad\n",
      "38 9968\n",
      "emirical grad=0.02174179985780139,  computed grad=0.021815956149141853,  error=0.3399176769218859\n",
      "--- 0.09486508369445801 seconds ---Lasso\n",
      "--- 0.0459284782409668 seconds ---Grad\n",
      "33 6191\n",
      "emirical grad=0.0589643344751405,  computed grad=0.05897409094066762,  error=0.01654364717030843\n",
      "--- 0.0927896499633789 seconds ---Lasso\n",
      "--- 0.04654860496520996 seconds ---Grad\n",
      "74 5174\n",
      "emirical grad=0.003661919816240613,  computed grad=0.0037376952385132045,  error=2.0273301443039515\n",
      "--- 0.09125709533691406 seconds ---Lasso\n",
      "--- 0.04530000686645508 seconds ---Grad\n",
      "52 6011\n",
      "emirical grad=-0.06316061919175403,  computed grad=-0.06341623156337575,  error=0.40307089418625264\n",
      "--- 0.09151077270507812 seconds ---Lasso\n",
      "--- 0.05198240280151367 seconds ---Grad\n",
      "31 7912\n",
      "emirical grad=-0.017141464882298188,  computed grad=-0.016934832022148287,  error=1.2201648051758385\n",
      "--- 0.09392523765563965 seconds ---Lasso\n",
      "--- 0.0515439510345459 seconds ---Grad\n",
      "31 995\n",
      "emirical grad=0.005151462184475721,  computed grad=0.008665650240370233,  error=40.55307978532458\n",
      "--- 0.09244656562805176 seconds ---Lasso\n",
      "--- 0.04485964775085449 seconds ---Grad\n",
      "11 9806\n",
      "emirical grad=-0.05953654454753122,  computed grad=-0.058676843341528735,  error=1.4651456299354493\n",
      "--- 0.09613847732543945 seconds ---Lasso\n",
      "--- 0.05528068542480469 seconds ---Grad\n",
      "22 6176\n",
      "emirical grad=-0.06428116954345278,  computed grad=-0.06410540178286407,  error=0.27418556892297996\n",
      "--- 0.09406471252441406 seconds ---Lasso\n",
      "--- 0.0552525520324707 seconds ---Grad\n",
      "28 8895\n",
      "emirical grad=-0.026008607855452573,  computed grad=-0.02592444235708035,  error=0.3246569288277712\n",
      "--- 0.09362030029296875 seconds ---Lasso\n",
      "--- 0.05244731903076172 seconds ---Grad\n",
      "69 2113\n",
      "emirical grad=0.0013271507216780542,  computed grad=0.0009765638935838406,  error=35.90003996641873\n",
      "--- 0.09354281425476074 seconds ---Lasso\n",
      "--- 0.04182100296020508 seconds ---Grad\n",
      "13 9267\n",
      "emirical grad=0.017720101817921316,  computed grad=0.017862564382097205,  error=0.7975482194408356\n",
      "--- 0.09348678588867188 seconds ---Lasso\n",
      "--- 0.048395395278930664 seconds ---Grad\n",
      "71 2440\n",
      "emirical grad=0.009638290338572705,  computed grad=0.009595359658770335,  error=0.44741084575325996\n",
      "--- 0.09342169761657715 seconds ---Lasso\n",
      "--- 0.05628800392150879 seconds ---Grad\n",
      "44 1443\n",
      "emirical grad=0.052511616652401345,  computed grad=0.05229677650556254,  error=0.41080953969687695\n",
      "--- 0.09168434143066406 seconds ---Lasso\n",
      "--- 0.0458984375 seconds ---Grad\n",
      "18 8072\n",
      "emirical grad=-0.008226487537182015,  computed grad=-0.008075124769519935,  error=1.8744325565520414\n",
      "--- 0.09528756141662598 seconds ---Lasso\n",
      "--- 0.052489519119262695 seconds ---Grad\n",
      "1 47\n",
      "emirical grad=-0.040317311283158394,  computed grad=-0.018156334617162057,  error=122.05644549560648\n",
      "--- 0.09336256980895996 seconds ---Lasso\n",
      "--- 0.045952558517456055 seconds ---Grad\n",
      "40 70\n",
      "emirical grad=0.038737103232526895,  computed grad=0.04924704036196018,  error=21.341256352029358\n",
      "--- 0.09094500541687012 seconds ---Lasso\n",
      "--- 0.06067013740539551 seconds ---Grad\n",
      "59 6776\n",
      "emirical grad=0.046232150323142776,  computed grad=0.04630307964096871,  error=0.15318488181762513\n",
      "--- 0.09362220764160156 seconds ---Lasso\n",
      "--- 0.0510098934173584 seconds ---Grad\n",
      "34 8753\n",
      "emirical grad=-0.012250639717517818,  computed grad=-0.013266838754456686,  error=7.659692378468829\n",
      "--- 0.09461355209350586 seconds ---Lasso\n",
      "--- 0.05435466766357422 seconds ---Grad\n",
      "28 1406\n",
      "emirical grad=-0.07789053699154908,  computed grad=-0.07767034480727764,  error=0.28349582433011555\n",
      "--- 0.09310197830200195 seconds ---Lasso\n",
      "--- 0.04651355743408203 seconds ---Grad\n",
      "61 4531\n",
      "emirical grad=-0.02119218220536575,  computed grad=-0.021177640053268065,  error=0.06866748165095249\n",
      "--- 0.09363698959350586 seconds ---Lasso\n",
      "--- 0.04449057579040527 seconds ---Grad\n",
      "56 5794\n",
      "emirical grad=-0.0697923154477231,  computed grad=-0.06978047373439604,  error=0.016969952614731554\n",
      "--- 0.09406185150146484 seconds ---Lasso\n",
      "--- 0.04973173141479492 seconds ---Grad\n",
      "9 9066\n",
      "emirical grad=-0.013089135936092476,  computed grad=-0.013810711575238641,  error=5.224753519868486\n",
      "--- 0.09345841407775879 seconds ---Lasso\n",
      "--- 0.04912590980529785 seconds ---Grad\n",
      "47 6659\n",
      "emirical grad=-0.0489160617494412,  computed grad=-0.048855715286146136,  error=0.12351976210278615\n",
      "--- 0.09217643737792969 seconds ---Lasso\n",
      "--- 0.04563474655151367 seconds ---Grad\n",
      "43 7663\n",
      "emirical grad=0.07437936227866902,  computed grad=0.07304370034475086,  error=1.828579230808569\n",
      "--- 0.09088397026062012 seconds ---Lasso\n",
      "--- 0.041761159896850586 seconds ---Grad\n",
      "63 2800\n",
      "emirical grad=-0.017374767705752206,  computed grad=-0.017527703107069286,  error=0.8725353252668766\n",
      "--- 0.09568166732788086 seconds ---Lasso\n",
      "--- 0.05033111572265625 seconds ---Grad\n",
      "35 8976\n",
      "emirical grad=0.0303299367621677,  computed grad=0.030442936537264344,  error=0.3711855292222732\n",
      "--- 0.09353828430175781 seconds ---Lasso\n",
      "--- 0.05356597900390625 seconds ---Grad\n",
      "57 2877\n",
      "emirical grad=-0.013415921614766296,  computed grad=-0.013293686595778918,  error=0.9194967709423185\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 0.09227895736694336 seconds ---Lasso\n",
      "--- 0.05404210090637207 seconds ---Grad\n",
      "24 4582\n",
      "emirical grad=-0.002290569806767756,  computed grad=-0.002466634104133814,  error=7.1378360118751765\n",
      "--- 0.09397149085998535 seconds ---Lasso\n",
      "--- 0.04558086395263672 seconds ---Grad\n",
      "26 4137\n",
      "emirical grad=0.036994401500081285,  computed grad=0.03707625347913912,  error=0.22076658609503363\n",
      "--- 0.0909278392791748 seconds ---Lasso\n",
      "--- 0.04556536674499512 seconds ---Grad\n",
      "48 9101\n",
      "emirical grad=-0.09281705824827156,  computed grad=-0.09313099652313678,  error=0.3370932198574983\n",
      "--- 0.0921635627746582 seconds ---Lasso\n",
      "--- 0.0531008243560791 seconds ---Grad\n",
      "7 5954\n",
      "emirical grad=0.07117462027749978,  computed grad=0.07142945852738448,  error=0.3567691189861222\n",
      "--- 0.09296941757202148 seconds ---Lasso\n",
      "--- 0.05133223533630371 seconds ---Grad\n",
      "23 9899\n",
      "emirical grad=0.08658700234309769,  computed grad=0.0867203308913277,  error=0.15374543300243904\n",
      "--- 0.09421992301940918 seconds ---Lasso\n",
      "--- 0.04630088806152344 seconds ---Grad\n",
      "2 2117\n",
      "emirical grad=-0.020327000654144292,  computed grad=-0.021604592821424632,  error=5.913521156544965\n",
      "--- 0.09486603736877441 seconds ---Lasso\n",
      "--- 0.04847145080566406 seconds ---Grad\n",
      "18 8958\n",
      "emirical grad=0.047746928980165165,  computed grad=0.048651851310287625,  error=1.8599956748842406\n",
      "--- 0.09067082405090332 seconds ---Lasso\n",
      "--- 0.05169272422790527 seconds ---Grad\n",
      "26 1925\n",
      "emirical grad=0.035388585504171816,  computed grad=0.03513993729054276,  error=0.7075943578760402\n",
      "--- 0.09614086151123047 seconds ---Lasso\n",
      "--- 0.04694342613220215 seconds ---Grad\n",
      "17 8751\n",
      "emirical grad=-0.037617726788310324,  computed grad=-0.04582399217764955,  error=17.908228854276466\n",
      "--- 0.09221792221069336 seconds ---Lasso\n",
      "--- 0.05280447006225586 seconds ---Grad\n",
      "4 2563\n",
      "emirical grad=-0.051648145864679376,  computed grad=-0.051643772012801285,  error=0.00846927268791653\n",
      "--- 0.09441089630126953 seconds ---Lasso\n",
      "--- 0.04724860191345215 seconds ---Grad\n",
      "25 9168\n",
      "emirical grad=0.031063319480021424,  computed grad=0.03100701977883822,  error=0.1815708235901825\n",
      "--- 0.09133672714233398 seconds ---Lasso\n",
      "--- 0.047341108322143555 seconds ---Grad\n",
      "68 3894\n",
      "emirical grad=-0.04206431614961032,  computed grad=-0.041942714501637904,  error=0.2899231712045536\n"
     ]
    }
   ],
   "source": [
    "#Gradient testing Col\n",
    "\n",
    "tf.reset_default_graph()\n",
    "X0= tf.placeholder(tf.float64, [75,None])\n",
    "D1=tf.placeholder(tf.float64, [75,50])\n",
    "\n",
    "A1= lasso_fista_col(Dict=D1,\n",
    "                 X=X0,\n",
    "                 lambda1=0.01,\n",
    "                 maxItr=500,\n",
    "                 NonNeg=True,\n",
    "                 debug=False)\n",
    "\n",
    "\n",
    "X0_tmp= tf.placeholder(tf.float64, [75, None])\n",
    "D1_tmp=tf.placeholder(tf.float64, [75, 50])\n",
    "A1_tmp=tf.placeholder(tf.float64, [50, None])\n",
    "grad_output1=tf.placeholder(tf.float64, [50, None])\n",
    "\n",
    "[lambda1_grad, X_grad, D_grad]= compute_lasso_grad_col2(X=X0_tmp,\n",
    "                                                       A=A1_tmp,\n",
    "                                                       D=D1_tmp,\n",
    "                                                       lambda1=0.01,\n",
    "                                                       grad_output=grad_output1,               \n",
    "                                                       NonNeg=True)\n",
    "\n",
    "\n",
    "\n",
    "eta=10**(-5)\n",
    "error9=[]\n",
    "with tf.Session() as sess:\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        tf.global_variables_initializer().run()\n",
    "        for itr in range(100):\n",
    "            start_time = time.time()\n",
    "            images=np.random.rand(75,10000).astype('float64')\n",
    "            noise=np.zeros([75,10000]).astype('float64')\n",
    "            i=np.random.randint(0,75)\n",
    "            j=np.random.randint(0,10000)\n",
    "            noise[i,j]=eta\n",
    "            D1_np=np.random.rand(75, 50).astype('float64')\n",
    "            \n",
    "            feed_dict = {X0: images+noise, D1: D1_np}\n",
    "            variables=A1\n",
    "            A1_np_eta_plus = sess.run(variables,feed_dict=feed_dict)\n",
    "            \n",
    "            \n",
    "            \n",
    "            feed_dict = {X0: images-noise, D1: D1_np}\n",
    "            variables=A1\n",
    "            A1_np_eta_min = sess.run(variables,feed_dict=feed_dict)\n",
    "            \n",
    "            \n",
    "            feed_dict = {X0: images, D1: D1_np}\n",
    "            variables=A1\n",
    "            start_time = time.time()\n",
    "            A1_np = sess.run(variables,feed_dict=feed_dict)\n",
    "            print(\"--- %s seconds ---Lasso\" % (time.time() - start_time))\n",
    "            \n",
    "            grad_output1_np=np.random.rand(50,10000).astype('float64')\n",
    "            grad_emp=np.sum(np.multiply((A1_np_eta_plus - A1_np_eta_min)/(2*eta), grad_output1_np ) )\n",
    "            \n",
    "            \n",
    "            feed_dict = {X0_tmp: images, D1_tmp: D1_np,\n",
    "                         A1_tmp: A1_np, grad_output1: grad_output1_np}\n",
    "            variables=[lambda1_grad, X_grad, D_grad]\n",
    "            start_time = time.time()\n",
    "            [lambda1_grad_np, X_grad_np, D_grad_np] = sess.run(variables,feed_dict=feed_dict)\n",
    "            print(\"--- %s seconds ---Grad\" % (time.time() - start_time))\n",
    "            grad_closed = X_grad_np[i, j]\n",
    "            \n",
    "            error9.append( np.absolute(1 - grad_emp/grad_closed)*100 )\n",
    "            print(i, j)\n",
    "            print(\"emirical grad=\" + str(grad_emp) +\",  computed grad=\" + str(grad_closed) +\",  error=\"+  str(error9[itr]))\n",
    "           \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 0.09533095359802246 seconds ---Lasso\n",
      "--- 1.069457769393921 seconds ---Grad\n",
      "1 14 14 0\n",
      "emirical grad=0.46397678572067386,  computed grad=0.41451956002035906,  error=11.931216393717525\n",
      "--- 0.09440970420837402 seconds ---Lasso\n",
      "--- 0.06403040885925293 seconds ---Grad\n",
      "9 16 6 2\n",
      "emirical grad=1.2330396686065277,  computed grad=1.28957184362811,  error=4.383794148493003\n",
      "--- 0.09453272819519043 seconds ---Lasso\n",
      "--- 0.06737470626831055 seconds ---Grad\n",
      "6 4 21 2\n",
      "emirical grad=0.1468103878387707,  computed grad=0.1555363591084454,  error=5.610245295500738\n",
      "--- 0.09436869621276855 seconds ---Lasso\n",
      "--- 0.06827211380004883 seconds ---Grad\n",
      "6 12 30 2\n",
      "emirical grad=0.8441468986255583,  computed grad=0.8725992431336332,  error=3.260642813062542\n",
      "--- 0.09276175498962402 seconds ---Lasso\n",
      "--- 0.06577730178833008 seconds ---Grad\n",
      "8 10 13 0\n",
      "emirical grad=0.6596880942391775,  computed grad=0.6663145686184123,  error=0.9944963972459253\n",
      "--- 0.09407973289489746 seconds ---Lasso\n",
      "--- 0.07292747497558594 seconds ---Grad\n",
      "0 29 23 2\n",
      "emirical grad=0.8425659369452032,  computed grad=0.8617827014408095,  error=2.2298851512658424\n",
      "--- 0.09356331825256348 seconds ---Lasso\n",
      "--- 0.09647345542907715 seconds ---Grad\n",
      "4 19 9 2\n",
      "emirical grad=-0.10547295055876815,  computed grad=-0.1476564089379796,  error=28.56866063763601\n",
      "--- 0.09199643135070801 seconds ---Lasso\n",
      "--- 0.058274030685424805 seconds ---Grad\n",
      "5 17 4 0\n",
      "emirical grad=0.12754848065533947,  computed grad=0.14544015521202236,  error=12.3017433050731\n",
      "--- 0.09528684616088867 seconds ---Lasso\n",
      "--- 0.072784423828125 seconds ---Grad\n",
      "0 19 2 1\n",
      "emirical grad=0.8537951960312004,  computed grad=0.8582444550479686,  error=0.5184139542759536\n",
      "--- 0.09436202049255371 seconds ---Lasso\n",
      "--- 0.06462740898132324 seconds ---Grad\n",
      "2 27 21 1\n",
      "emirical grad=-0.2503852267897987,  computed grad=-0.2515780941824156,  error=0.4741539188829358\n",
      "--- 0.09195065498352051 seconds ---Lasso\n",
      "--- 0.05892777442932129 seconds ---Grad\n",
      "3 5 13 2\n",
      "emirical grad=0.2832011869099282,  computed grad=0.2826529113526714,  error=0.19397484874044135\n",
      "--- 0.09191417694091797 seconds ---Lasso\n",
      "--- 0.05699563026428223 seconds ---Grad\n",
      "9 14 29 1\n",
      "emirical grad=0.38929866094801174,  computed grad=0.4313923578541802,  error=9.757636207453867\n",
      "--- 0.09212255477905273 seconds ---Lasso\n",
      "--- 0.055805206298828125 seconds ---Grad\n",
      "0 31 21 0\n",
      "emirical grad=0.40172267990834254,  computed grad=0.38950098797217303,  error=3.1377820117474586\n",
      "--- 0.09392023086547852 seconds ---Lasso\n",
      "--- 0.06646466255187988 seconds ---Grad\n",
      "2 13 23 2\n",
      "emirical grad=0.43787328157875527,  computed grad=0.43605953854151824,  error=0.4159393103298381\n",
      "--- 0.09442639350891113 seconds ---Lasso\n",
      "--- 0.07895350456237793 seconds ---Grad\n",
      "4 17 8 2\n",
      "emirical grad=0.45898426884928334,  computed grad=0.44983569888289754,  error=2.0337581008143646\n",
      "--- 0.09439253807067871 seconds ---Lasso\n",
      "--- 0.07097721099853516 seconds ---Grad\n",
      "9 23 29 0\n",
      "emirical grad=-0.3136067753834648,  computed grad=-0.30954761689838695,  error=1.311319572009606\n",
      "--- 0.09520149230957031 seconds ---Lasso\n",
      "--- 0.07230162620544434 seconds ---Grad\n",
      "8 15 22 1\n",
      "emirical grad=0.13803619642063453,  computed grad=0.12499975749944527,  error=10.42917136959014\n",
      "--- 0.09466004371643066 seconds ---Lasso\n",
      "--- 0.05663180351257324 seconds ---Grad\n",
      "4 15 14 1\n",
      "emirical grad=-0.30739564558741767,  computed grad=-0.3004665500917767,  error=2.3061121091597414\n",
      "--- 0.09226512908935547 seconds ---Lasso\n",
      "--- 0.06789088249206543 seconds ---Grad\n",
      "8 25 27 2\n",
      "emirical grad=0.5377766129344979,  computed grad=0.555175037181966,  error=3.1338628508553845\n",
      "--- 0.093292236328125 seconds ---Lasso\n",
      "--- 0.07343101501464844 seconds ---Grad\n",
      "2 27 29 2\n",
      "emirical grad=0.4219195334013502,  computed grad=0.42202070575481065,  error=0.023973315072189916\n",
      "--- 0.09242701530456543 seconds ---Lasso\n",
      "--- 0.06691789627075195 seconds ---Grad\n",
      "0 3 25 1\n",
      "emirical grad=0.6046718725510399,  computed grad=0.6171431003628439,  error=2.0208000064282694\n",
      "--- 0.0921483039855957 seconds ---Lasso\n",
      "--- 0.06034064292907715 seconds ---Grad\n",
      "4 1 17 1\n",
      "emirical grad=1.2202207879977154,  computed grad=1.2135649744695056,  error=0.5484513535106927\n",
      "--- 0.09526515007019043 seconds ---Lasso\n",
      "--- 0.07232785224914551 seconds ---Grad\n",
      "9 20 24 2\n",
      "emirical grad=-0.006961718405727679,  computed grad=0.011059488279081243,  error=162.9479251666246\n",
      "--- 0.09450936317443848 seconds ---Lasso\n",
      "--- 0.06912660598754883 seconds ---Grad\n",
      "5 2 25 2\n",
      "emirical grad=-0.014980717012205338,  computed grad=-0.00401588295422825,  error=273.0366941205897\n",
      "--- 0.0925145149230957 seconds ---Lasso\n",
      "--- 0.0787656307220459 seconds ---Grad\n",
      "5 30 21 2\n",
      "emirical grad=0.6826746306110207,  computed grad=0.6807493973362083,  error=0.28281086730974625\n",
      "--- 0.09255123138427734 seconds ---Lasso\n",
      "--- 0.07504391670227051 seconds ---Grad\n",
      "4 23 23 0\n",
      "emirical grad=0.774490903825142,  computed grad=0.7502909343570432,  error=3.225411418416879\n",
      "--- 0.09376144409179688 seconds ---Lasso\n",
      "--- 0.061661481857299805 seconds ---Grad\n",
      "4 25 17 1\n",
      "emirical grad=0.3680845614223792,  computed grad=0.35704848649185983,  error=3.0909177179136327\n",
      "--- 0.0946204662322998 seconds ---Lasso\n",
      "--- 0.07072257995605469 seconds ---Grad\n",
      "5 0 12 0\n",
      "emirical grad=0.4443684847931258,  computed grad=0.44742144119846344,  error=0.6823446809254397\n",
      "--- 0.09487724304199219 seconds ---Lasso\n",
      "--- 0.060534000396728516 seconds ---Grad\n",
      "6 20 22 2\n",
      "emirical grad=-0.15124031960149326,  computed grad=-0.07459007834765524,  error=102.76197981262416\n",
      "--- 0.09212088584899902 seconds ---Lasso\n",
      "--- 0.07106471061706543 seconds ---Grad\n",
      "9 10 15 2\n",
      "emirical grad=0.34151769727065434,  computed grad=0.3397592609186692,  error=0.5175536193569918\n",
      "--- 0.0940241813659668 seconds ---Lasso\n",
      "--- 0.055887699127197266 seconds ---Grad\n",
      "4 8 18 2\n",
      "emirical grad=-0.013610356298021417,  computed grad=-0.010885878079615736,  error=25.02763854674601\n",
      "--- 0.0937650203704834 seconds ---Lasso\n",
      "--- 0.06657290458679199 seconds ---Grad\n",
      "7 30 22 0\n",
      "emirical grad=0.4090156581210834,  computed grad=0.40899099208364653,  error=0.006030948826341032\n",
      "--- 0.09225344657897949 seconds ---Lasso\n",
      "--- 0.05570816993713379 seconds ---Grad\n",
      "3 26 24 2\n",
      "emirical grad=0.7080400732468081,  computed grad=0.7140610848559928,  error=0.8432067979729063\n",
      "--- 0.09510016441345215 seconds ---Lasso\n",
      "--- 0.0849604606628418 seconds ---Grad\n",
      "3 2 5 0\n",
      "emirical grad=0.3173023698748235,  computed grad=0.31610113687909375,  error=0.3800153987390642\n",
      "--- 0.09329485893249512 seconds ---Lasso\n",
      "--- 0.07310295104980469 seconds ---Grad\n",
      "8 18 16 1\n",
      "emirical grad=0.16079320999329455,  computed grad=0.15393415552264056,  error=4.455836618823139\n",
      "--- 0.09255576133728027 seconds ---Lasso\n",
      "--- 0.07323288917541504 seconds ---Grad\n",
      "1 27 13 1\n",
      "emirical grad=1.0168162014211277,  computed grad=1.0100627534664577,  error=0.6686166707457364\n",
      "--- 0.09211325645446777 seconds ---Lasso\n",
      "--- 0.0767524242401123 seconds ---Grad\n",
      "2 9 25 0\n",
      "emirical grad=-0.4102372143849543,  computed grad=-0.394717354609805,  error=3.9318919206102088\n",
      "--- 0.09288287162780762 seconds ---Lasso\n",
      "--- 0.05557727813720703 seconds ---Grad\n",
      "6 5 0 1\n",
      "emirical grad=0.09560532503247056,  computed grad=0.09308430568383338,  error=2.708318368082341\n",
      "--- 0.09487009048461914 seconds ---Lasso\n",
      "--- 0.07096576690673828 seconds ---Grad\n",
      "7 28 27 1\n",
      "emirical grad=0.20441663806023802,  computed grad=0.25555989492867126,  error=20.012238963671404\n",
      "--- 0.09487581253051758 seconds ---Lasso\n",
      "--- 0.0702972412109375 seconds ---Grad\n",
      "0 11 4 1\n",
      "emirical grad=-0.07740807288102702,  computed grad=-0.07556292164428266,  error=2.4418738669615303\n",
      "--- 0.09333157539367676 seconds ---Lasso\n",
      "--- 0.08086562156677246 seconds ---Grad\n",
      "8 28 18 1\n",
      "emirical grad=0.5055141544098822,  computed grad=0.5022824696574429,  error=0.643399869130068\n",
      "--- 0.09453773498535156 seconds ---Lasso\n",
      "--- 0.07209634780883789 seconds ---Grad\n",
      "5 2 20 0\n",
      "emirical grad=0.9730051407043256,  computed grad=0.9828273161814095,  error=0.9993795772024416\n",
      "--- 0.09216570854187012 seconds ---Lasso\n",
      "--- 0.07358241081237793 seconds ---Grad\n",
      "8 29 29 0\n",
      "emirical grad=0.42311393009819553,  computed grad=0.43248343961124214,  error=2.1664435339926125\n",
      "--- 0.09416532516479492 seconds ---Lasso\n",
      "--- 0.059221506118774414 seconds ---Grad\n",
      "7 1 1 0\n",
      "emirical grad=1.2390467962206124,  computed grad=1.2472768139547228,  error=0.6598389100183399\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 0.09226655960083008 seconds ---Lasso\n",
      "--- 0.0563967227935791 seconds ---Grad\n",
      "9 18 21 2\n",
      "emirical grad=-0.7010399275441912,  computed grad=-0.7261668828968069,  error=3.4602177466947825\n",
      "--- 0.0957791805267334 seconds ---Lasso\n",
      "--- 0.07125329971313477 seconds ---Grad\n",
      "6 24 17 2\n",
      "emirical grad=0.3606246527577038,  computed grad=0.34355364545186146,  error=4.968949545969603\n",
      "--- 0.09451723098754883 seconds ---Lasso\n",
      "--- 0.0627138614654541 seconds ---Grad\n",
      "8 3 14 0\n",
      "emirical grad=0.15405987004356414,  computed grad=0.17029644873556393,  error=9.534302572105846\n",
      "--- 0.09522104263305664 seconds ---Lasso\n",
      "--- 0.0755162239074707 seconds ---Grad\n",
      "2 27 0 0\n",
      "emirical grad=0.6407148280158836,  computed grad=0.6298088747526127,  error=1.7316290227816777\n",
      "--- 0.09233927726745605 seconds ---Lasso\n",
      "--- 0.07012677192687988 seconds ---Grad\n",
      "1 17 3 2\n",
      "emirical grad=0.22983552197555046,  computed grad=0.24942581864897084,  error=7.85415751245494\n",
      "--- 0.09283447265625 seconds ---Lasso\n",
      "--- 0.06923961639404297 seconds ---Grad\n",
      "9 6 30 2\n",
      "emirical grad=0.3639401399110155,  computed grad=0.3682963571972353,  error=1.1828021649117004\n",
      "--- 0.09544634819030762 seconds ---Lasso\n",
      "--- 0.06910085678100586 seconds ---Grad\n",
      "1 13 24 0\n",
      "emirical grad=0.1267326279222516,  computed grad=0.1265242947651954,  error=0.1646586194713251\n",
      "--- 0.09518027305603027 seconds ---Lasso\n",
      "--- 0.06559634208679199 seconds ---Grad\n",
      "4 25 22 1\n",
      "emirical grad=0.3419295352419503,  computed grad=0.35934574882770354,  error=4.846645227491986\n",
      "--- 0.09467244148254395 seconds ---Lasso\n",
      "--- 0.0657501220703125 seconds ---Grad\n",
      "2 9 22 2\n",
      "emirical grad=0.3289624657159413,  computed grad=0.2276980743487783,  error=44.473099588910195\n",
      "--- 0.09345316886901855 seconds ---Lasso\n",
      "--- 0.056917667388916016 seconds ---Grad\n",
      "9 0 6 0\n",
      "emirical grad=0.495581150266049,  computed grad=0.4976530478933347,  error=0.41633375622965874\n",
      "--- 0.09244418144226074 seconds ---Lasso\n",
      "--- 0.07526683807373047 seconds ---Grad\n",
      "3 10 18 1\n",
      "emirical grad=0.013172079045912916,  computed grad=0.026555252695996588,  error=50.397462992703105\n",
      "--- 0.09461498260498047 seconds ---Lasso\n",
      "--- 0.07277774810791016 seconds ---Grad\n",
      "1 7 18 1\n",
      "emirical grad=0.061743985958609804,  computed grad=0.06362995019921194,  error=2.9639568076001654\n",
      "--- 0.09405732154846191 seconds ---Lasso\n",
      "--- 0.07268452644348145 seconds ---Grad\n",
      "5 30 15 0\n",
      "emirical grad=0.6831010172217151,  computed grad=0.6770558737850394,  error=0.8928573948972174\n",
      "--- 0.09472537040710449 seconds ---Lasso\n",
      "--- 0.06816530227661133 seconds ---Grad\n",
      "2 6 21 1\n",
      "emirical grad=-0.06920742144328756,  computed grad=-0.06631073454097955,  error=4.368352910519913\n",
      "--- 0.09185099601745605 seconds ---Lasso\n",
      "--- 0.07314157485961914 seconds ---Grad\n",
      "7 28 0 0\n",
      "emirical grad=0.4905753330487425,  computed grad=0.4923607980780476,  error=0.3626334664081132\n",
      "--- 0.09508395195007324 seconds ---Lasso\n",
      "--- 0.07111144065856934 seconds ---Grad\n",
      "5 23 4 0\n",
      "emirical grad=0.036488821680317196,  computed grad=0.09484163302420902,  error=61.52657802612577\n",
      "--- 0.09364700317382812 seconds ---Lasso\n",
      "--- 0.06889772415161133 seconds ---Grad\n",
      "7 26 16 2\n",
      "emirical grad=0.9824571815807935,  computed grad=0.9752233204049443,  error=0.7417645809418838\n",
      "--- 0.09403491020202637 seconds ---Lasso\n",
      "--- 0.055902957916259766 seconds ---Grad\n",
      "1 22 16 1\n",
      "emirical grad=0.31865485884206873,  computed grad=0.22369700890066796,  error=42.449315888513524\n",
      "--- 0.09244561195373535 seconds ---Lasso\n",
      "--- 0.07478475570678711 seconds ---Grad\n",
      "2 19 12 1\n",
      "emirical grad=-0.38076155133737355,  computed grad=-0.3846926194986344,  error=1.0218725189956035\n",
      "--- 0.09521651268005371 seconds ---Lasso\n",
      "--- 0.0679008960723877 seconds ---Grad\n",
      "4 23 6 2\n",
      "emirical grad=0.22622327352103555,  computed grad=0.23220698601909423,  error=2.5768873713242435\n",
      "--- 0.09531617164611816 seconds ---Lasso\n",
      "--- 0.07435417175292969 seconds ---Grad\n",
      "0 5 22 0\n",
      "emirical grad=-0.1664035800219004,  computed grad=-0.135107177720434,  error=23.16413001108306\n",
      "--- 0.09391283988952637 seconds ---Lasso\n",
      "--- 0.0708165168762207 seconds ---Grad\n",
      "2 0 10 1\n",
      "emirical grad=0.007618315265521258,  computed grad=0.011776416323582558,  error=35.308713141659254\n",
      "--- 0.09405875205993652 seconds ---Lasso\n",
      "--- 0.065277099609375 seconds ---Grad\n",
      "1 29 26 1\n",
      "emirical grad=1.0103510214724953,  computed grad=1.0305917603778616,  error=1.9639919203259582\n",
      "--- 0.09398007392883301 seconds ---Lasso\n",
      "--- 0.06522893905639648 seconds ---Grad\n",
      "1 16 22 2\n",
      "emirical grad=0.7035805976065739,  computed grad=0.6601782284501526,  error=6.574341183336729\n",
      "--- 0.09232020378112793 seconds ---Lasso\n",
      "--- 0.06117749214172363 seconds ---Grad\n",
      "2 28 26 2\n",
      "emirical grad=0.6060829090129193,  computed grad=0.6254901960641658,  error=3.102732412652498\n",
      "--- 0.09506487846374512 seconds ---Lasso\n",
      "--- 0.07434248924255371 seconds ---Grad\n",
      "6 24 18 1\n",
      "emirical grad=0.17245022283205547,  computed grad=0.1898453274727014,  error=9.162777336801842\n",
      "--- 0.09380960464477539 seconds ---Lasso\n",
      "--- 0.07004880905151367 seconds ---Grad\n",
      "7 2 3 1\n",
      "emirical grad=0.8494365796659356,  computed grad=0.840239752126531,  error=1.094548016340413\n",
      "--- 0.09351038932800293 seconds ---Lasso\n",
      "--- 0.07197165489196777 seconds ---Grad\n",
      "2 28 0 1\n",
      "emirical grad=-0.23013109082583488,  computed grad=-0.22261274980581944,  error=3.3773182473032293\n",
      "--- 0.0953214168548584 seconds ---Lasso\n",
      "--- 0.08424592018127441 seconds ---Grad\n",
      "4 18 30 1\n",
      "emirical grad=-0.06705405216289109,  computed grad=-0.06315596907553013,  error=6.172153075031006\n",
      "--- 0.09430289268493652 seconds ---Lasso\n",
      "--- 0.08301329612731934 seconds ---Grad\n",
      "3 21 4 0\n",
      "emirical grad=-0.7801141717413679,  computed grad=-0.7825659840973174,  error=0.31330423322419865\n",
      "--- 0.0940713882446289 seconds ---Lasso\n",
      "--- 0.07070350646972656 seconds ---Grad\n",
      "2 16 27 2\n",
      "emirical grad=0.7893701955499187,  computed grad=0.7589538740345875,  error=4.007664043354642\n",
      "--- 0.09462118148803711 seconds ---Lasso\n",
      "--- 0.0689687728881836 seconds ---Grad\n",
      "2 28 7 2\n",
      "emirical grad=0.8123162836631909,  computed grad=0.8129770105038758,  error=0.08127251227872945\n",
      "--- 0.09370994567871094 seconds ---Lasso\n",
      "--- 0.06934571266174316 seconds ---Grad\n",
      "3 10 27 1\n",
      "emirical grad=1.053784813066526,  computed grad=1.046381080272313,  error=0.70755606478341\n",
      "--- 0.09379863739013672 seconds ---Lasso\n",
      "--- 0.07673406600952148 seconds ---Grad\n",
      "6 1 16 2\n",
      "emirical grad=-0.6265626518663184,  computed grad=-0.6304417862076069,  error=0.61530412897014\n",
      "--- 0.09501433372497559 seconds ---Lasso\n",
      "--- 0.0708777904510498 seconds ---Grad\n",
      "2 6 16 0\n",
      "emirical grad=0.05328279670835882,  computed grad=0.05283752516717926,  error=0.8427183895739043\n",
      "--- 0.09511208534240723 seconds ---Lasso\n",
      "--- 0.0689842700958252 seconds ---Grad\n",
      "1 28 12 0\n",
      "emirical grad=0.009503744550260665,  computed grad=-0.010666929875323819,  error=189.09540665722392\n",
      "--- 0.09210634231567383 seconds ---Lasso\n",
      "--- 0.0578916072845459 seconds ---Grad\n",
      "9 11 3 0\n",
      "emirical grad=0.6833521691696594,  computed grad=0.578031509664039,  error=18.2205740941068\n",
      "--- 0.09515237808227539 seconds ---Lasso\n",
      "--- 0.0784294605255127 seconds ---Grad\n",
      "3 23 8 2\n",
      "emirical grad=0.8810301473862081,  computed grad=0.7869218904498992,  error=11.959034064042262\n",
      "--- 0.09478211402893066 seconds ---Lasso\n",
      "--- 0.07502365112304688 seconds ---Grad\n",
      "0 12 24 1\n",
      "emirical grad=0.46522722763893853,  computed grad=0.6002746615364303,  error=22.49760693743622\n",
      "--- 0.09476971626281738 seconds ---Lasso\n",
      "--- 0.06757688522338867 seconds ---Grad\n",
      "6 15 24 1\n",
      "emirical grad=0.07190990698440847,  computed grad=0.09697224398455223,  error=25.844856188061627\n",
      "--- 0.09496688842773438 seconds ---Lasso\n",
      "--- 0.08542752265930176 seconds ---Grad\n",
      "2 8 16 1\n",
      "emirical grad=-0.099777287351584,  computed grad=-0.10410735260536762,  error=4.15923097208829\n",
      "--- 0.09429931640625 seconds ---Lasso\n",
      "--- 0.0768730640411377 seconds ---Grad\n",
      "9 5 30 1\n",
      "emirical grad=0.29433393462962176,  computed grad=0.2801354137634139,  error=5.0684491030466905\n",
      "--- 0.09214997291564941 seconds ---Lasso\n",
      "--- 0.06429576873779297 seconds ---Grad\n",
      "7 17 5 2\n",
      "emirical grad=0.3995734182001326,  computed grad=0.38657349317285844,  error=3.3628599106926282\n",
      "--- 0.09466695785522461 seconds ---Lasso\n",
      "--- 0.07204103469848633 seconds ---Grad\n",
      "4 21 2 0\n",
      "emirical grad=0.9779524442061185,  computed grad=0.9852918587922042,  error=0.7448975164660809\n",
      "--- 0.09138011932373047 seconds ---Lasso\n",
      "--- 0.07041478157043457 seconds ---Grad\n",
      "1 3 21 1\n",
      "emirical grad=-0.1117662023703167,  computed grad=-0.1076092505364799,  error=3.8630060270028244\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--- 0.09526538848876953 seconds ---Lasso\n",
      "--- 0.07237911224365234 seconds ---Grad\n",
      "0 19 29 1\n",
      "emirical grad=0.21272383187781313,  computed grad=0.2272673447169248,  error=6.399297205336074\n",
      "--- 0.09206962585449219 seconds ---Lasso\n",
      "--- 0.07242655754089355 seconds ---Grad\n",
      "3 10 14 2\n",
      "emirical grad=0.3538078152725942,  computed grad=0.3598118755933578,  error=1.668666524934026\n",
      "--- 0.09512162208557129 seconds ---Lasso\n",
      "--- 0.06103944778442383 seconds ---Grad\n",
      "9 17 11 1\n",
      "emirical grad=0.00901636257386687,  computed grad=0.021286716914550224,  error=57.64324479880752\n",
      "--- 0.09472775459289551 seconds ---Lasso\n",
      "--- 0.0965580940246582 seconds ---Grad\n",
      "4 12 22 1\n",
      "emirical grad=-0.11970461632574916,  computed grad=-0.13976043562173673,  error=14.350140801126921\n",
      "--- 0.09356045722961426 seconds ---Lasso\n",
      "--- 0.07283473014831543 seconds ---Grad\n",
      "0 26 20 2\n",
      "emirical grad=-0.19500552712343266,  computed grad=0.3067935674380341,  error=163.5624562639566\n",
      "--- 0.09428095817565918 seconds ---Lasso\n",
      "--- 0.08034276962280273 seconds ---Grad\n",
      "9 14 24 1\n",
      "emirical grad=0.5020289018598332,  computed grad=0.5016592322156442,  error=0.07368939320746559\n",
      "--- 0.09526562690734863 seconds ---Lasso\n",
      "--- 0.05846667289733887 seconds ---Grad\n",
      "9 4 7 0\n",
      "emirical grad=0.5061453202887021,  computed grad=0.47894288233281973,  error=5.679683101956878\n",
      "--- 0.09273624420166016 seconds ---Lasso\n",
      "--- 0.06512761116027832 seconds ---Grad\n",
      "1 27 9 2\n",
      "emirical grad=0.533508964469285,  computed grad=0.5345382694162232,  error=0.19255963620010697\n",
      "--- 0.09508061408996582 seconds ---Lasso\n",
      "--- 0.05992555618286133 seconds ---Grad\n",
      "0 8 7 2\n",
      "emirical grad=0.32055585624585314,  computed grad=0.3421509891930121,  error=6.311579866564953\n",
      "--- 0.09236383438110352 seconds ---Lasso\n",
      "--- 0.0706183910369873 seconds ---Grad\n",
      "0 1 16 0\n",
      "emirical grad=0.6074796239379749,  computed grad=0.65431991745172,  error=7.158622604087439\n",
      "--- 0.0950005054473877 seconds ---Lasso\n",
      "--- 0.056501150131225586 seconds ---Grad\n",
      "7 2 30 1\n",
      "emirical grad=0.26175131261194784,  computed grad=0.26400503674805986,  error=0.853667098125388\n",
      "(10, 32, 32, 50)\n",
      "(27, 50)\n",
      "(10, 32, 32, 3)\n"
     ]
    }
   ],
   "source": [
    "#Gradient testing\n",
    "\n",
    "#images=X_train[0:10,:,:,:]\n",
    "#from lasso_codes.lasso import  lasso_fista\n",
    "tf.reset_default_graph()\n",
    "X0= tf.placeholder(tf.float64, [None,32,32,3])\n",
    "D1=tf.placeholder(tf.float64, [27,50])\n",
    "A1= lasso_fista(Dict=D1,\n",
    "                 input=X0,\n",
    "                 lambda1=0.01,\n",
    "                 ksizes=[1,3,3,1],\n",
    "                 strides=[1,1,1,1],\n",
    "                 rates=[1,1,1,1],\n",
    "                 padding='SAME',\n",
    "                 maxItr=500,\n",
    "                 NonNeg=True,\n",
    "                 debug=False,\n",
    "                 name='lasso')\n",
    "\n",
    "X0_tmp= tf.placeholder(tf.float64, [None,32,32,3])\n",
    "D1_tmp=tf.placeholder(tf.float64, [27,50])\n",
    "A1_tmp=tf.placeholder(tf.float64, [None,32,32,50])\n",
    "grad_output1=tf.placeholder(tf.float64, [None,32,32,50])\n",
    "\n",
    "[lambda1_grad, X_grad, D_grad]= compute_lasso_grad(X=X0_tmp,\n",
    "                 A=A1_tmp,\n",
    "                 D=D1_tmp,\n",
    "                 lambda1=0.01,\n",
    "                 grad_output=grad_output1,               \n",
    "                 ksizes=[1,3,3,1],\n",
    "                 strides=[1,1,1,1],\n",
    "                 rates=[1,1,1,1],\n",
    "                 padding='SAME',\n",
    "                 params=[10,32,32,3],\n",
    "                 NonNeg=True)\n",
    "\n",
    "\n",
    "eta=10**(-5)\n",
    "error=[]\n",
    "with tf.Session() as sess:\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        tf.global_variables_initializer().run()\n",
    "        for itr in range(100):\n",
    "            \n",
    "            images=np.random.rand(10,32,32,3).astype('float64')\n",
    "            noise=np.zeros([10,32,32,3]).astype('float64')\n",
    "            i=np.random.randint(0,10)\n",
    "            j=np.random.randint(0,32)\n",
    "            c=np.random.randint(0,32)  #c\n",
    "            b=np.random.randint(0,3)\n",
    "            noise[i,j,c,b]=eta\n",
    "            D1_np=np.random.rand(27, 50).astype('float64')\n",
    "            \n",
    "            feed_dict = {X0: images+noise, D1: D1_np}\n",
    "            variables=A1\n",
    "            A1_np_eta_plus = sess.run(variables,feed_dict=feed_dict)\n",
    "            \n",
    "            \n",
    "            feed_dict = {X0: images-noise, D1: D1_np}\n",
    "            variables=A1\n",
    "            A1_np_eta_min = sess.run(variables,feed_dict=feed_dict)\n",
    "            \n",
    "            \n",
    "            feed_dict = {X0: images, D1: D1_np}\n",
    "            variables=A1\n",
    "            start_time = time.time()\n",
    "            A1_np = sess.run(variables,feed_dict=feed_dict)\n",
    "            print(\"--- %s seconds ---Lasso\" % (time.time() - start_time))\n",
    "            \n",
    "            grad_output1_np=np.random.rand(10,32,32, 50).astype('float64')\n",
    "            grad_emp=np.sum(np.multiply((A1_np_eta_plus - A1_np_eta_min)/(2*eta), grad_output1_np ) )\n",
    "            \n",
    "            \n",
    "            feed_dict = {X0_tmp: images, D1_tmp: D1_np,\n",
    "                         A1_tmp: A1_np, grad_output1: grad_output1_np}\n",
    "            variables=[lambda1_grad, X_grad, D_grad]\n",
    "            start_time = time.time()\n",
    "            [lambda1_grad_np, X_grad_np, D_grad_np] = sess.run(variables,feed_dict=feed_dict)\n",
    "            print(\"--- %s seconds ---Grad\" % (time.time() - start_time))\n",
    "            grad_closed = X_grad_np[i, j, c, b]\n",
    "            \n",
    "            error.append( np.absolute(1 - grad_emp/grad_closed)*100 )\n",
    "            print(i, j, c, b)\n",
    "            print(\"emirical grad=\" + str(grad_emp) +\",  computed grad=\" + str(grad_closed) +\",  error=\"+  str(error[itr]))\n",
    "            \n",
    "        \n",
    "print(A1_np_eta_plus.shape)\n",
    "print(D1_np.shape)\n",
    "print(images.shape)\n",
    "\n",
    "#from matplotlib import pyplot\n",
    "#bins = np.linspace(0, 1000, 20)\n",
    "#pyplot.hist(error, bins, alpha=0.5, label='error')\n",
    "#pyplot.hist(error9, bins, alpha=0.5, label='error9')\n",
    "\n",
    "#pyplot.legend(loc='upper right')\n",
    "#pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test Lasso Fista reconstruction\n",
    "#mat = spio.loadmat('test.mat', squeeze_me=True)\n",
    "\n",
    "X_np=np.random.rand(10,32,32,3)\n",
    "D_np=np.random.rand(27,50)\n",
    "\n",
    "print(X_np.shape)\n",
    "print(D_np.shape)\n",
    "\n",
    "\n",
    "tf.reset_default_graph()\n",
    "X0= tf.placeholder(tf.float64, [None,32,32,3])\n",
    "D1=tf.placeholder(tf.float64, [27,50])\n",
    "A1= lasso_fista(Dict=D1,\n",
    "                 input=X0,\n",
    "                 lambda1=0,\n",
    "                 ksizes=[1,3,3,1],\n",
    "                 strides=[1,3,3,1],\n",
    "                 rates=[1,1,1,1],\n",
    "                 padding='SAME',\n",
    "                 maxItr=500,\n",
    "                 NonNeg=False,\n",
    "                 debug=False,\n",
    "                 name='lasso')\n",
    "\n",
    "#A1=tf.transpose(A1,[0,2,1,3])\n",
    "A1_col=tf.reshape(A1,[-1,50])\n",
    "A1_col=tf.transpose(A1_col)\n",
    "Xp_rec=tf.matmul(D1,A1_col)\n",
    "\n",
    "X_rec= col_to_image(Xp_rec,ksizes=[1,3,3,1],strides=[1,3,3,1],rates=[1,1,1,1],padding='SAME', target_h=32,\n",
    "                         target_w=32, target_dep=3, bsize=10)\n",
    "nrm = col_to_image(tf.ones_like(Xp_rec),ksizes=[1,3,3,1],strides=[1,3,3,1],rates=[1,1,1,1],padding='SAME', target_h=32,\n",
    "                       target_w=32, target_dep=3, bsize=10)\n",
    "X_rec=tf.divide(X_rec,nrm)\n",
    "\n",
    "\n",
    "\n",
    "feed_dict = {X0: X_np, D1: D_np}\n",
    "variables=[X_rec,A1]\n",
    "with tf.Session() as sess:\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        tf.global_variables_initializer().run()\n",
    "        [X_rec_np, A1_np] = sess.run(variables,feed_dict=feed_dict)\n",
    "\n",
    "print(A1_np.shape)\n",
    "print(X_rec_np.shape)\n",
    "#print(X_rec_np - X_np)\n",
    "print( np.linalg.norm(np.reshape(X_rec_np - X_np,[1,-1]),2)  )\n",
    "print( np.linalg.norm(np.reshape( X_np,[1,-1]),2)  )\n",
    "print( np.linalg.norm(np.reshape(X_rec_np,[1,-1]),2)  )\n",
    "#Xp_rec_np=np.matmul(D_np, A1_np, out=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(A1_np[0,0,0,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test Lasso Fista\n",
    "mat = spio.loadmat('test.mat', squeeze_me=True)\n",
    "\n",
    "X_mat = mat['Y'] # array\n",
    "D_mat = mat['A'] # structure containing an array\n",
    "A_mat = mat['X_clean'] # array of structures\n",
    "lambda1=mat['lambda']\n",
    "\n",
    "X_mat=np.transpose(X_mat,(3,0,1,2))\n",
    "D_mat=np.reshape(D_mat,[27,50])\n",
    "A_mat=np.transpose(A_mat,(3,0,1,2))\n",
    "\n",
    "print(X_mat.shape)\n",
    "print(D_mat.shape)\n",
    "print(A_mat.shape)\n",
    "\n",
    "\n",
    "tf.reset_default_graph()\n",
    "X0= tf.placeholder(tf.float64, [None,32,32,3])\n",
    "D1=tf.placeholder(tf.float64, [27,50])\n",
    "A1= lasso_fista(Dict=D1,\n",
    "                 input=X0,\n",
    "                 lambda1=0.01,\n",
    "                 ksizes=[1,3,3,1],\n",
    "                 strides=[1,1,1,1],\n",
    "                 rates=[1,1,1,1],\n",
    "                 padding='SAME',\n",
    "                 maxItr=50,\n",
    "                 NonNeg=True,\n",
    "                 debug=False,\n",
    "                 name='lasso')\n",
    "\n",
    "feed_dict = {X0: X_mat, D1: D_mat}\n",
    "variables=A1\n",
    "with tf.Session() as sess:\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        tf.global_variables_initializer().run()\n",
    "        A1_np = sess.run(variables,feed_dict=feed_dict)\n",
    "\n",
    "print('------------------')\n",
    "print(A1_np.shape)\n",
    "print(A_mat.shape)\n",
    "\n",
    "E= A1_np -A_mat\n",
    "print(np.linalg.norm (np.reshape (E,[-1,1]),2))\n",
    "#print(np.linalg.norm (np.reshape (A1_np,[-1,1]),2))\n",
    "#print(np.linalg.norm (np.reshape (A_mat,[-1,1]),2))\n",
    "\n",
    "#print(A1_np[0,0,1,:])\n",
    "#print(A_mat[0,1,0,:])\n",
    "#print(np.linalg.norm (np.reshape (A1_np[0,:,:,1] - A_mat[0,:,:,1],[-1,1]),2) )\n",
    "#print(np.linalg.norm (np.reshape (A1_np[0,:,:,1],[-1,1]),2))\n",
    "#print(np.linalg.norm (np.reshape (A_mat[0,:,:,1],[-1,1]),2))\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Test Lasso Fista_col\n",
    "mat = spio.loadmat('test_lasso_col.mat', squeeze_me=True)\n",
    "X_mat = mat['X'] # array\n",
    "D_mat = mat['D'] # structure containing an array\n",
    "A_mat = mat['A'] # array of structures\n",
    "print(A_mat.shape)\n",
    "print(D_mat.shape)\n",
    "print(X_mat.shape)\n",
    "\n",
    "tf.reset_default_graph()\n",
    "X0= tf.placeholder(tf.float64, [27,None])\n",
    "D1=tf.placeholder(tf.float64, [27,50])\n",
    "\n",
    "A1= lasso_fista_col(Dict=D1,\n",
    "                 X=X0,\n",
    "                 lambda1=0.01,\n",
    "                 maxItr=50,\n",
    "                 NonNeg=True,\n",
    "                 debug=False)\n",
    "\n",
    "feed_dict = {X0: X_mat, D1: D_mat}\n",
    "variables=A1\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        tf.global_variables_initializer().run()\n",
    "        A1_np = sess.run(variables,feed_dict=feed_dict)\n",
    "#print(np.linalg.norm (np.reshape (A1_np[0,:,:,1] - A_mat[0,:,:,1],[-1,1]),2) )\n",
    "\n",
    "print(A1_np.shape)\n",
    "#print(A1_np.shape)\n",
    "\n",
    "print(np.linalg.norm(np.reshape(A1_np-A_mat,[-1,1]),2))\n",
    "\n",
    "E_np=X_mat-np.matmul(D_mat,A1_np)\n",
    "print(np.linalg.norm(np.reshape(E_np,[-1,1]),2))\n",
    "\n",
    "E_mat=X_mat-np.matmul(D_mat,A_mat)\n",
    "print(np.linalg.norm(np.reshape(E_mat,[-1,1]),2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test Recon"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Test Recon\n",
    "#from lasso_codes.lasso import  lasso_reconstruct\n",
    "tf.reset_default_graph()\n",
    "\n",
    "A1= tf.placeholder(tf.float32, [None,32,32,50])\n",
    "D1=tf.placeholder(tf.float32, [27,50])\n",
    "res= lasso_reconstruct(Dict=D1_np, A=A1_np,\n",
    "                       ksizes=[1,3,3,1],\n",
    "                       strides=[1,1,1,1],\n",
    "                       rates=[1,1,1,1],\n",
    "                       padding='SAME')\n",
    "\n",
    "\n",
    "variables=res\n",
    "\n",
    "with tf.Session() as sess:\n",
    "    with tf.device(\"/gpu:0\"):\n",
    "        tf.global_variables_initializer().run()\n",
    "        X_rec = sess.run(variables)\n",
    "print(X_rec.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "f, (ax1, ax2) = plt.subplots(1, 2, sharey=False, figsize=(12,4))\n",
    "\n",
    "ax1.imshow(toimage(X_rec[0,:,:,:]))\n",
    "ax2.imshow(toimage(images[0,:,:,:]))            \n",
    "            \n",
    "f.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "   @ops.RegisterGradient(\"LassoFistaGrad\")\n",
    "   def _lasso_fista_identity_grad(op, grad):\n",
    "      #print('helloooo')\n",
    "      #print(op.inputs[0])\n",
    "\n",
    "\n",
    "      print(len(op.inputs))\n",
    "      D = op.inputs[0]\n",
    "      X = op.inputs[1]\n",
    "      lambda1=op.inputs[2]\n",
    "      A = op.inputs[3]\n",
    "      ksizes=[1,3,3,1]\n",
    "      strides=[1,1,1,1]\n",
    "      rates=[1,1,1,1]\n",
    "      pad=1\n",
    "      dims=[10,32,32,3]\n",
    "\n",
    "      if (pad==0):\n",
    "         padding='VALID'\n",
    "      else:\n",
    "         padding='SAME'\n",
    "      #print('hello')\n",
    "      #print(tf.shape(grad))\n",
    "      #print('Bye')\n",
    "\n",
    "      [lambda_grad, X_grad, D_grad]= compute_lasso_grad(X=X, A=A, D=D, lambda1=lambda1, grad_output=grad,\n",
    "                                                            ksizes=ksizes, strides=strides, rates=rates, padding=padding, dims=dims)\n",
    "\n",
    "      #D_grad=D\n",
    "      #X_grad=X\n",
    "      lambda_grad=tf.expand_dims(lambda_grad,0)\n",
    "      #print(lambda_grad)\n",
    "      #lambda_grad=lambda1\n",
    "      #print(lambda_grad)\n",
    "      A_grad=grad\n",
    "\n",
    "      return[D_grad, X_grad, lambda_grad, A_grad,None,None,None,None, None]\n",
    "    \n",
    "    \n",
    "    \n",
    "     g = tf.get_default_graph()\n",
    "   #with g.as_default():\n",
    "   with g.gradient_override_map({'LassoFista': 'LassoFistaGrad'}):\n",
    "      A=_lasso_fista(Dict,input,lambda1,ksizes,strides,rates,padding,maxItr,NonNeg, debug,name)\n",
    "      dims=[10,32,32,3]\n",
    "      if (padding=='VALID'):\n",
    "         pad=0\n",
    "      else:\n",
    "         pad=1\n",
    "      _module=tf.load_op_library(tf.sysconfig.get_lib()+'/python/user_ops/lasso_fista.so')\n",
    "      A_n=_module.lasso_fista(Dict, input, lambda1, A, ksizes, strides, rates, pad, dims)\n",
    "\n",
    "      #[A_n,Dict_n,lambda1_n]=tf.tuple([A,Dict,lambda1],name=\"NoOp\")\n",
    "      #A_idn=_lasso_fista_identity(Dict,input,lambda1,A,ksizes,strides,rates,padding,maxItr=50,NonNeg=True, debug=False, name='lassoIdn')\n",
    "\n",
    "      A_out=A_n + tf.stop_gradient(A-A_n)\n",
    "      #grad = tf.gradients(A_out, Dict)\n",
    "      #print(A_out.name)\n",
    "   return(A_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pad_per_col_np.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(pad_per_col_np[0:100,:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(A1_np[:,0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}