2017-7-11 提高lstm的精确度测试.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:358: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 8 position: [495, 862, 3378, 3432, 3486, 4217, 4264, 4321]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:432: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/main_inp...)`\n",
      "  model_lstm2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_3 (LSTM)                    (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_4 (LSTM)                    (None, 50)            50200       main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_2 (Concatenate)      (None, 100)           0           lstm_3[0][0]                     \n",
      "                                                                   lstm_4[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_3 (Dense)                  (None, 100)           10100       concatenate_2[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_3 (Dropout)              (None, 100)           0           dense_3[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_4 (Dense)                  (None, 50)            5050        dropout_3[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)              (None, 50)            0           dense_4[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_4[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 115,601\n",
      "Trainable params: 115,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.2332 - val_loss: 0.1972\n",
      "Epoch 2/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.1529 - val_loss: 0.1358\n",
      "Epoch 3/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.1101 - val_loss: 0.1266\n",
      "Epoch 4/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0881 - val_loss: 0.1198\n",
      "Epoch 5/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0725 - val_loss: 0.1170\n",
      "Epoch 6/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0624 - val_loss: 0.1279\n",
      "Epoch 7/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0518 - val_loss: 0.1194\n",
      "Epoch 8/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0466 - val_loss: 0.1239\n",
      "Epoch 9/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0415 - val_loss: 0.1202\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0373 - val_loss: 0.1272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:103: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_5 (LSTM)                    (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_6 (LSTM)                    (None, 50)            50200       main_input_2[0][0]               \n",
      "====================================================================================================\n",
      "Total params: 100,400\n",
      "Trainable params: 100,400\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:138: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "lstm1 (InputLayer)               (None, 50)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lsmt2 (InputLayer)               (None, 50)            0                                            \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_3 (Concatenate)      (None, 100)           0           lstm1[0][0]                      \n",
      "                                                                   lsmt2[0][0]                      \n",
      "____________________________________________________________________________________________________\n",
      "dense_5 (Dense)                  (None, 100)           10100       concatenate_3[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_5 (Dropout)              (None, 100)           0           dense_5[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_6 (Dense)                  (None, 50)            5050        dropout_5[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_6 (Dropout)              (None, 50)            0           dense_6[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_6[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 15,201\n",
      "Trainable params: 15,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "TOP1 19\n",
      "TOP5 117\n",
      "TOP10 244\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:358: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [522, 1966, 2603, 3366, 3379, 3389, 3423]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:432: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/main_inp...)`\n",
      "  model_lstm2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_21 (LSTM)                   (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_22 (LSTM)                   (None, 50)            50200       main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_11 (Concatenate)     (None, 100)           0           lstm_21[0][0]                    \n",
      "                                                                   lstm_22[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_21 (Dense)                 (None, 100)           10100       concatenate_11[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_19 (Dropout)             (None, 100)           0           dense_21[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_22 (Dense)                 (None, 50)            5050        dropout_19[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_20 (Dropout)             (None, 50)            0           dense_22[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_20[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 115,601\n",
      "Trainable params: 115,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 12s - loss: 0.1823 - val_loss: 0.1405\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:103: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:138: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 19\n",
      "TOP5 88\n",
      "TOP10 144\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 12s - loss: 0.0885 - val_loss: 0.1194\n",
      "TOP1 19\n",
      "TOP5 116\n",
      "TOP10 202\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 12s - loss: 0.0626 - val_loss: 0.1125\n",
      "TOP1 16\n",
      "TOP5 141\n",
      "TOP10 245\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 12s - loss: 0.0457 - val_loss: 0.1134\n",
      "TOP1 23\n",
      "TOP5 146\n",
      "TOP10 255\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 12s - loss: 0.0343 - val_loss: 0.1211\n",
      "TOP1 21\n",
      "TOP5 147\n",
      "TOP10 240\n",
      "TOP1 21\n",
      "TOP5 147\n",
      "TOP10 240\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "classification report of TEST data:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.83      0.87      0.85      1000\n",
      "        1.0       0.87      0.82      0.84      1000\n",
      "\n",
      "avg / total       0.85      0.85      0.85      2000\n",
      "\n",
      "classification report of Training data:\n",
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.98      0.97      0.97      4000\n",
      "        1.0       0.97      0.98      0.97      4000\n",
      "\n",
      "avg / total       0.97      0.97      0.97      8000\n",
      "\n"
     ]
    }
   ],
   "source": [
    "y_pred = model_lstm2.predict([X1_test, X2_test])\n",
    "print \"classification report of TEST data:\"\n",
    "print(classification_report(y_test, y_pred>0.5))\n",
    "y_pred2 = model_lstm2.predict([X1_train, X2_train])\n",
    "print \"classification report of Training data:\"\n",
    "print(classification_report(y_train, y_pred2>0.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:358: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [343, 1006, 1013, 2104, 3369, 4261, 4295]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:433: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_35 (LSTM)                   (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_36 (LSTM)                   (None, 50)            50200       main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_18 (Concatenate)     (None, 100)           0           lstm_35[0][0]                    \n",
      "                                                                   lstm_36[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_35 (Dense)                 (None, 100)           10100       concatenate_18[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_33 (Dropout)             (None, 100)           0           dense_35[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_36 (Dense)                 (None, 50)            5050        dropout_33[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_34 (Dropout)             (None, 50)            0           dense_36[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_34[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 115,601\n",
      "Trainable params: 115,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 13s - loss: 0.7425 - val_loss: 0.5343\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:103: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:138: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 17\n",
      "TOP5 107\n",
      "TOP10 204\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 12s - loss: 0.3922 - val_loss: 0.4220\n",
      "TOP1 33\n",
      "TOP5 174\n",
      "TOP10 297\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 12s - loss: 0.2763 - val_loss: 0.4093\n",
      "TOP1 43\n",
      "TOP5 229\n",
      "TOP10 341\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 12s - loss: 0.2125 - val_loss: 0.4035\n",
      "TOP1 47\n",
      "TOP5 234\n",
      "TOP10 357\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 15s - loss: 0.1658 - val_loss: 0.4221\n",
      "TOP1 39\n",
      "TOP5 240\n",
      "TOP10 358\n",
      "TOP1 39\n",
      "TOP5 240\n",
      "TOP10 358\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:358: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 6 position: [988, 3404, 3407, 4268, 4312, 4314]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:433: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_49 (LSTM)                   (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_50 (LSTM)                   (None, 50)            50200       main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_25 (Concatenate)     (None, 100)           0           lstm_49[0][0]                    \n",
      "                                                                   lstm_50[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_49 (Dense)                 (None, 100)           10100       concatenate_25[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_47 (Dropout)             (None, 100)           0           dense_49[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_50 (Dense)                 (None, 50)            5050        dropout_47[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_48 (Dropout)             (None, 50)            0           dense_50[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_48[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 115,601\n",
      "Trainable params: 115,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 12s - loss: 0.9303 - val_loss: 0.7410\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:103: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:138: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 10\n",
      "TOP5 50\n",
      "TOP10 108\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 13s - loss: 0.6473 - val_loss: 0.5811\n",
      "TOP1 13\n",
      "TOP5 92\n",
      "TOP10 183\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 12s - loss: 0.5039 - val_loss: 0.5025\n",
      "TOP1 29\n",
      "TOP5 137\n",
      "TOP10 235\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 13s - loss: 0.3970 - val_loss: 0.4861\n",
      "TOP1 34\n",
      "TOP5 148\n",
      "TOP10 245\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 13s - loss: 0.3347 - val_loss: 0.4478\n",
      "TOP1 37\n",
      "TOP5 172\n",
      "TOP10 276\n",
      "TOP1 37\n",
      "TOP5 172\n",
      "TOP10 276\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 13s - loss: 0.2861 - val_loss: 0.4650\n",
      "TOP1 31\n",
      "TOP5 157\n",
      "TOP10 262\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 13s - loss: 0.2548 - val_loss: 0.4583\n",
      "TOP1 41\n",
      "TOP5 190\n",
      "TOP10 303\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 13s - loss: 0.2217 - val_loss: 0.4606\n",
      "TOP1 34\n",
      "TOP5 204\n",
      "TOP10 321\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 13s - loss: 0.2065 - val_loss: 0.4924\n",
      "TOP1 34\n",
      "TOP5 185\n",
      "TOP10 311\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 13s - loss: 0.1779 - val_loss: 0.4702\n",
      "TOP1 40\n",
      "TOP5 182\n",
      "TOP10 293\n"
     ]
    }
   ],
   "source": [
    "    for i in range(epoch, epoch+5):\n",
    "        hist = model_lstm2.fit([X1_train, X2_train], [y_train],\n",
    "                           validation_data=([X1_test, X2_test], y_test),\n",
    "                           epochs=i+1,\n",
    "                           batch_size=32,\n",
    "                           initial_epoch=i)\n",
    "\n",
    "        sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
    "                                                                 X2_test_1,\n",
    "                                                                 model_lstm2)\n",
    "\n",
    "        # Evaluation\n",
    "        print \"TOP1\", (pd.Series(rank_results_test) <= 1).sum()\n",
    "        print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n",
    "        print \"TOP10\", (pd.Series(rank_results_test) <= 10).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 13s - loss: 0.1695 - val_loss: 0.4462\n",
      "TOP1 42\n",
      "TOP5 201\n",
      "TOP10 332\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 13s - loss: 0.1534 - val_loss: 0.4830\n",
      "TOP1 40\n",
      "TOP5 197\n",
      "TOP10 298\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 13s - loss: 0.1380 - val_loss: 0.4914\n",
      "TOP1 27\n",
      "TOP5 187\n",
      "TOP10 316\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 13s - loss: 0.1437 - val_loss: 0.4817\n",
      "TOP1 29\n",
      "TOP5 189\n",
      "TOP10 307\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 13s - loss: 0.1274 - val_loss: 0.4909\n",
      "TOP1 28\n",
      "TOP5 208\n",
      "TOP10 321\n"
     ]
    }
   ],
   "source": [
    "    for i in range(epoch+5, epoch+10):\n",
    "        hist = model_lstm2.fit([X1_train, X2_train], [y_train],\n",
    "                           validation_data=([X1_test, X2_test], y_test),\n",
    "                           epochs=i+1,\n",
    "                           batch_size=32,\n",
    "                           initial_epoch=i)\n",
    "\n",
    "        sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
    "                                                                 X2_test_1,\n",
    "                                                                 model_lstm2)\n",
    "\n",
    "        # Evaluation\n",
    "        print \"TOP1\", (pd.Series(rank_results_test) <= 1).sum()\n",
    "        print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n",
    "        print \"TOP10\", (pd.Series(rank_results_test) <= 10).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 13s - loss: 0.1210 - val_loss: 0.5103\n",
      "TOP1 31\n",
      "TOP5 195\n",
      "TOP10 339\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 17/17\n",
      "8000/8000 [==============================] - 13s - loss: 0.1155 - val_loss: 0.5195\n",
      "TOP1 27\n",
      "TOP5 198\n",
      "TOP10 328\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 18/18\n",
      "8000/8000 [==============================] - 13s - loss: 0.1176 - val_loss: 0.5111\n",
      "TOP1 44\n",
      "TOP5 196\n",
      "TOP10 330\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 19/19\n",
      "8000/8000 [==============================] - 13s - loss: 0.1211 - val_loss: 0.5529\n",
      "TOP1 28\n",
      "TOP5 194\n",
      "TOP10 314\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 20/20\n",
      "8000/8000 [==============================] - 13s - loss: 0.1193 - val_loss: 0.5128\n",
      "TOP1 23\n",
      "TOP5 183\n",
      "TOP10 334\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 21/21\n",
      "8000/8000 [==============================] - 13s - loss: 0.1072 - val_loss: 0.4711\n",
      "TOP1 33\n",
      "TOP5 177\n",
      "TOP10 314\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 22/22\n",
      "8000/8000 [==============================] - 13s - loss: 0.1111 - val_loss: 0.5244\n",
      "TOP1 26\n",
      "TOP5 197\n",
      "TOP10 337\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 23/23\n",
      "1696/8000 [=====>........................] - ETA: 9s - loss: 0.0931"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-12-254d74b46823>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m                        \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m                        \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m                        initial_epoch=i)\n\u001b[0m\u001b[1;32m      7\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "    for i in range(epoch+10, epoch+20):\n",
    "        hist = model_lstm2.fit([X1_train, X2_train], [y_train],\n",
    "                           validation_data=([X1_test, X2_test], y_test),\n",
    "                           epochs=i+1,\n",
    "                           batch_size=32,\n",
    "                           initial_epoch=i)\n",
    "\n",
    "        sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
    "                                                                 X2_test_1,\n",
    "                                                                 model_lstm2)\n",
    "\n",
    "        # Evaluation\n",
    "        print \"TOP1\", (pd.Series(rank_results_test) <= 1).sum()\n",
    "        print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n",
    "        print \"TOP10\", (pd.Series(rank_results_test) <= 10).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:358: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [690, 997, 1041, 3779, 4069]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:433: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  model_lstm2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                    (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                    (None, 50)            50200       main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 100)           0           lstm_1[0][0]                     \n",
      "                                                                   lstm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 100)           10100       concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 100)           0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 50)            5050        dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 50)            0           dense_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 115,601\n",
      "Trainable params: 115,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 12s - loss: 0.7873 - val_loss: 0.6442\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:103: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:138: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 11\n",
      "TOP5 46\n",
      "TOP10 107\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 12s - loss: 0.4842 - val_loss: 0.5130\n",
      "TOP1 32\n",
      "TOP5 127\n",
      "TOP10 203\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 13s - loss: 0.3570 - val_loss: 0.4321\n",
      "TOP1 32\n",
      "TOP5 166\n",
      "TOP10 274\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 12s - loss: 0.2692 - val_loss: 0.4087\n",
      "TOP1 28\n",
      "TOP5 197\n",
      "TOP10 309\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 13s - loss: 0.2111 - val_loss: 0.3723\n",
      "TOP1 48\n",
      "TOP5 217\n",
      "TOP10 359\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 13s - loss: 0.1688 - val_loss: 0.3896\n",
      "TOP1 44\n",
      "TOP5 215\n",
      "TOP10 348\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 13s - loss: 0.1392 - val_loss: 0.4119\n",
      "TOP1 54\n",
      "TOP5 228\n",
      "TOP10 358\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 12s - loss: 0.1097 - val_loss: 0.4085\n",
      "TOP1 57\n",
      "TOP5 229\n",
      "TOP10 360\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 12s - loss: 0.0906 - val_loss: 0.3805\n",
      "TOP1 44\n",
      "TOP5 240\n",
      "TOP10 379\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0787 - val_loss: 0.3843\n",
      "TOP1 39\n",
      "TOP5 236\n",
      "TOP10 362\n",
      "TOP1 39\n",
      "TOP5 236\n",
      "TOP10 362\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:358: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [1014, 1477, 1595, 4185, 4226]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:433: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  model_lstm2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_25 (LSTM)                   (None, 50)            50200       main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_26 (LSTM)                   (None, 50)            50200       main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_13 (Concatenate)     (None, 100)           0           lstm_25[0][0]                    \n",
      "                                                                   lstm_26[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_25 (Dense)                 (None, 100)           10100       concatenate_13[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_25 (Dropout)             (None, 100)           0           dense_25[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_26 (Dense)                 (None, 50)            5050        dropout_25[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_26 (Dropout)             (None, 50)            0           dense_26[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_26[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 115,601\n",
      "Trainable params: 115,601\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 12s - loss: 0.6878 - val_loss: 0.5433\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:103: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:138: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 16\n",
      "TOP5 108\n",
      "TOP10 198\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 12s - loss: 0.3438 - val_loss: 0.4452\n",
      "TOP1 32\n",
      "TOP5 177\n",
      "TOP10 304\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 12s - loss: 0.2278 - val_loss: 0.4404\n",
      "TOP1 43\n",
      "TOP5 221\n",
      "TOP10 319\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 12s - loss: 0.1591 - val_loss: 0.3960\n",
      "TOP1 50\n",
      "TOP5 246\n",
      "TOP10 375\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 13s - loss: 0.1127 - val_loss: 0.4091\n",
      "TOP1 53\n",
      "TOP5 241\n",
      "TOP10 384\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 12s - loss: 0.0865 - val_loss: 0.4561\n",
      "TOP1 47\n",
      "TOP5 240\n",
      "TOP10 374\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 12s - loss: 0.0608 - val_loss: 0.4689\n",
      "TOP1 43\n",
      "TOP5 233\n",
      "TOP10 359\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 12s - loss: 0.0460 - val_loss: 0.4295\n",
      "TOP1 51\n",
      "TOP5 246\n",
      "TOP10 379\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 12s - loss: 0.0366 - val_loss: 0.4773\n",
      "TOP1 50\n",
      "TOP5 253\n",
      "TOP10 384\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0292 - val_loss: 0.4710\n",
      "TOP1 54\n",
      "TOP5 259\n",
      "TOP10 384\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 12s - loss: 0.0228 - val_loss: 0.4586\n",
      "TOP1 53\n",
      "TOP5 241\n",
      "TOP10 367\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 12s - loss: 0.0172 - val_loss: 0.4844\n",
      "TOP1 52\n",
      "TOP5 249\n",
      "TOP10 373\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 12s - loss: 0.0154 - val_loss: 0.4503\n",
      "TOP1 55\n",
      "TOP5 239\n",
      "TOP10 366\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 12s - loss: 0.0140 - val_loss: 0.4502\n",
      "TOP1 49\n",
      "TOP5 233\n",
      "TOP10 379\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 12s - loss: 0.0122 - val_loss: 0.4507\n",
      "TOP1 54\n",
      "TOP5 250\n",
      "TOP10 374\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 12s - loss: 0.0095 - val_loss: 0.4420\n",
      "TOP1 56\n",
      "TOP5 241\n",
      "TOP10 386\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 17/17\n",
      "8000/8000 [==============================] - 12s - loss: 0.0086 - val_loss: 0.4419\n",
      "TOP1 48\n",
      "TOP5 242\n",
      "TOP10 389\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 18/18\n",
      "8000/8000 [==============================] - 12s - loss: 0.0097 - val_loss: 0.4541\n",
      "TOP1 44\n",
      "TOP5 251\n",
      "TOP10 366\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 19/19\n",
      "8000/8000 [==============================] - 12s - loss: 0.0093 - val_loss: 0.4982\n",
      "TOP1 47\n",
      "TOP5 237\n",
      "TOP10 376\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 20/20\n",
      "8000/8000 [==============================] - 12s - loss: 0.0305 - val_loss: 0.4966\n",
      "TOP1 43\n",
      "TOP5 205\n",
      "TOP10 341\n",
      "TOP1 43\n",
      "TOP5 205\n",
      "TOP10 341\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4998\n",
      "True        2\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/ipykernel/__main__.py:7: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/ipykernel/__main__.py:8: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/ipykernel/__main__.py:12: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/ipykernel/__main__.py:13: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 2 position: [3114, 3773]\n"
     ]
    }
   ],
   "source": [
    "    # --- Apply the word2vec model to the data sets --- #\n",
    "    \n",
    "    pairs_correct, pairs_wrong, df_pairs = prepare_train(dir_en_jp, 51000, 56000)\n",
    "\n",
    "    df_pairs_sample = df_pairs.iloc[:5000]\n",
    "\n",
    "    df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
    "    df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
    "\n",
    "\n",
    "    # ---- Padding the vector ---- #\n",
    "    df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
    "    df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n",
    "\n",
    "\n",
    "    # --- Prepare the training data --- #\n",
    "\n",
    "    # Generate training data (similarity = 1)\n",
    "    features_en_1 = np.stack(df_pairs_sample[\"padding_en\"].values)\n",
    "    features_jp_1 = np.stack(df_pairs_sample[\"padding_jp\"].values)\n",
    "\n",
    "    # Generate training data (similarity = 0)\n",
    "    features_en_0 = np.array(features_en_1)\n",
    "    np.random.shuffle((features_en_0))\n",
    "\n",
    "    # check the duplicated amount\n",
    "    c = np.all(features_en_1 == features_en_0, axis=(1,2))\n",
    "    print \"C value =\", c.sum(), \"position:\", np.where(c== True)[0].tolist()\n",
    "\n",
    "    # Prepare the final training and test data\n",
    "    X_1 = np.concatenate((features_en_1, features_en_0), axis = 0)\n",
    "    X_2 = np.concatenate((features_jp_1, features_jp_1), axis = 0)\n",
    "    y = np.concatenate((np.ones(len(features_en_1)), np.zeros(len(features_en_0))-1), axis = 0)\n",
    "\n",
    "    # --- Split into test data and training data --- #\n",
    "\n",
    "    X1_train1, X1_test_1, X1_train2, X1_train3_wrong, X1_test_0 = np.split(X_1, [2000, 3000, 5000, 9000])\n",
    "    X2_train1, X2_test_1, X2_train2, X2_train3_wrong, X2_test_0 = np.split(X_2, [2000, 3000, 5000, 9000])\n",
    "    y_train1, y_test, y_train2, y_train3_wrong, Y_o = np.split(y, [2000, 3000, 9000, 9000])\n",
    "\n",
    "    X1_train = np.concatenate((X1_train1, X1_train2, X1_train3_wrong), axis = 0)\n",
    "    X2_train = np.concatenate((X2_train1, X2_train2, X2_train3_wrong), axis = 0)\n",
    "    y_train = np.concatenate((y_train1, y_train2, y_train3_wrong), axis = 0)\n",
    "    # X_train_correct = np.concatenate((X_train1, X_train2), axis = 0)\n",
    "    # y_train_correct = np.concatenate((y_train1, y_train2), axis = 0)\n",
    "\n",
    "    # --- Generate balanced test data --- #\n",
    "    X1_test = np.concatenate((X1_test_1, X1_test_0), axis=0)\n",
    "    X2_test = np.concatenate((X2_test_1, X2_test_0), axis=0)\n",
    "    y_test = np.concatenate((np.ones(len(X1_test_1)), np.zeros(len(X1_test_0))-1), axis = 0)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# pairs_correct, pairs_wrong, df_pairs = prepare_train(dir_en_jp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 26\n",
      "TOP5 164\n",
      "TOP10 289\n"
     ]
    }
   ],
   "source": [
    "sim_results_test, rank_results_test = find_ranking_quick(X1_test_1, X2_test_1, model_lstm2)\n",
    "print \"TOP1\", (pd.Series(rank_results_test) <= 1).sum()\n",
    "print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n",
    "print \"TOP10\", (pd.Series(rank_results_test) <= 10).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 32\n",
      "TOP5 159\n",
      "TOP10 299\n"
     ]
    }
   ],
   "source": [
    "sim_results_test, rank_results_test = find_ranking_quick(X1_test_1, X2_test_1, model_lstm2)\n",
    "print \"TOP1\", (pd.Series(rank_results_test) <= 1).sum()\n",
    "print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n",
    "print \"TOP10\", (pd.Series(rank_results_test) <= 10).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[81.0,\n",
       " 49.0,\n",
       " 35.0,\n",
       " 23.5,\n",
       " 24.5,\n",
       " 84.0,\n",
       " 22.0,\n",
       " 63.0,\n",
       " 155.0,\n",
       " 79.0,\n",
       " 10.5,\n",
       " 9.5,\n",
       " 1.0,\n",
       " 23.0,\n",
       " 1.0,\n",
       " 15.0,\n",
       " 20.0,\n",
       " 9.0,\n",
       " 6.0,\n",
       " 8.0,\n",
       " 291.5,\n",
       " 291.5,\n",
       " 90.0,\n",
       " 145.0,\n",
       " 79.0,\n",
       " 70.0,\n",
       " 19.0,\n",
       " 2.0,\n",
       " 116.0,\n",
       " 39.0,\n",
       " 6.0,\n",
       " 73.0,\n",
       " 11.5,\n",
       " 5.5,\n",
       " 17.0,\n",
       " 39.0,\n",
       " 10.0,\n",
       " 1.5,\n",
       " 1.5,\n",
       " 93.5,\n",
       " 122.0,\n",
       " 93.5,\n",
       " 43.5,\n",
       " 48.5,\n",
       " 48.5,\n",
       " 54.5,\n",
       " 2.0,\n",
       " 15.0,\n",
       " 27.0,\n",
       " 15.0,\n",
       " 57.0,\n",
       " 42.0,\n",
       " 85.0,\n",
       " 39.0,\n",
       " 61.0,\n",
       " 32.0,\n",
       " 11.0,\n",
       " 98.0,\n",
       " 50.0,\n",
       " 197.5,\n",
       " 219.5,\n",
       " 34.5,\n",
       " 34.5,\n",
       " 15.0,\n",
       " 36.0,\n",
       " 7.0,\n",
       " 53.0,\n",
       " 31.0,\n",
       " 9.5,\n",
       " 26.0,\n",
       " 9.5,\n",
       " 10.0,\n",
       " 114.0,\n",
       " 35.0,\n",
       " 30.0,\n",
       " 20.0,\n",
       " 73.0,\n",
       " 73.0,\n",
       " 73.0,\n",
       " 144.0,\n",
       " 5.0,\n",
       " 8.0,\n",
       " 21.0,\n",
       " 145.0,\n",
       " 145.0,\n",
       " 124.0,\n",
       " 124.0,\n",
       " 474.0,\n",
       " 12.0,\n",
       " 2.0,\n",
       " 11.0,\n",
       " 26.0,\n",
       " 3.0,\n",
       " 8.5,\n",
       " 8.5,\n",
       " 8.5,\n",
       " 1.0,\n",
       " 28.0,\n",
       " 1.5,\n",
       " 51.5,\n",
       " 14.0,\n",
       " 7.5,\n",
       " 5.5,\n",
       " 47.0,\n",
       " 170.0,\n",
       " 21.0,\n",
       " 16.0,\n",
       " 113.0,\n",
       " 40.0,\n",
       " 22.5,\n",
       " 6.0,\n",
       " 120.0,\n",
       " 12.0,\n",
       " 3.0,\n",
       " 11.0,\n",
       " 2.0,\n",
       " 7.0,\n",
       " 8.0,\n",
       " 9.0,\n",
       " 2.0,\n",
       " 2.0,\n",
       " 372.0,\n",
       " 12.0,\n",
       " 40.0,\n",
       " 34.0,\n",
       " 153.0,\n",
       " 58.0,\n",
       " 9.0,\n",
       " 3.5,\n",
       " 3.5,\n",
       " 93.5,\n",
       " 87.5,\n",
       " 180.0,\n",
       " 113.0,\n",
       " 20.0,\n",
       " 18.0,\n",
       " 5.0,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 5.0,\n",
       " 1.0,\n",
       " 2.0,\n",
       " 79.5,\n",
       " 67.5,\n",
       " 114.5,\n",
       " 114.5,\n",
       " 114.5,\n",
       " 114.5,\n",
       " 114.5,\n",
       " 158.5,\n",
       " 2.0,\n",
       " 8.0,\n",
       " 62.5,\n",
       " 62.5,\n",
       " 62.5,\n",
       " 62.5,\n",
       " 62.5,\n",
       " 102.5,\n",
       " 3.0,\n",
       " 122.0,\n",
       " 122.0,\n",
       " 312.0,\n",
       " 312.0,\n",
       " 39.5,\n",
       " 39.5,\n",
       " 39.5,\n",
       " 39.5,\n",
       " 39.5,\n",
       " 57.5,\n",
       " 3.0,\n",
       " 3.0,\n",
       " 6.0,\n",
       " 3.0,\n",
       " 104.0,\n",
       " 66.0,\n",
       " 14.0,\n",
       " 26.0,\n",
       " 10.0,\n",
       " 7.0,\n",
       " 4.0,\n",
       " 30.0,\n",
       " 111.0,\n",
       " 14.0,\n",
       " 4.0,\n",
       " 4.0,\n",
       " 14.0,\n",
       " 231.0,\n",
       " 43.0,\n",
       " 2.0,\n",
       " 50.0,\n",
       " 17.0,\n",
       " 17.0,\n",
       " 66.0,\n",
       " 66.0,\n",
       " 11.0,\n",
       " 6.5,\n",
       " 28.0,\n",
       " 1.0,\n",
       " 2.0,\n",
       " 210.0,\n",
       " 80.0,\n",
       " 45.0,\n",
       " 8.0,\n",
       " 81.0,\n",
       " 12.0,\n",
       " 8.0,\n",
       " 2.0,\n",
       " 128.0,\n",
       " 64.0,\n",
       " 12.0,\n",
       " 12.0,\n",
       " 7.0,\n",
       " 137.0,\n",
       " 6.0,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 45.0,\n",
       " 210.0,\n",
       " 49.0,\n",
       " 194.0,\n",
       " 1.0,\n",
       " 7.0,\n",
       " 25.0,\n",
       " 109.0,\n",
       " 24.0,\n",
       " 38.0,\n",
       " 159.0,\n",
       " 14.0,\n",
       " 31.0,\n",
       " 33.0,\n",
       " 14.0,\n",
       " 35.0,\n",
       " 8.0,\n",
       " 39.0,\n",
       " 153.5,\n",
       " 157.5,\n",
       " 1.0,\n",
       " 63.0,\n",
       " 80.0,\n",
       " 9.0,\n",
       " 16.0,\n",
       " 64.0,\n",
       " 310.0,\n",
       " 734.0,\n",
       " 304.0,\n",
       " 23.0,\n",
       " 167.0,\n",
       " 37.0,\n",
       " 13.5,\n",
       " 13.5,\n",
       " 19.0,\n",
       " 16.0,\n",
       " 33.0,\n",
       " 52.5,\n",
       " 52.5,\n",
       " 452.0,\n",
       " 78.0,\n",
       " 57.5,\n",
       " 18.5,\n",
       " 28.0,\n",
       " 23.0,\n",
       " 106.0,\n",
       " 12.0,\n",
       " 47.0,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 12.0,\n",
       " 119.0,\n",
       " 24.0,\n",
       " 66.0,\n",
       " 113.0,\n",
       " 19.0,\n",
       " 19.0,\n",
       " 19.0,\n",
       " 32.0,\n",
       " 7.0,\n",
       " 22.0,\n",
       " 4.5,\n",
       " 24.5,\n",
       " 4.5,\n",
       " 19.0,\n",
       " 166.5,\n",
       " 87.5,\n",
       " 5.0,\n",
       " 6.0,\n",
       " 11.0,\n",
       " 70.0,\n",
       " 23.0,\n",
       " 69.0,\n",
       " 58.0,\n",
       " 131.0,\n",
       " 19.0,\n",
       " 14.5,\n",
       " 20.5,\n",
       " 224.0,\n",
       " 116.5,\n",
       " 120.5,\n",
       " 2.0,\n",
       " 8.0,\n",
       " 14.5,\n",
       " 20.5,\n",
       " 66.0,\n",
       " 25.0,\n",
       " 14.0,\n",
       " 4.5,\n",
       " 24.5,\n",
       " 4.5,\n",
       " 113.0,\n",
       " 9.0,\n",
       " 42.0,\n",
       " 8.0,\n",
       " 3.0,\n",
       " 15.0,\n",
       " 8.0,\n",
       " 12.0,\n",
       " 6.0,\n",
       " 26.0,\n",
       " 183.0,\n",
       " 100.0,\n",
       " 230.0,\n",
       " 176.0,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 10.0,\n",
       " 37.0,\n",
       " 63.0,\n",
       " 10.0,\n",
       " 4.5,\n",
       " 4.5,\n",
       " 7.0,\n",
       " 51.0,\n",
       " 724.0,\n",
       " 27.0,\n",
       " 38.0,\n",
       " 49.0,\n",
       " 30.0,\n",
       " 17.0,\n",
       " 6.0,\n",
       " 9.0,\n",
       " 64.0,\n",
       " 41.0,\n",
       " 25.0,\n",
       " 46.0,\n",
       " 11.5,\n",
       " 4.5,\n",
       " 9.0,\n",
       " 3.0,\n",
       " 155.0,\n",
       " 29.0,\n",
       " 25.0,\n",
       " 2.0,\n",
       " 96.0,\n",
       " 129.0,\n",
       " 143.5,\n",
       " 141.0,\n",
       " 62.0,\n",
       " 2.0,\n",
       " 143.5,\n",
       " 8.5,\n",
       " 8.5,\n",
       " 8.5,\n",
       " 8.5,\n",
       " 6.5,\n",
       " 110.0,\n",
       " 122.0,\n",
       " 17.0,\n",
       " 10.0,\n",
       " 5.0,\n",
       " 61.0,\n",
       " 74.0,\n",
       " 33.5,\n",
       " 28.5,\n",
       " 28.5,\n",
       " 2.0,\n",
       " 153.0,\n",
       " 21.0,\n",
       " 33.0,\n",
       " 72.0,\n",
       " 114.0,\n",
       " 114.0,\n",
       " 64.0,\n",
       " 100.0,\n",
       " 100.0,\n",
       " 6.0,\n",
       " 16.5,\n",
       " 26.5,\n",
       " 54.0,\n",
       " 134.0,\n",
       " 723.0,\n",
       " 196.0,\n",
       " 196.0,\n",
       " 274.0,\n",
       " 75.0,\n",
       " 75.0,\n",
       " 88.0,\n",
       " 88.0,\n",
       " 88.0,\n",
       " 193.0,\n",
       " 88.0,\n",
       " 276.0,\n",
       " 276.0,\n",
       " 6.0,\n",
       " 6.0,\n",
       " 13.0,\n",
       " 142.5,\n",
       " 145.5,\n",
       " 13.0,\n",
       " 1.0,\n",
       " 19.0,\n",
       " 18.0,\n",
       " 30.0,\n",
       " 4.0,\n",
       " 4.0,\n",
       " 4.0,\n",
       " 4.0,\n",
       " 76.0,\n",
       " 23.0,\n",
       " 26.5,\n",
       " 39.5,\n",
       " 42.0,\n",
       " 1.0,\n",
       " 2.0,\n",
       " 3.0,\n",
       " 4.0,\n",
       " 2.0,\n",
       " 2.0,\n",
       " 1.0,\n",
       " 1.0,\n",
       " 40.5,\n",
       " 56.5,\n",
       " 18.0,\n",
       " 73.0,\n",
       " 19.0,\n",
       " 63.0,\n",
       " 43.0,\n",
       " 51.0,\n",
       " 47.0,\n",
       " 45.0,\n",
       " 52.0,\n",
       " 30.0,\n",
       " 19.0,\n",
       " 28.5,\n",
       " 28.5,\n",
       " 41.0,\n",
       " 1.0,\n",
       " 19.0,\n",
       " 34.0,\n",
       " 63.0,\n",
       " 1.0,\n",
       " 272.0,\n",
       " 59.0,\n",
       " 38.0,\n",
       " 17.0,\n",
       " 12.5,\n",
       " 12.5,\n",
       " 103.0,\n",
       " 3.0,\n",
       " 14.0,\n",
       " 45.0,\n",
       " 15.0,\n",
       " 14.0,\n",
       " 14.0,\n",
       " 14.0,\n",
       " 52.5,\n",
       " 52.5,\n",
       " 148.5,\n",
       " 148.5,\n",
       " 2.0,\n",
       " 1.0,\n",
       " 7.0,\n",
       " 2.5,\n",
       " 2.5,\n",
       " 61.5,\n",
       " 61.5,\n",
       " 68.0,\n",
       " 62.0,\n",
       " 3.0,\n",
       " 52.0,\n",
       " 59.0,\n",
       " 2.0,\n",
       " 9.5,\n",
       " 1.5,\n",
       " 1.5,\n",
       " 159.5,\n",
       " 102.5,\n",
       " 146.5,\n",
       " 22.0,\n",
       " 1.0,\n",
       " 31.0,\n",
       " 37.0,\n",
       " 226.0,\n",
       " 159.5,\n",
       " 185.5,\n",
       " 205.5,\n",
       " 6.0,\n",
       " 16.0,\n",
       " 104.0,\n",
       " 6.0,\n",
       " 128.0,\n",
       " 34.0,\n",
       " 6.0,\n",
       " 54.0,\n",
       " 7.5,\n",
       " 2.0,\n",
       " 164.0,\n",
       " 35.0,\n",
       " 52.0,\n",
       " 149.0,\n",
       " 86.0,\n",
       " 3.0,\n",
       " 4.0,\n",
       " 22.0,\n",
       " 13.0,\n",
       " 151.5,\n",
       " 151.5,\n",
       " 50.0,\n",
       " 8.5,\n",
       " 2.5,\n",
       " 23.0,\n",
       " 171.0,\n",
       " 14.0,\n",
       " 345.0,\n",
       " 18.0,\n",
       " 14.0,\n",
       " 8.5,\n",
       " 2.5,\n",
       " 55.0,\n",
       " 40.5,\n",
       " 40.5,\n",
       " 49.5,\n",
       " 66.5,\n",
       " 55.5,\n",
       " 52.5,\n",
       " 58.5,\n",
       " 93.5,\n",
       " 93.5,\n",
       " 93.5,\n",
       " 42.0,\n",
       " 3.0,\n",
       " 105.0,\n",
       " 13.0,\n",
       " 208.0,\n",
       " 3.0,\n",
       " 140.0,\n",
       " 324.0,\n",
       " 9.0,\n",
       " 4.0,\n",
       " 10.0,\n",
       " 12.0,\n",
       " 9.0,\n",
       " 27.0,\n",
       " 5.0,\n",
       " 218.0,\n",
       " 309.0,\n",
       " 3.0,\n",
       " 15.0,\n",
       " 2.0,\n",
       " 7.0,\n",
       " 6.0,\n",
       " 125.0,\n",
       " 125.0,\n",
       " 6.0,\n",
       " 3.0,\n",
       " 1.0,\n",
       " 53.0,\n",
       " 85.0,\n",
       " 3.0,\n",
       " 55.0,\n",
       " 8.0,\n",
       " 29.0,\n",
       " 29.0,\n",
       " 32.0,\n",
       " 29.0,\n",
       " 26.5,\n",
       " 19.5,\n",
       " 40.0,\n",
       " 14.0,\n",
       " 13.0,\n",
       " 3.5,\n",
       " 3.5,\n",
       " 2.5,\n",
       " 2.5,\n",
       " 4.0,\n",
       " 6.0,\n",
       " 16.0,\n",
       " 13.0,\n",
       " 15.0,\n",
       " 77.5,\n",
       " 81.5,\n",
       " 10.0,\n",
       " 22.0,\n",
       " 31.0,\n",
       " 6.0,\n",
       " 101.0,\n",
       " 6.0,\n",
       " 1.0,\n",
       " 44.0,\n",
       " 1.0,\n",
       " 60.0,\n",
       " 26.5,\n",
       " 39.5,\n",
       " 5.0,\n",
       " 8.0,\n",
       " 12.0,\n",
       " 13.0,\n",
       " 11.0,\n",
       " 7.0,\n",
       " 2.0,\n",
       " 2.0,\n",
       " 38.0,\n",
       " 39.0,\n",
       " 69.5,\n",
       " 67.5,\n",
       " 1.0,\n",
       " 10.0,\n",
       " 58.0,\n",
       " 7.0,\n",
       " 9.0,\n",
       " 1.0,\n",
       " 188.5,\n",
       " 188.5,\n",
       " 4.5,\n",
       " 1.5,\n",
       " 1.5,\n",
       " 14.0,\n",
       " 1.0,\n",
       " 45.5,\n",
       " 45.5,\n",
       " 3.0,\n",
       " 146.5,\n",
       " 146.5,\n",
       " 7.0,\n",
       " 76.0,\n",
       " 4.0,\n",
       " 76.5,\n",
       " 76.5,\n",
       " 77.5,\n",
       " 13.0,\n",
       " 376.5,\n",
       " 11.0,\n",
       " 1.5,\n",
       " 4.0,\n",
       " 55.0,\n",
       " 376.5,\n",
       " 25.0,\n",
       " 28.0,\n",
       " 70.0,\n",
       " 1.5,\n",
       " 12.0,\n",
       " 19.0,\n",
       " 76.5,\n",
       " 76.5,\n",
       " 77.5,\n",
       " 24.0,\n",
       " 3.0,\n",
       " 258.0,\n",
       " 6.0,\n",
       " 91.0,\n",
       " 68.0,\n",
       " 30.0,\n",
       " 36.0,\n",
       " 21.5,\n",
       " 21.5,\n",
       " 21.5,\n",
       " 1.0,\n",
       " 68.5,\n",
       " 68.5,\n",
       " 42.0,\n",
       " 27.0,\n",
       " 86.0,\n",
       " 54.0,\n",
       " 16.5,\n",
       " 16.5,\n",
       " 32.0,\n",
       " 8.0,\n",
       " 38.0,\n",
       " 15.0,\n",
       " 5.0,\n",
       " 3.0,\n",
       " 28.5,\n",
       " 39.5,\n",
       " 44.0,\n",
       " 50.0,\n",
       " 27.0,\n",
       " 29.0,\n",
       " 91.0,\n",
       " 327.0,\n",
       " 14.0,\n",
       " 7.0,\n",
       " 7.0,\n",
       " 21.0,\n",
       " 1.0,\n",
       " 51.0,\n",
       " 4.0,\n",
       " 9.0,\n",
       " 5.0,\n",
       " 34.0,\n",
       " 2.5,\n",
       " 4.5,\n",
       " 6.0,\n",
       " 51.0,\n",
       " 51.0,\n",
       " 37.0,\n",
       " 8.0,\n",
       " 5.0,\n",
       " 35.0,\n",
       " 14.5,\n",
       " 14.5,\n",
       " 26.0,\n",
       " 12.0,\n",
       " 11.0,\n",
       " 6.0,\n",
       " 51.0,\n",
       " 51.0,\n",
       " 36.0,\n",
       " 91.0,\n",
       " 91.0,\n",
       " 12.0,\n",
       " 21.0,\n",
       " 21.0,\n",
       " 21.0,\n",
       " 25.0,\n",
       " 64.0,\n",
       " 64.0,\n",
       " 148.5,\n",
       " 148.5,\n",
       " 13.5,\n",
       " 40.5,\n",
       " 17.0,\n",
       " 4.0,\n",
       " 52.5,\n",
       " 77.5,\n",
       " 77.5,\n",
       " 97.0,\n",
       " 37.0,\n",
       " 2.0,\n",
       " 128.0,\n",
       " 47.0,\n",
       " 6.5,\n",
       " 8.5,\n",
       " 86.0,\n",
       " 6.5,\n",
       " 8.5,\n",
       " 29.0,\n",
       " 5.5,\n",
       " 6.5,\n",
       " 5.0,\n",
       " 13.0,\n",
       " 11.0,\n",
       " 55.0,\n",
       " 2.0,\n",
       " 37.0,\n",
       " 7.0,\n",
       " 66.5,\n",
       " 66.5,\n",
       " 6.0,\n",
       " 51.0,\n",
       " 51.0,\n",
       " 23.5,\n",
       " 23.5,\n",
       " 23.5,\n",
       " 23.5,\n",
       " 23.5,\n",
       " 23.5,\n",
       " 34.0,\n",
       " 28.0,\n",
       " 94.0,\n",
       " 4.0,\n",
       " 12.5,\n",
       " 20.0,\n",
       " 3.0,\n",
       " 10.0,\n",
       " 12.5,\n",
       " 5.0,\n",
       " 379.5,\n",
       " 379.5,\n",
       " 379.5,\n",
       " 379.5,\n",
       " 7.0,\n",
       " 47.0,\n",
       " 6.0,\n",
       " 15.0,\n",
       " 124.0,\n",
       " 106.0,\n",
       " 27.0,\n",
       " 11.5,\n",
       " 9.5,\n",
       " 2.0,\n",
       " 12.0,\n",
       " 582.0,\n",
       " 2.5,\n",
       " 9.5,\n",
       " 29.0,\n",
       " 454.0,\n",
       " 45.0,\n",
       " 4.0,\n",
       " 24.0,\n",
       " 8.0,\n",
       " 11.0,\n",
       " 11.0,\n",
       " 6.5,\n",
       " 68.5,\n",
       " 221.0,\n",
       " 5.0,\n",
       " 15.5,\n",
       " 1.0,\n",
       " 84.0,\n",
       " 23.0,\n",
       " 16.0,\n",
       " 4.5,\n",
       " 21.0,\n",
       " 4.5,\n",
       " 62.0,\n",
       " 6.0,\n",
       " 123.0,\n",
       " 26.0,\n",
       " 33.5,\n",
       " 48.0,\n",
       " 30.0,\n",
       " 68.0,\n",
       " 44.0,\n",
       " 167.0,\n",
       " 36.5,\n",
       " 47.5,\n",
       " 23.0,\n",
       " 33.0,\n",
       " 18.0,\n",
       " 1.0,\n",
       " 195.5,\n",
       " 193.5,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 14.5,\n",
       " 6.0,\n",
       " 61.0,\n",
       " 214.5,\n",
       " 129.5,\n",
       " 64.0,\n",
       " 40.5,\n",
       " 81.0,\n",
       " 18.0,\n",
       " 21.5,\n",
       " 21.5,\n",
       " 21.5,\n",
       " 24.5,\n",
       " 47.5,\n",
       " 40.0,\n",
       " 20.0,\n",
       " 32.0,\n",
       " 8.0,\n",
       " 26.0,\n",
       " 30.0,\n",
       " 91.0,\n",
       " 1.0,\n",
       " 3.0,\n",
       " 12.0,\n",
       " 21.0,\n",
       " 43.0,\n",
       " 78.0,\n",
       " 26.0,\n",
       " 7.5,\n",
       " 4.5,\n",
       " 16.0,\n",
       " 100.0,\n",
       " 69.0,\n",
       " 64.0,\n",
       " 19.0,\n",
       " 11.0,\n",
       " 1.0,\n",
       " 182.0,\n",
       " 10.0,\n",
       " 10.0,\n",
       " 5.5,\n",
       " 6.5,\n",
       " 5.0,\n",
       " 83.0,\n",
       " 31.0,\n",
       " 111.0,\n",
       " 4.0,\n",
       " 4.0,\n",
       " 26.0,\n",
       " 58.5,\n",
       " 58.5,\n",
       " 8.5,\n",
       " 12.0,\n",
       " 23.5,\n",
       " 14.5,\n",
       " 16.5,\n",
       " 16.5,\n",
       " 41.5,\n",
       " 9.0,\n",
       " 36.0,\n",
       " 4.0,\n",
       " 23.5,\n",
       " 8.5,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 9.0,\n",
       " 10.5,\n",
       " 7.5,\n",
       " 7.5,\n",
       " 58.5,\n",
       " 56.0,\n",
       " 26.0,\n",
       " 79.5,\n",
       " 79.5,\n",
       " 5.0,\n",
       " 54.0,\n",
       " 23.0,\n",
       " 12.0,\n",
       " 198.5,\n",
       " 167.5,\n",
       " 9.0,\n",
       " 365.5,\n",
       " 365.5,\n",
       " 61.0,\n",
       " 61.0,\n",
       " 20.0,\n",
       " 26.0,\n",
       " 112.0,\n",
       " 12.0,\n",
       " 24.0,\n",
       " 188.0,\n",
       " 571.5,\n",
       " 10.5,\n",
       " 13.5,\n",
       " 13.5,\n",
       " 6.0,\n",
       " 7.0,\n",
       " 118.5,\n",
       " 132.5,\n",
       " 63.0,\n",
       " 30.0,\n",
       " 26.0,\n",
       " 22.0,\n",
       " 187.0,\n",
       " 4.0,\n",
       " 177.5,\n",
       " 156.5,\n",
       " 8.0,\n",
       " 65.0,\n",
       " 31.0,\n",
       " 2.0,\n",
       " 1.0,\n",
       " 12.0,\n",
       " 30.5,\n",
       " 12.5,\n",
       " 53.0,\n",
       " 105.0,\n",
       " 438.0,\n",
       " 20.5,\n",
       " 3.5,\n",
       " 3.0,\n",
       " 52.0,\n",
       " 17.0,\n",
       " 118.0,\n",
       " 17.0,\n",
       " 1.0,\n",
       " 54.0,\n",
       " 18.0,\n",
       " 321.0,\n",
       " 29.0,\n",
       " 84.0,\n",
       " 10.0,\n",
       " 2.0,\n",
       " 41.0,\n",
       " 42.0,\n",
       " 42.0,\n",
       " 19.0,\n",
       " 2.0,\n",
       " 32.0,\n",
       " 12.0,\n",
       " 20.0,\n",
       " 33.0,\n",
       " 9.5,\n",
       " 16.5,\n",
       " 27.5,\n",
       " 27.5,\n",
       " 67.0,\n",
       " 34.5,\n",
       " 9.5,\n",
       " 16.5,\n",
       " 150.0,\n",
       " 29.5,\n",
       " 31.5,\n",
       " 31.0,\n",
       " 56.0,\n",
       " 38.0,\n",
       " 76.0,\n",
       " 34.5,\n",
       " 400.0,\n",
       " 106.0,\n",
       " 7.0,\n",
       " 16.0,\n",
       " 86.0,\n",
       " 48.5,\n",
       " 72.5,\n",
       " 31.5,\n",
       " 17.0]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "rank_results_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[[-0.22372659, -0.31156296, -0.20765123, ..., -0.42033401,\n",
       "          0.30742168, -0.4493041 ],\n",
       "        [ 0.32920173, -0.02098615,  0.19615731, ..., -0.0712066 ,\n",
       "         -0.04218373, -0.1483334 ],\n",
       "        [ 0.02773944,  0.03396924, -0.05110404, ...,  0.06384794,\n",
       "          0.13285932, -0.00319888],\n",
       "        ..., \n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ]],\n",
       "\n",
       "       [[ 0.18881664,  0.48994318, -0.05012255, ...,  0.06011058,\n",
       "          0.04016069, -0.20638183],\n",
       "        [ 0.05268412, -0.33071521, -0.15881902, ...,  0.11555439,\n",
       "          0.29032961, -0.17280847],\n",
       "        [ 0.04722372, -0.0245274 ,  0.2057476 , ...,  0.02759955,\n",
       "         -0.17411187,  0.11709464],\n",
       "        ..., \n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ]],\n",
       "\n",
       "       [[-0.04185124,  0.06431665,  0.19773521, ..., -0.07123161,\n",
       "          0.04165912, -0.26342642],\n",
       "        [-0.05411424,  0.07968321, -0.2059041 , ...,  0.08781962,\n",
       "         -0.2405041 , -0.053283  ],\n",
       "        [-0.25541565, -0.21479842, -0.1863151 , ...,  0.25159356,\n",
       "         -0.25664818,  0.26411185],\n",
       "        ..., \n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ]],\n",
       "\n",
       "       ..., \n",
       "       [[ 0.04935583, -0.07944034,  0.20348723, ..., -0.16266069,\n",
       "          0.35211655,  0.16719684],\n",
       "        [ 0.36271214, -0.13460168, -0.14109996, ..., -0.20520478,\n",
       "         -0.23590836,  0.15635321],\n",
       "        [ 0.14209744,  0.16969153,  0.31780741, ...,  0.10322033,\n",
       "         -0.08274973, -0.07952023],\n",
       "        ..., \n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ]],\n",
       "\n",
       "       [[ 0.00406894, -0.0461982 ,  0.01190926, ..., -0.03445234,\n",
       "         -0.03110966,  0.01853556],\n",
       "        [ 0.04935583, -0.07944034,  0.20348723, ..., -0.16266069,\n",
       "          0.35211655,  0.16719684],\n",
       "        [ 0.36271214, -0.13460168, -0.14109996, ..., -0.20520478,\n",
       "         -0.23590836,  0.15635321],\n",
       "        ..., \n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ]],\n",
       "\n",
       "       [[ 0.04935583, -0.07944034,  0.20348723, ..., -0.16266069,\n",
       "          0.35211655,  0.16719684],\n",
       "        [ 0.36271214, -0.13460168, -0.14109996, ..., -0.20520478,\n",
       "         -0.23590836,  0.15635321],\n",
       "        [ 0.14209744,  0.16969153,  0.31780741, ...,  0.10322033,\n",
       "         -0.08274973, -0.07952023],\n",
       "        ..., \n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ],\n",
       "        [ 0.        ,  0.        ,  0.        , ...,  0.        ,\n",
       "          0.        ,  0.        ]]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X1_test_1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:360: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n",
      "keras_title.py:365: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [240, 689, 923, 2985, 3422]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:434: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 50, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_87 (LSTM)                   (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_88 (LSTM)                   (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_44 (Concatenate)     (None, 400)           0           lstm_87[0][0]                    \n",
      "                                                                   lstm_88[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_87 (Dense)                 (None, 100)           40100       concatenate_44[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_87 (Dropout)             (None, 100)           0           dense_87[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_88 (Dense)                 (None, 50)            5050        dropout_87[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_88 (Dropout)             (None, 50)            0           dense_88[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_88[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 686,801\n",
      "Trainable params: 686,801\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 19s - loss: 0.6086 - val_loss: 0.5272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:104: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:139: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 23\n",
      "TOP5 130\n",
      "TOP10 257\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 19s - loss: 0.3140 - val_loss: 0.4590\n",
      "TOP1 33\n",
      "TOP5 170\n",
      "TOP10 261\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 19s - loss: 0.2165 - val_loss: 0.4084\n",
      "TOP1 49\n",
      "TOP5 212\n",
      "TOP10 334\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 19s - loss: 0.1479 - val_loss: 0.3831\n",
      "TOP1 48\n",
      "TOP5 231\n",
      "TOP10 379\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 19s - loss: 0.1043 - val_loss: 0.3901\n",
      "TOP1 48\n",
      "TOP5 254\n",
      "TOP10 388\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 19s - loss: 0.0765 - val_loss: 0.4082\n",
      "TOP1 43\n",
      "TOP5 236\n",
      "TOP10 385\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 19s - loss: 0.0540 - val_loss: 0.3916\n",
      "TOP1 45\n",
      "TOP5 242\n",
      "TOP10 421\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 19s - loss: 0.0370 - val_loss: 0.4110\n",
      "TOP1 60\n",
      "TOP5 228\n",
      "TOP10 384\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 19s - loss: 0.0334 - val_loss: 0.3847\n",
      "TOP1 61\n",
      "TOP5 280\n",
      "TOP10 414\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 19s - loss: 0.0334 - val_loss: 0.4075\n",
      "TOP1 59\n",
      "TOP5 252\n",
      "TOP10 387\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 19s - loss: 0.0209 - val_loss: 0.4400\n",
      "TOP1 52\n",
      "TOP5 237\n",
      "TOP10 372\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 19s - loss: 0.0188 - val_loss: 0.4224\n",
      "TOP1 54\n",
      "TOP5 224\n",
      "TOP10 376\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 19s - loss: 0.0169 - val_loss: 0.4382\n",
      "TOP1 62\n",
      "TOP5 240\n",
      "TOP10 392\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 19s - loss: 0.0099 - val_loss: 0.4308\n",
      "TOP1 58\n",
      "TOP5 249\n",
      "TOP10 395\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 19s - loss: 0.0102 - val_loss: 0.4210\n",
      "TOP1 54\n",
      "TOP5 227\n",
      "TOP10 382\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 19s - loss: 0.0301 - val_loss: 0.4545\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    452\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n\u001b[1;32m    453\u001b[0m                                                                  \u001b[0mX2_test_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m                                                                  model_lstm2)\n\u001b[0m\u001b[1;32m    455\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m         \u001b[0;31m# Evaluation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36mfind_ranking_quick\u001b[0;34m(projection1, projection2, dlmodel)\u001b[0m\n\u001b[1;32m    116\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m     \u001b[0;31m# Find the real projection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 118\u001b[0;31m     \u001b[0mv1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mprojection1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojection2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0;31m# ---- Prepare the model_2 ----\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1589\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1590\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1591\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_predict_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1592\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1593\u001b[0m         return self._predict_loop(f, ins,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_make_predict_function\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1051\u001b[0m                                                \u001b[0mupdates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_updates\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m                                                \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'predict_function'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m                                                **kwargs)\n\u001b[0m\u001b[1;32m   1054\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m     def _fit_loop(self, f, ins, out_labels=None, batch_size=32,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36mfunction\u001b[0;34m(inputs, outputs, updates, **kwargs)\u001b[0m\n\u001b[1;32m   1204\u001b[0m                 \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Invalid argument \"%s\" passed to K.function with Theano backend'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1205\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1206\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mFunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mupdates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, updates, name, **kwargs)\u001b[0m\n\u001b[1;32m   1189\u001b[0m                                         \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1190\u001b[0m                                         \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m                                         **kwargs)\n\u001b[0m\u001b[1;32m   1192\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function.pyc\u001b[0m in \u001b[0;36mfunction\u001b[0;34m(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)\u001b[0m\n\u001b[1;32m    324\u001b[0m                    \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m                    \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m                    output_keys=output_keys)\n\u001b[0m\u001b[1;32m    327\u001b[0m     \u001b[0;31m# We need to add the flag check_aliased inputs if we have any mutable or\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m     \u001b[0;31m# borrowed used defined inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mpfunc\u001b[0;34m(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys)\u001b[0m\n\u001b[1;32m    484\u001b[0m                          \u001b[0maccept_inplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maccept_inplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m                          \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 486\u001b[0;31m                          output_keys=output_keys)\n\u001b[0m\u001b[1;32m    487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36morig_function\u001b[0;34m(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys)\u001b[0m\n\u001b[1;32m   1792\u001b[0m                    \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1793\u001b[0m                    \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1794\u001b[0;31m                    \u001b[0moutput_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1795\u001b[0m             defaults)\n\u001b[1;32m   1796\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, mode, accept_inplace, function_builder, profile, on_unused_input, fgraph, output_keys)\u001b[0m\n\u001b[1;32m   1472\u001b[0m                         optimizer, inputs, outputs)\n\u001b[1;32m   1473\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1474\u001b[0;31m                     \u001b[0moptimizer_profile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1476\u001b[0m                 \u001b[0mend_optimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \"\"\"\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0madd_requirements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    233\u001b[0m                 \u001b[0mnb_nodes_before\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_nodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0msub_prof\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m                 \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m                 \u001b[0msub_profs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_prof\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m   1623\u001b[0m                                                                scan_op.Scan)]\n\u001b[1;32m   1624\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnodelist\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1625\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_node\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1626\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1627\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_opt.pyc\u001b[0m in \u001b[0;36mprocess_node\u001b[0;34m(self, fgraph, node)\u001b[0m\n\u001b[1;32m   1525\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1526\u001b[0m             \u001b[0;31m# Do not call make_node for test_value\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1527\u001b[0;31m             new_outs = scan_op.Scan(inps, outs, info)(*node_ins,\n\u001b[0m\u001b[1;32m   1528\u001b[0m                                                       **dict(return_list=True))\n\u001b[1;32m   1529\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, info, typeConstructor)\u001b[0m\n\u001b[1;32m    209\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    210\u001b[0m             tmp_in, tmp_out = scan_utils.reconstruct_graph(self.inputs,\n\u001b[0;32m--> 211\u001b[0;31m                                                            self.outputs)\n\u001b[0m\u001b[1;32m    212\u001b[0m             \u001b[0;31m# This is actually required for the line just after.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    213\u001b[0m             \u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFunctionGraph\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtmp_in\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtmp_out\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_utils.pyc\u001b[0m in \u001b[0;36mreconstruct_graph\u001b[0;34m(inputs, outputs, tag)\u001b[0m\n\u001b[1;32m   1120\u001b[0m             \u001b[0mgivens\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0minp\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1122\u001b[0;31m     \u001b[0mnw_outputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mgivens\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1123\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mnw_inputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnw_outputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_utils.pyc\u001b[0m in \u001b[0;36mclone\u001b[0;34m(output, replace, strict, share_inputs, copy_inputs)\u001b[0m\n\u001b[1;32m    248\u001b[0m                                          \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    249\u001b[0m                                          \u001b[0mstrict\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 250\u001b[0;31m                                          share_inputs)\n\u001b[0m\u001b[1;32m    251\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    252\u001b[0m     \u001b[0;31m# TODO Explain why we call it twice ?!\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mrebuild_collect_shared\u001b[0;34m(outputs, inputs, replace, updates, rebuild_strict, copy_inputs_over, no_default_updates)\u001b[0m\n\u001b[1;32m    217\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    218\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 219\u001b[0;31m                 \u001b[0mcloned_v\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclone_v_get_shared_updates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_inputs_over\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    220\u001b[0m                 \u001b[0mcloned_outputs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcloned_v\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    221\u001b[0m             \u001b[0;32melif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mOut\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mclone_v_get_shared_updates\u001b[0;34m(v, copy_inputs_over)\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mowner\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mclone_d\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mowner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                     \u001b[0mclone_v_get_shared_updates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_inputs_over\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 clone_d[owner] = owner.clone_with_new_inputs(\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mclone_v_get_shared_updates\u001b[0;34m(v, copy_inputs_over)\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mowner\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mclone_d\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mowner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                     \u001b[0mclone_v_get_shared_updates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_inputs_over\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 clone_d[owner] = owner.clone_with_new_inputs(\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mclone_v_get_shared_updates\u001b[0;34m(v, copy_inputs_over)\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mowner\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mclone_d\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mowner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                     \u001b[0mclone_v_get_shared_updates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_inputs_over\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 clone_d[owner] = owner.clone_with_new_inputs(\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mclone_v_get_shared_updates\u001b[0;34m(v, copy_inputs_over)\u001b[0m\n\u001b[1;32m     91\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mowner\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mclone_d\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     92\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mowner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m                     \u001b[0mclone_v_get_shared_updates\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcopy_inputs_over\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 clone_d[owner] = owner.clone_with_new_inputs(\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mclone_v_get_shared_updates\u001b[0;34m(v, copy_inputs_over)\u001b[0m\n\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m                 clone_d[owner] = owner.clone_with_new_inputs(\n\u001b[0;32m---> 96\u001b[0;31m                     [clone_d[i] for i in owner.inputs], strict=rebuild_strict)\n\u001b[0m\u001b[1;32m     97\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mold_o\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_o\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mowner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclone_d\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mowner\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m                     \u001b[0mclone_d\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msetdefault\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mold_o\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_o\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mclone_with_new_inputs\u001b[0;34m(self, inputs, strict)\u001b[0m\n\u001b[1;32m    243\u001b[0m             \u001b[0mnew_node\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__update__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnew_node\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 245\u001b[0;31m             \u001b[0mnew_node\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclone\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m             \u001b[0mnew_node\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mnew_node\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mclone\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    201\u001b[0m         \"\"\"\n\u001b[1;32m    202\u001b[0m         cp = self.__class__(self.op, self.inputs,\n\u001b[0;32m--> 203\u001b[0;31m                             [output.clone() for output in self.outputs])\n\u001b[0m\u001b[1;32m    204\u001b[0m         \u001b[0mcp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    205\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mclone\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    446\u001b[0m         \u001b[0;31m# return copy(self)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m         \u001b[0mcp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__class__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtype\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m         \u001b[0mcp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcopy\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtag\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    449\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mcp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/copy.pyc\u001b[0m in \u001b[0;36mcopy\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m     94\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"un(shallow)copyable object of type %s\"\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mcls\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 96\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_reconstruct\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mrv\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     97\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     98\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# 使用了200维的lstm，maxlen 50\n",
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:359: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:360: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n",
      "keras_title.py:365: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 2 position: [1272, 1690]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:434: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_125 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_126 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_62 (Concatenate)     (None, 400)           0           lstm_125[0][0]                   \n",
      "                                                                   lstm_126[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_123 (Dense)                (None, 400)           160400      concatenate_62[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_123 (Dropout)            (None, 400)           0           dense_123[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_124 (Dense)                (None, 50)            20050       dropout_123[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_124 (Dropout)            (None, 50)            0           dense_124[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             51          dropout_124[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 822,101\n",
      "Trainable params: 822,101\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.5742 - val_loss: 0.5272\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:104: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:139: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 22\n",
      "TOP5 132\n",
      "TOP10 265\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.2932 - val_loss: 0.4486\n",
      "TOP1 50\n",
      "TOP5 202\n",
      "TOP10 359\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.1822 - val_loss: 0.3744\n",
      "TOP1 59\n",
      "TOP5 260\n",
      "TOP10 386\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1201 - val_loss: 0.3712\n",
      "TOP1 52\n",
      "TOP5 238\n",
      "TOP10 380\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0779 - val_loss: 0.3902\n",
      "TOP1 61\n",
      "TOP5 283\n",
      "TOP10 415\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0510 - val_loss: 0.4307\n",
      "TOP1 62\n",
      "TOP5 262\n",
      "TOP10 408\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 5s - loss: 0.0372 - val_loss: 0.3861\n",
      "TOP1 47\n",
      "TOP5 259\n",
      "TOP10 403\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0283 - val_loss: 0.5089\n",
      "TOP1 50\n",
      "TOP5 251\n",
      "TOP10 390\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0505 - val_loss: 0.4424\n",
      "TOP1 37\n",
      "TOP5 209\n",
      "TOP10 355\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0580 - val_loss: 0.4494\n",
      "TOP1 21\n",
      "TOP5 158\n",
      "TOP10 286\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 4s - loss: 0.0409 - val_loss: 0.4550\n",
      "TOP1 20\n",
      "TOP5 129\n",
      "TOP10 270\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 4s - loss: 0.0285 - val_loss: 0.3867\n",
      "TOP1 15\n",
      "TOP5 107\n",
      "TOP10 235\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 4s - loss: 0.0236 - val_loss: 0.3792\n",
      "TOP1 20\n",
      "TOP5 127\n",
      "TOP10 242\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 4s - loss: 0.0224 - val_loss: 0.4641\n",
      "TOP1 21\n",
      "TOP5 129\n",
      "TOP10 325\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 4s - loss: 0.0383 - val_loss: 0.4340\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    452\u001b[0m                            initial_epoch=i)\n\u001b[1;32m    453\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 454\u001b[0;31m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n\u001b[0m\u001b[1;32m    455\u001b[0m                                                                  \u001b[0mX2_test_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    456\u001b[0m                                                                  model_lstm2)\n",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36mfind_ranking_quick\u001b[0;34m(projection1, projection2, dlmodel)\u001b[0m\n\u001b[1;32m    116\u001b[0m     \u001b[0mmodel_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdlmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m     \u001b[0mmodel_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mset_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdlmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m3\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_weights\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 118\u001b[0;31m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m     \u001b[0;31m# Find the real projection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0mv1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mprojection1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojection2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1589\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1590\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1591\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_predict_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1592\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1593\u001b[0m         return self._predict_loop(f, ins,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_make_predict_function\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1051\u001b[0m                                                \u001b[0mupdates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_updates\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m                                                \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'predict_function'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m                                                **kwargs)\n\u001b[0m\u001b[1;32m   1054\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m     def _fit_loop(self, f, ins, out_labels=None, batch_size=32,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36mfunction\u001b[0;34m(inputs, outputs, updates, **kwargs)\u001b[0m\n\u001b[1;32m   1204\u001b[0m                 \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Invalid argument \"%s\" passed to K.function with Theano backend'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1205\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1206\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mFunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mupdates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, updates, name, **kwargs)\u001b[0m\n\u001b[1;32m   1189\u001b[0m                                         \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1190\u001b[0m                                         \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m                                         **kwargs)\n\u001b[0m\u001b[1;32m   1192\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function.pyc\u001b[0m in \u001b[0;36mfunction\u001b[0;34m(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)\u001b[0m\n\u001b[1;32m    324\u001b[0m                    \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m                    \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m                    output_keys=output_keys)\n\u001b[0m\u001b[1;32m    327\u001b[0m     \u001b[0;31m# We need to add the flag check_aliased inputs if we have any mutable or\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m     \u001b[0;31m# borrowed used defined inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mpfunc\u001b[0;34m(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys)\u001b[0m\n\u001b[1;32m    484\u001b[0m                          \u001b[0maccept_inplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maccept_inplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m                          \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 486\u001b[0;31m                          output_keys=output_keys)\n\u001b[0m\u001b[1;32m    487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36morig_function\u001b[0;34m(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys)\u001b[0m\n\u001b[1;32m   1792\u001b[0m                    \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1793\u001b[0m                    \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1794\u001b[0;31m                    \u001b[0moutput_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1795\u001b[0m             defaults)\n\u001b[1;32m   1796\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, mode, accept_inplace, function_builder, profile, on_unused_input, fgraph, output_keys)\u001b[0m\n\u001b[1;32m   1472\u001b[0m                         optimizer, inputs, outputs)\n\u001b[1;32m   1473\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1474\u001b[0;31m                     \u001b[0moptimizer_profile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1476\u001b[0m                 \u001b[0mend_optimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \"\"\"\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0madd_requirements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    233\u001b[0m                 \u001b[0mnb_nodes_before\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_nodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0msub_prof\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m                 \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m                 \u001b[0msub_profs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_prof\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    233\u001b[0m                 \u001b[0mnb_nodes_before\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_nodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0msub_prof\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m                 \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m                 \u001b[0msub_profs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_prof\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph, start_from)\u001b[0m\n\u001b[1;32m   2438\u001b[0m             \u001b[0;31m# apply local optimizer\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2439\u001b[0m             \u001b[0mtopo_t0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2440\u001b[0;31m             \u001b[0mq\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdeque\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mio_toposort\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstart_from\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2441\u001b[0m             \u001b[0mio_toposort_timing\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtopo_t0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2442\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mio_toposort\u001b[0;34m(inputs, outputs, orderings, clients)\u001b[0m\n\u001b[1;32m   1029\u001b[0m     topo = general_toposort(outputs, deps=compute_deps,\n\u001b[1;32m   1030\u001b[0m                             \u001b[0mcompute_deps_cache\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcompute_deps_cache\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1031\u001b[0;31m                             deps_cache=deps_cache, clients=clients)\n\u001b[0m\u001b[1;32m   1032\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtopo\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mApply\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mgeneral_toposort\u001b[0;34m(r_out, deps, debug_print, compute_deps_cache, deps_cache, clients)\u001b[0m\n\u001b[1;32m    936\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    937\u001b[0m     reachable, _clients = stack_search(deque(r_out), compute_deps_cache,\n\u001b[0;32m--> 938\u001b[0;31m                                        'dfs', True)\n\u001b[0m\u001b[1;32m    939\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mclients\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    940\u001b[0m         \u001b[0mclients\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_clients\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mstack_search\u001b[0;34m(start, expand, mode, build_inv)\u001b[0m\n\u001b[1;32m    639\u001b[0m             \u001b[0mrval_list\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    640\u001b[0m             \u001b[0mrval_set\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mid\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 641\u001b[0;31m             \u001b[0mexpand_l\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mexpand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0ml\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    642\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mexpand_l\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    643\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mbuild_inv\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/graph.pyc\u001b[0m in \u001b[0;36mcompute_deps_cache\u001b[0;34m(obj)\u001b[0m\n\u001b[1;32m    997\u001b[0m             \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    998\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0miset\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 999\u001b[0;31m                 \u001b[0;32mif\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mVariable\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1000\u001b[0m                     \u001b[0;32mif\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mowner\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m                         \u001b[0mrval\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mobj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mowner\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:361: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:362: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # ---- Padding the vector ---- #\n",
      "keras_title.py:366: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:367: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # --- Prepare the training data --- #\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 3 position: [61, 1467, 3411]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:436: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # model_lstm2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_157 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_158 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_77 (Concatenate)     (None, 400)           0           lstm_157[0][0]                   \n",
      "                                                                   lstm_158[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_153 (Dense)                (None, 600)           240600      concatenate_77[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_153 (Dropout)            (None, 600)           0           dense_153[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_154 (Dense)                (None, 100)           60100       dropout_153[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_154 (Dropout)            (None, 100)           0           dense_154[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_154[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 942,401\n",
      "Trainable params: 942,401\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.6026 - val_loss: 0.4736\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:106: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
      "keras_title.py:141: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 24\n",
      "TOP5 109\n",
      "TOP10 200\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.3194 - val_loss: 0.4272\n",
      "TOP1 51\n",
      "TOP5 235\n",
      "TOP10 344\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.2027 - val_loss: 0.3758\n",
      "TOP1 66\n",
      "TOP5 264\n",
      "TOP10 425\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1373 - val_loss: 0.3581\n",
      "TOP1 60\n",
      "TOP5 270\n",
      "TOP10 409\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0892 - val_loss: 0.3996\n",
      "TOP1 69\n",
      "TOP5 250\n",
      "TOP10 379\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0621 - val_loss: 0.3829\n",
      "TOP1 66\n",
      "TOP5 309\n",
      "TOP10 428\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0446 - val_loss: 0.3847\n",
      "TOP1 62\n",
      "TOP5 287\n",
      "TOP10 417\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0382 - val_loss: 0.3470\n",
      "TOP1 56\n",
      "TOP5 276\n",
      "TOP10 438\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0376 - val_loss: 0.4675\n",
      "TOP1 43\n",
      "TOP5 247\n",
      "TOP10 378\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0632 - val_loss: 0.4282\n",
      "TOP1 30\n",
      "TOP5 202\n",
      "TOP10 342\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 4s - loss: 0.0481 - val_loss: 0.4732\n",
      "TOP1 42\n",
      "TOP5 208\n",
      "TOP10 337\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 4s - loss: 0.0369 - val_loss: 0.4297\n",
      "TOP1 29\n",
      "TOP5 173\n",
      "TOP10 310\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 4s - loss: 0.0393 - val_loss: 0.5356\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    454\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n\u001b[1;32m    455\u001b[0m                                                                  \u001b[0mX2_test_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 456\u001b[0;31m                                                                  model_lstm2)\n\u001b[0m\u001b[1;32m    457\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    458\u001b[0m         \u001b[0;31m# Evaluation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36mfind_ranking_quick\u001b[0;34m(projection1, projection2, dlmodel)\u001b[0m\n\u001b[1;32m    164\u001b[0m         \u001b[0;31m# print(\"Find answer for doc.\", i)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    165\u001b[0m         \u001b[0mproj1_tile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproj1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprojection2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 166\u001b[0;31m         \u001b[0msim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mproj1_tile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojection2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    167\u001b[0m         \u001b[0mrank\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrank\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mascending\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    168\u001b[0m         \u001b[0msim_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1592\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1593\u001b[0m         return self._predict_loop(f, ins,\n\u001b[0;32m-> 1594\u001b[0;31m                                   batch_size=batch_size, verbose=verbose)\n\u001b[0m\u001b[1;32m   1595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1596\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1216\u001b[0m                 \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1218\u001b[0;31m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1219\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1220\u001b[0m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 600 100\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:363: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n",
      "keras_title.py:368: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:369: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Generate training data (similarity = 1)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [637, 995, 2424, 3391, 3929, 4263, 4729]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:438: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # model.compile(optimizer='adam', loss='sparse_categorical_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_185 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_186 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_91 (Concatenate)     (None, 400)           0           lstm_185[0][0]                   \n",
      "                                                                   lstm_186[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_181 (Dense)                (None, 600)           240600      concatenate_91[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_181 (Dropout)            (None, 600)           0           dense_181[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_182 (Dense)                (None, 100)           60100       dropout_181[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_182 (Dropout)            (None, 100)           0           dense_182[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_182[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 942,401\n",
      "Trainable params: 942,401\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.5907 - val_loss: 0.4887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:108: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:143: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 25\n",
      "TOP5 117\n",
      "TOP10 203\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.3237 - val_loss: 0.4447\n",
      "TOP1 45\n",
      "TOP5 196\n",
      "TOP10 326\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.2101 - val_loss: 0.4365\n",
      "TOP1 53\n",
      "TOP5 226\n",
      "TOP10 347\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.1462 - val_loss: 0.4015\n",
      "TOP1 50\n",
      "TOP5 283\n",
      "TOP10 408\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.0883 - val_loss: 0.4455\n",
      "TOP1 59\n",
      "TOP5 286\n",
      "TOP10 400\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.0577 - val_loss: 0.4541\n",
      "TOP1 18\n",
      "TOP5 136\n",
      "TOP10 285\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0405 - val_loss: 0.4873\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n\u001b[0;32m--> 458\u001b[0;31m                                                                  \u001b[0mX2_test_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    459\u001b[0m                                                                  model_lstm2)\n\u001b[1;32m    460\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36mfind_ranking_quick\u001b[0;34m(projection1, projection2, dlmodel)\u001b[0m\n\u001b[1;32m    166\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mproj1\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprojection1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    167\u001b[0m         \u001b[0;31m# print(\"Find answer for doc.\", i)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 168\u001b[0;31m         \u001b[0mproj1_tile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproj1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprojection2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    169\u001b[0m         \u001b[0msim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mproj1_tile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojection2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m         \u001b[0mrank\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrank\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mascending\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1592\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1593\u001b[0m         return self._predict_loop(f, ins,\n\u001b[0;32m-> 1594\u001b[0;31m                                   batch_size=batch_size, verbose=verbose)\n\u001b[0m\u001b[1;32m   1595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1596\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1214\u001b[0m                 \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1215\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1216\u001b[0;31m                 \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1218\u001b[0m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_slice_arrays\u001b[0;34m(arrays, start, stop)\u001b[0m\n\u001b[1;32m    392\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mhasattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'shape'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m                 \u001b[0mstart\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mstart\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtolist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 394\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    395\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    396\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mstart\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0mstop\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marrays\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using Theano backend.\n",
      "WARNING (theano.sandbox.cuda): The cuda backend is deprecated and will be removed in the next release (v0.10).  Please switch to the gpuarray backend. You can get more information about how to switch at this URL:\n",
      " https://github.com/Theano/Theano/wiki/Converting-to-the-new-gpu-back-end%28gpuarray%29\n",
      "\n",
      "Using gpu device 0: GeForce GTX TITAN X (CNMeM is enabled with initial size: 95.0% of memory, cuDNN 4007)\n",
      "/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/gensim/utils.py:1015: UserWarning: Pattern library is not installed, lemmatization won't be available.\n",
      "  warnings.warn(\"Pattern library is not installed, lemmatization won't be available.\")\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 600 100\n",
      "dropout_rate 0.3\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:365: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:369: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:370: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 1 position: [4307]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:439: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  model_lstm2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                    (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                    (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 400)           0           lstm_1[0][0]                     \n",
      "                                                                   lstm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 600)           240600      concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 600)           0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 100)           60100       dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 100)           0           dense_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 942,401\n",
      "Trainable params: 942,401\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.6733 - val_loss: 0.4948\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:109: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:144: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 27\n",
      "TOP5 110\n",
      "TOP10 189\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.4164 - val_loss: 0.4076\n",
      "TOP1 37\n",
      "TOP5 190\n",
      "TOP10 326\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.3023 - val_loss: 0.3899\n",
      "TOP1 46\n",
      "TOP5 218\n",
      "TOP10 347\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.2293 - val_loss: 0.3767\n",
      "TOP1 54\n",
      "TOP5 232\n",
      "TOP10 367\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.1694 - val_loss: 0.3615\n",
      "TOP1 51\n",
      "TOP5 251\n",
      "TOP10 382\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.1329 - val_loss: 0.3674\n",
      "TOP1 51\n",
      "TOP5 229\n",
      "TOP10 375\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.1105 - val_loss: 0.3693\n",
      "TOP1 47\n",
      "TOP5 237\n",
      "TOP10 393\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0831 - val_loss: 0.3527\n",
      "TOP1 48\n",
      "TOP5 265\n",
      "TOP10 401\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0693 - val_loss: 0.3782\n",
      "TOP1 49\n",
      "TOP5 230\n",
      "TOP10 356\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0650 - val_loss: 0.3858\n",
      "TOP1 51\n",
      "TOP5 219\n",
      "TOP10 371\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 6s - loss: 0.0580 - val_loss: 0.4151\n",
      "TOP1 46\n",
      "TOP5 214\n",
      "TOP10 368\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0556 - val_loss: 0.3837\n",
      "TOP1 38\n",
      "TOP5 229\n",
      "TOP10 391\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 6s - loss: 0.0516 - val_loss: 0.4159\n",
      "TOP1 57\n",
      "TOP5 262\n",
      "TOP10 399\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 6s - loss: 0.0391 - val_loss: 0.3933\n",
      "TOP1 43\n",
      "TOP5 262\n",
      "TOP10 420\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 6s - loss: 0.0415 - val_loss: 0.4160\n",
      "TOP1 42\n",
      "TOP5 239\n",
      "TOP10 387\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 6s - loss: 0.0336 - val_loss: 0.4054\n",
      "TOP1 50\n",
      "TOP5 260\n",
      "TOP10 403\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 17/17\n",
      "3104/8000 [==========>...................] - ETA: 3s - loss: 0.0319"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    453\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 400 dim_Dense 600 100\n",
      "dropout_rate 0.0\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:364: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_en'] = df_pairs_sample['en_article'].apply(doc2vec_en)\n",
      "keras_title.py:365: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['word2vec_jp'] = df_pairs_sample['jp_article'].apply(doc2vec_jp)\n",
      "keras_title.py:369: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_en'] = df_pairs_sample['word2vec_en'].apply(padding)\n",
      "keras_title.py:370: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  df_pairs_sample['padding_jp'] = df_pairs_sample['word2vec_jp'].apply(padding)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [2560, 3372, 3434, 3936, 4548]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:439: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  model_lstm2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                    (None, 400)           961600      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_2 (LSTM)                    (None, 400)           961600      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)      (None, 800)           0           lstm_1[0][0]                     \n",
      "                                                                   lstm_2[0][0]                     \n",
      "____________________________________________________________________________________________________\n",
      "dense_1 (Dense)                  (None, 600)           480600      concatenate_1[0][0]              \n",
      "____________________________________________________________________________________________________\n",
      "dropout_1 (Dropout)              (None, 600)           0           dense_1[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_2 (Dense)                  (None, 100)           60100       dropout_1[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dropout_2 (Dropout)              (None, 100)           0           dense_2[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_2[0][0]                  \n",
      "====================================================================================================\n",
      "Total params: 2,464,001\n",
      "Trainable params: 2,464,001\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 12s - loss: 0.5899 - val_loss: 0.5504\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:109: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1 = Model(input=[input_1, input_2], output=[lstm_out_1, lstm_out_2])\n",
      "keras_title.py:144: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=sigmoid.0, inputs=[/lstm1, /...)`\n",
      "  model_2 = Model(input=[input_1, input_2], output=main_output)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 17\n",
      "TOP5 97\n",
      "TOP10 190\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 12s - loss: 0.3407 - val_loss: 0.4800\n",
      "TOP1 28\n",
      "TOP5 159\n",
      "TOP10 258\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 13s - loss: 0.2336 - val_loss: 0.4005\n",
      "TOP1 48\n",
      "TOP5 198\n",
      "TOP10 332\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 12s - loss: 0.1604 - val_loss: 0.4050\n",
      "TOP1 37\n",
      "TOP5 184\n",
      "TOP10 352\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 13s - loss: 0.1175 - val_loss: 0.4605\n",
      "TOP1 45\n",
      "TOP5 203\n",
      "TOP10 345\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 13s - loss: 0.0843 - val_loss: 0.4239\n",
      "TOP1 57\n",
      "TOP5 238\n",
      "TOP10 380\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 13s - loss: 0.0574 - val_loss: 0.4428\n",
      "TOP1 39\n",
      "TOP5 235\n",
      "TOP10 398\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 13s - loss: 0.0396 - val_loss: 0.4580\n",
      "TOP1 36\n",
      "TOP5 205\n",
      "TOP10 354\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 13s - loss: 0.0363 - val_loss: 0.4235\n",
      "TOP1 36\n",
      "TOP5 220\n",
      "TOP10 376\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 13s - loss: 0.0297 - val_loss: 0.4275\n",
      "TOP1 34\n",
      "TOP5 226\n",
      "TOP10 397\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 13s - loss: 0.0273 - val_loss: 0.4789\n",
      "TOP1 41\n",
      "TOP5 216\n",
      "TOP10 368\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 13s - loss: 0.0308 - val_loss: 0.4402\n",
      "TOP1 41\n",
      "TOP5 256\n",
      "TOP10 391\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 13s - loss: 0.0237 - val_loss: 0.4119\n",
      "TOP1 42\n",
      "TOP5 218\n",
      "TOP10 343\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 13s - loss: 0.0258 - val_loss: 0.4353\n",
      "TOP1 44\n",
      "TOP5 218\n",
      "TOP10 372\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 13s - loss: 0.0347 - val_loss: 0.4453\n",
      "TOP1 39\n",
      "TOP5 215\n",
      "TOP10 352\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 13s - loss: 0.0421 - val_loss: 0.5160\n",
      "TOP1 37\n",
      "TOP5 194\n",
      "TOP10 333\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 17/17\n",
      "8000/8000 [==============================] - 13s - loss: 0.0316 - val_loss: 0.4641\n",
      "TOP1 34\n",
      "TOP5 168\n",
      "TOP10 332\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 18/18\n",
      "8000/8000 [==============================] - 13s - loss: 0.0160 - val_loss: 0.4791\n",
      "TOP1 33\n",
      "TOP5 204\n",
      "TOP10 363\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 19/19\n",
      "3424/8000 [===========>..................] - ETA: 6s - loss: 0.0081"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    453\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    454\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 455\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    456\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.3\n",
      "Activation function: ['relu', 'relu']\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:367: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:368: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # ---- Padding the vector ---- #\n",
      "keras_title.py:372: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:373: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # --- Prepare the training data --- #\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 3 position: [1012, 1147, 4933]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:442: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # model_lstm2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_45 (LSTM)                   (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_46 (LSTM)                   (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_23 (Concatenate)     (None, 400)           0           lstm_45[0][0]                    \n",
      "                                                                   lstm_46[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_45 (Dense)                 (None, 400)           160400      concatenate_23[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_45 (Dropout)             (None, 400)           0           dense_45[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_46 (Dense)                 (None, 100)           40100       dropout_45[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_46 (Dropout)             (None, 100)           0           dense_46[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_46[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.6851 - val_loss: 0.5131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:111: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # Compile the model\n",
      "keras_title.py:147: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 18\n",
      "TOP5 130\n",
      "TOP10 220\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.4019 - val_loss: 0.4660\n",
      "TOP1 28\n",
      "TOP5 162\n",
      "TOP10 281\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.2894 - val_loss: 0.4402\n",
      "TOP1 27\n",
      "TOP5 199\n",
      "TOP10 322\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.2256 - val_loss: 0.3843\n",
      "TOP1 57\n",
      "TOP5 237\n",
      "TOP10 362\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.1717 - val_loss: 0.3451\n",
      "TOP1 53\n",
      "TOP5 231\n",
      "TOP10 393\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.1270 - val_loss: 0.3339\n",
      "TOP1 59\n",
      "TOP5 267\n",
      "TOP10 394\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.1048 - val_loss: 0.3463\n",
      "TOP1 48\n",
      "TOP5 246\n",
      "TOP10 406\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0904 - val_loss: 0.3582\n",
      "TOP1 58\n",
      "TOP5 268\n",
      "TOP10 419\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0801 - val_loss: 0.3443\n",
      "TOP1 49\n",
      "TOP5 273\n",
      "TOP10 420\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0635 - val_loss: 0.3407\n",
      "TOP1 51\n",
      "TOP5 289\n",
      "TOP10 434\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 6s - loss: 0.0516 - val_loss: 0.3326\n",
      "TOP1 63\n",
      "TOP5 277\n",
      "TOP10 440\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0450 - val_loss: 0.3446\n",
      "TOP1 54\n",
      "TOP5 261\n",
      "TOP10 381\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 6s - loss: 0.0422 - val_loss: 0.3865\n",
      "TOP1 70\n",
      "TOP5 273\n",
      "TOP10 393\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 6s - loss: 0.0432 - val_loss: 0.4190\n",
      "TOP1 56\n",
      "TOP5 259\n",
      "TOP10 421\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 7s - loss: 0.0382 - val_loss: 0.3413\n",
      "TOP1 59\n",
      "TOP5 274\n",
      "TOP10 406\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 6s - loss: 0.0379 - val_loss: 0.3983\n",
      "TOP1 47\n",
      "TOP5 249\n",
      "TOP10 428\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 17/17\n",
      "3200/8000 [===========>..................] - ETA: 3s - loss: 0.0387"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 400 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:367: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:368: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # ---- Padding the vector ---- #\n",
      "keras_title.py:372: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:373: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # --- Prepare the training data --- #\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [3311, 4001, 4265, 4888, 4964]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:442: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # model_lstm2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_89 (LSTM)                   (None, 400)           961600      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_90 (LSTM)                   (None, 400)           961600      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_45 (Concatenate)     (None, 800)           0           lstm_89[0][0]                    \n",
      "                                                                   lstm_90[0][0]                    \n",
      "____________________________________________________________________________________________________\n",
      "dense_89 (Dense)                 (None, 400)           320400      concatenate_45[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_89 (Dropout)             (None, 400)           0           dense_89[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_90 (Dense)                 (None, 100)           40100       dropout_89[0][0]                 \n",
      "____________________________________________________________________________________________________\n",
      "dropout_90 (Dropout)             (None, 100)           0           dense_90[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_90[0][0]                 \n",
      "====================================================================================================\n",
      "Total params: 2,283,801\n",
      "Trainable params: 2,283,801\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 12s - loss: 0.5978 - val_loss: 0.5490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:111: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # Compile the model\n",
      "keras_title.py:147: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 15\n",
      "TOP5 100\n",
      "TOP10 165\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 12s - loss: 0.3367 - val_loss: 0.4422\n",
      "TOP1 27\n",
      "TOP5 163\n",
      "TOP10 258\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 12s - loss: 0.2267 - val_loss: 0.3884\n",
      "TOP1 42\n",
      "TOP5 194\n",
      "TOP10 325\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 12s - loss: 0.1460 - val_loss: 0.4236\n",
      "TOP1 34\n",
      "TOP5 221\n",
      "TOP10 368\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 12s - loss: 0.1045 - val_loss: 0.4089\n",
      "TOP1 49\n",
      "TOP5 230\n",
      "TOP10 380\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 12s - loss: 0.0718 - val_loss: 0.4383\n",
      "TOP1 47\n",
      "TOP5 229\n",
      "TOP10 366\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 12s - loss: 0.0486 - val_loss: 0.4192\n",
      "TOP1 53\n",
      "TOP5 244\n",
      "TOP10 389\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 12s - loss: 0.0407 - val_loss: 0.4239\n",
      "TOP1 37\n",
      "TOP5 235\n",
      "TOP10 410\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 12s - loss: 0.0256 - val_loss: 0.4581\n",
      "TOP1 55\n",
      "TOP5 230\n",
      "TOP10 381\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 12s - loss: 0.0224 - val_loss: 0.3794\n",
      "TOP1 42\n",
      "TOP5 216\n",
      "TOP10 380\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 12s - loss: 0.0190 - val_loss: 0.4463\n",
      "TOP1 39\n",
      "TOP5 207\n",
      "TOP10 386\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 12s - loss: 0.0201 - val_loss: 0.4404\n",
      "TOP1 45\n",
      "TOP5 236\n",
      "TOP10 400\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 12s - loss: 0.0239 - val_loss: 0.4047\n",
      "TOP1 29\n",
      "TOP5 188\n",
      "TOP10 358\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "2848/8000 [=========>....................] - ETA: 7s - loss: 0.0357"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    456\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    457\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 458\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    459\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    460\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['sigmoid', 'sigmoid']\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:367: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:368: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # ---- Padding the vector ---- #\n",
      "keras_title.py:372: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:373: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # --- Prepare the training data --- #\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 4 position: [192, 3246, 3400, 4062]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:442: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # model_lstm2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_117 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_118 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_59 (Concatenate)     (None, 400)           0           lstm_117[0][0]                   \n",
      "                                                                   lstm_118[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_117 (Dense)                (None, 400)           160400      concatenate_59[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_117 (Dropout)            (None, 400)           0           dense_117[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_118 (Dense)                (None, 100)           40100       dropout_117[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_118 (Dropout)            (None, 100)           0           dense_118[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_118[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 1.0411 - val_loss: 1.0513\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:111: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # Compile the model\n",
      "keras_title.py:147: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 1\n",
      "TOP5 5\n",
      "TOP10 9\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 1.0045 - val_loss: 1.4071\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    460\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n\u001b[1;32m    461\u001b[0m                                                                  \u001b[0mX2_test_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m                                                                  model_lstm2)\n\u001b[0m\u001b[1;32m    463\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m         \u001b[0;31m# Evaluation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36mfind_ranking_quick\u001b[0;34m(projection1, projection2, dlmodel)\u001b[0m\n\u001b[1;32m    123\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m     \u001b[0;31m# Find the real projection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 125\u001b[0;31m     \u001b[0mv1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mv2\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_1\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mprojection1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojection2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    126\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    127\u001b[0m     \u001b[0;31m# ---- Prepare the model_2 ----\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1589\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1590\u001b[0m             \u001b[0mins\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1591\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_predict_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1592\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1593\u001b[0m         return self._predict_loop(f, ins,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_make_predict_function\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1051\u001b[0m                                                \u001b[0mupdates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstate_updates\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m                                                \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'predict_function'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1053\u001b[0;31m                                                **kwargs)\n\u001b[0m\u001b[1;32m   1054\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1055\u001b[0m     def _fit_loop(self, f, ins, out_labels=None, batch_size=32,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36mfunction\u001b[0;34m(inputs, outputs, updates, **kwargs)\u001b[0m\n\u001b[1;32m   1204\u001b[0m                 \u001b[0mmsg\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'Invalid argument \"%s\" passed to K.function with Theano backend'\u001b[0m \u001b[0;34m%\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1205\u001b[0m                 \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmsg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1206\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0mFunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moutputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mupdates\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mupdates\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1207\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1208\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, updates, name, **kwargs)\u001b[0m\n\u001b[1;32m   1189\u001b[0m                                         \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1190\u001b[0m                                         \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1191\u001b[0;31m                                         **kwargs)\n\u001b[0m\u001b[1;32m   1192\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mname\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function.pyc\u001b[0m in \u001b[0;36mfunction\u001b[0;34m(inputs, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input)\u001b[0m\n\u001b[1;32m    324\u001b[0m                    \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    325\u001b[0m                    \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 326\u001b[0;31m                    output_keys=output_keys)\n\u001b[0m\u001b[1;32m    327\u001b[0m     \u001b[0;31m# We need to add the flag check_aliased inputs if we have any mutable or\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    328\u001b[0m     \u001b[0;31m# borrowed used defined inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/pfunc.pyc\u001b[0m in \u001b[0;36mpfunc\u001b[0;34m(params, outputs, mode, updates, givens, no_default_updates, accept_inplace, name, rebuild_strict, allow_input_downcast, profile, on_unused_input, output_keys)\u001b[0m\n\u001b[1;32m    484\u001b[0m                          \u001b[0maccept_inplace\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0maccept_inplace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mname\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m                          \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 486\u001b[0;31m                          output_keys=output_keys)\n\u001b[0m\u001b[1;32m    487\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    488\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36morig_function\u001b[0;34m(inputs, outputs, mode, accept_inplace, name, profile, on_unused_input, output_keys)\u001b[0m\n\u001b[1;32m   1792\u001b[0m                    \u001b[0mprofile\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mprofile\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1793\u001b[0m                    \u001b[0mon_unused_input\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mon_unused_input\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1794\u001b[0;31m                    \u001b[0moutput_keys\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_keys\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcreate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1795\u001b[0m             defaults)\n\u001b[1;32m   1796\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, inputs, outputs, mode, accept_inplace, function_builder, profile, on_unused_input, fgraph, output_keys)\u001b[0m\n\u001b[1;32m   1472\u001b[0m                         optimizer, inputs, outputs)\n\u001b[1;32m   1473\u001b[0m                 \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1474\u001b[0;31m                     \u001b[0moptimizer_profile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1475\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1476\u001b[0m                 \u001b[0mend_optimizer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m     96\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     97\u001b[0m         \"\"\"\n\u001b[0;32m---> 98\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     99\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    100\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0madd_requirements\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    233\u001b[0m                 \u001b[0mnb_nodes_before\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_nodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0msub_prof\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m                 \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m                 \u001b[0msub_profs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_prof\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    233\u001b[0m                 \u001b[0mnb_nodes_before\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_nodes\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    234\u001b[0m                 \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 235\u001b[0;31m                 \u001b[0msub_prof\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptimize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    236\u001b[0m                 \u001b[0ml\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    237\u001b[0m                 \u001b[0msub_profs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msub_prof\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36moptimize\u001b[0;34m(self, fgraph, *args, **kwargs)\u001b[0m\n\u001b[1;32m     85\u001b[0m             \u001b[0morig\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     86\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 87\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     88\u001b[0m         \u001b[0;32mfinally\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     89\u001b[0m             \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbasic\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconstant\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0menable\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mapply\u001b[0;34m(self, fgraph, start_from)\u001b[0m\n\u001b[1;32m   2468\u001b[0m                         \u001b[0mnb\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mchange_tracker\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnb_imported\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2469\u001b[0m                         \u001b[0mt_opt\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2470\u001b[0;31m                         \u001b[0mlopt_change\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprocess_node\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlopt\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2471\u001b[0m                         \u001b[0mtime_opts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mlopt\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt_opt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2472\u001b[0m                         \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0mlopt_change\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/opt.pyc\u001b[0m in \u001b[0;36mprocess_node\u001b[0;34m(self, fgraph, node, lopt)\u001b[0m\n\u001b[1;32m   2020\u001b[0m             fgraph.replace_all_validate_remove(repl_pairs,\n\u001b[1;32m   2021\u001b[0m                                                \u001b[0mreason\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlopt\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2022\u001b[0;31m                                                remove=remove)\n\u001b[0m\u001b[1;32m   2023\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2024\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/toolbox.pyc\u001b[0m in \u001b[0;36mreplace_all_validate_remove\u001b[0;34m(self, fgraph, replacements, remove, reason, warn)\u001b[0m\n\u001b[1;32m    389\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    390\u001b[0m         \"\"\"\n\u001b[0;32m--> 391\u001b[0;31m         \u001b[0mchk\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace_all_validate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mreplacements\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreason\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    392\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_nodes_removed\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mremove\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    393\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mrm\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mremove\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/toolbox.pyc\u001b[0m in \u001b[0;36mreplace_all_validate\u001b[0;34m(self, fgraph, replacements, reason, verbose)\u001b[0m\n\u001b[1;32m    363\u001b[0m                 \u001b[0;32mraise\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    364\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 365\u001b[0;31m             \u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalidate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    366\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    367\u001b[0m             \u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrevert\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mchk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/toolbox.pyc\u001b[0m in \u001b[0;36mvalidate_\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    254\u001b[0m         \u001b[0mt0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    255\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 256\u001b[0;31m             \u001b[0mret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_callbacks\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'validate'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    257\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    258\u001b[0m             \u001b[0mcf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minspect\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcurrentframe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/fg.pyc\u001b[0m in \u001b[0;36mexecute_callbacks\u001b[0;34m(self, name, *args, **kwargs)\u001b[0m\n\u001b[1;32m    587\u001b[0m                 \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    588\u001b[0m             \u001b[0mtf0\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 589\u001b[0;31m             \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    590\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_callbacks_times\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mfeature\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mtf0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    591\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute_callbacks_time\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mtime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0mt0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36mvalidate\u001b[0;34m(self, fgraph)\u001b[0m\n\u001b[1;32m    137\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    138\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mr\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotected\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 139\u001b[0;31m             \u001b[0;32mif\u001b[0m \u001b[0mfgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdestroyers\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    140\u001b[0m                 raise gof.InconsistencyError(\"Trying to destroy a protected\"\n\u001b[1;32m    141\u001b[0m                                              \"Variable.\", r)\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:376: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_en_1 = np.stack(df_pairs_sample[\"padding_en\"].values)\n",
      "keras_title.py:377: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_jp_1 = np.stack(df_pairs_sample[\"padding_jp\"].values)\n",
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [2490, 2765, 3386, 3433, 3455, 4339, 4749]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:461: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # Evaluation\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_123 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_124 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_61 (Concatenate)     (None, 400)           0           lstm_123[0][0]                   \n",
      "                                                                   lstm_124[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_121 (Dense)                (None, 400)           160400      concatenate_61[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_121 (Dropout)            (None, 400)           0           dense_121[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_122 (Dense)                (None, 100)           40100       dropout_121[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_122 (Dropout)            (None, 100)           0           dense_122[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_122[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.1564 - val_loss: 0.1325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:112: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  model_1.compile(optimizer='adam', loss='binary_crossentropy', metrics=['accuracy'])\n",
      "keras_title.py:154: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  v3 = model_2.predict([v1, v2])\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 18\n",
      "TOP5 100\n",
      "TOP10 187\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.0775 - val_loss: 0.1348\n",
      "TOP1 29\n",
      "TOP5 183\n",
      "TOP10 321\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.0483 - val_loss: 0.1175\n",
      "TOP1 46\n",
      "TOP5 210\n",
      "TOP10 329\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.0318 - val_loss: 0.1242\n",
      "TOP1 42\n",
      "TOP5 221\n",
      "TOP10 370\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.0243 - val_loss: 0.1035\n",
      "TOP1 51\n",
      "TOP5 265\n",
      "TOP10 441\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.0141 - val_loss: 0.1118\n",
      "TOP1 45\n",
      "TOP5 271\n",
      "TOP10 439\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.0089 - val_loss: 0.1184\n",
      "TOP1 47\n",
      "TOP5 270\n",
      "TOP10 446\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0072 - val_loss: 0.1062\n",
      "TOP1 63\n",
      "TOP5 277\n",
      "TOP10 449\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0071 - val_loss: 0.1194\n",
      "TOP1 48\n",
      "TOP5 284\n",
      "TOP10 436\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0054 - val_loss: 0.1072\n",
      "TOP1 51\n",
      "TOP5 262\n",
      "TOP10 448\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 6s - loss: 0.0096 - val_loss: 0.1107\n",
      "TOP1 52\n",
      "TOP5 266\n",
      "TOP10 435\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0086 - val_loss: 0.1447\n",
      "TOP1 42\n",
      "TOP5 218\n",
      "TOP10 393\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 6s - loss: 0.0130 - val_loss: 0.1315\n",
      "TOP1 14\n",
      "TOP5 138\n",
      "TOP10 247\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 6s - loss: 0.0104 - val_loss: 0.1201\n",
      "TOP1 28\n",
      "TOP5 177\n",
      "TOP10 301\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 6s - loss: 0.0060 - val_loss: 0.1236\n",
      "TOP1 21\n",
      "TOP5 125\n",
      "TOP10 262\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "4640/8000 [================>.............] - ETA: 2s - loss: 0.0055"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    475\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    476\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 477\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    478\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    479\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:378: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:379: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Generate training data (similarity = 0)\n",
      "keras_title.py:383: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # check the duplicated amount\n",
      "keras_title.py:384: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  c = np.all(features_en_1 == features_en_0, axis=(1,2))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 3 position: [1784, 1991, 4304]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:463: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_155 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_156 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_77 (Concatenate)     (None, 400)           0           lstm_155[0][0]                   \n",
      "                                                                   lstm_156[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_153 (Dense)                (None, 400)           160400      concatenate_77[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_153 (Dropout)            (None, 400)           0           dense_153[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_154 (Dense)                (None, 100)           40100       dropout_153[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_154 (Dropout)            (None, 100)           0           dense_154[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_154[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.6118 - val_loss: 0.4711\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:114: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:156: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  # Set the weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 22\n",
      "TOP5 108\n",
      "TOP10 215\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.3135 - val_loss: 0.4040\n",
      "TOP1 27\n",
      "TOP5 210\n",
      "TOP10 354\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.2022 - val_loss: 0.4102\n",
      "TOP1 34\n",
      "TOP5 218\n",
      "TOP10 352\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.1413 - val_loss: 0.3521\n",
      "TOP1 52\n",
      "TOP5 250\n",
      "TOP10 404\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.0889 - val_loss: 0.3586\n",
      "TOP1 61\n",
      "TOP5 279\n",
      "TOP10 450\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.0564 - val_loss: 0.3365\n",
      "TOP1 63\n",
      "TOP5 286\n",
      "TOP10 469\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.0374 - val_loss: 0.3358\n",
      "TOP1 53\n",
      "TOP5 238\n",
      "TOP10 396\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0289 - val_loss: 0.3520\n",
      "TOP1 53\n",
      "TOP5 225\n",
      "TOP10 396\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0363 - val_loss: 0.3918\n",
      "TOP1 32\n",
      "TOP5 210\n",
      "TOP10 378\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0467 - val_loss: 0.3667\n",
      "TOP1 26\n",
      "TOP5 152\n",
      "TOP10 282\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 6s - loss: 0.0409 - val_loss: 0.3680\n",
      "TOP1 24\n",
      "TOP5 152\n",
      "TOP10 299\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0423 - val_loss: 0.3423\n",
      "TOP1 26\n",
      "TOP5 151\n",
      "TOP10 310\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 6s - loss: 0.0302 - val_loss: 0.4088\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    481\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n\u001b[1;32m    482\u001b[0m                                                                  \u001b[0mX2_test_1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 483\u001b[0;31m                                                                  model_lstm2)\n\u001b[0m\u001b[1;32m    484\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    485\u001b[0m         \u001b[0;31m# Evaluation\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36mfind_ranking_quick\u001b[0;34m(projection1, projection2, dlmodel)\u001b[0m\n\u001b[1;32m    181\u001b[0m         \u001b[0;31m# print(\"Find answer for doc.\", i)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    182\u001b[0m         \u001b[0mproj1_tile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mproj1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprojection2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 183\u001b[0;31m         \u001b[0msim\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel_2\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mproj1_tile\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mprojection2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    184\u001b[0m         \u001b[0mrank\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mSeries\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrank\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mascending\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    185\u001b[0m         \u001b[0msim_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msim\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mpredict\u001b[0;34m(self, x, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1592\u001b[0m         \u001b[0mf\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict_function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1593\u001b[0m         return self._predict_loop(f, ins,\n\u001b[0;32m-> 1594\u001b[0;31m                                   batch_size=batch_size, verbose=verbose)\n\u001b[0m\u001b[1;32m   1595\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1596\u001b[0m     def train_on_batch(self, x, y,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_predict_loop\u001b[0;34m(self, f, ins, batch_size, verbose)\u001b[0m\n\u001b[1;32m   1216\u001b[0m                 \u001b[0mins_batch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_slice_arrays\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1217\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1218\u001b[0;31m             \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1219\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1220\u001b[0m                 \u001b[0mbatch_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mbatch_outs\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.2\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:378: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:379: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Generate training data (similarity = 0)\n",
      "keras_title.py:383: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # check the duplicated amount\n",
      "keras_title.py:384: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  c = np.all(features_en_1 == features_en_0, axis=(1,2))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [1902, 2270, 3928, 4069, 4224, 4260, 4282]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:463: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_183 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_184 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_91 (Concatenate)     (None, 400)           0           lstm_183[0][0]                   \n",
      "                                                                   lstm_184[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_181 (Dense)                (None, 400)           160400      concatenate_91[0][0]             \n",
      "____________________________________________________________________________________________________\n",
      "dropout_181 (Dropout)            (None, 400)           0           dense_181[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_182 (Dense)                (None, 100)           40100       dropout_181[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_182 (Dropout)            (None, 100)           0           dense_182[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_182[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.6530 - val_loss: 0.5346\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:114: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:156: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  # Set the weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 25\n",
      "TOP5 142\n",
      "TOP10 227\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.3712 - val_loss: 0.4770\n",
      "TOP1 34\n",
      "TOP5 193\n",
      "TOP10 311\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.2712 - val_loss: 0.4366\n",
      "TOP1 29\n",
      "TOP5 209\n",
      "TOP10 352\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.2023 - val_loss: 0.4371\n",
      "TOP1 57\n",
      "TOP5 255\n",
      "TOP10 396\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.1544 - val_loss: 0.4079\n",
      "TOP1 51\n",
      "TOP5 254\n",
      "TOP10 405\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.1154 - val_loss: 0.4105\n",
      "TOP1 52\n",
      "TOP5 268\n",
      "TOP10 413\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.0859 - val_loss: 0.3657\n",
      "TOP1 51\n",
      "TOP5 255\n",
      "TOP10 400\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0796 - val_loss: 0.4411\n",
      "TOP1 55\n",
      "TOP5 247\n",
      "TOP10 389\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0673 - val_loss: 0.4173\n",
      "TOP1 52\n",
      "TOP5 244\n",
      "TOP10 401\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0560 - val_loss: 0.4243\n",
      "TOP1 60\n",
      "TOP5 251\n",
      "TOP10 426\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 6s - loss: 0.0409 - val_loss: 0.4686\n",
      "TOP1 65\n",
      "TOP5 262\n",
      "TOP10 414\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0353 - val_loss: 0.4123\n",
      "TOP1 50\n",
      "TOP5 249\n",
      "TOP10 409\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 6s - loss: 0.0453 - val_loss: 0.4673\n",
      "TOP1 50\n",
      "TOP5 232\n",
      "TOP10 389\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "5120/8000 [==================>...........] - ETA: 2s - loss: 0.0420"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    477\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    478\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 479\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    480\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 20 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:378: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:379: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Generate training data (similarity = 0)\n",
      "keras_title.py:383: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # check the duplicated amount\n",
      "keras_title.py:384: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  c = np.all(features_en_1 == features_en_0, axis=(1,2))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 11 position: [1033, 1088, 1474, 1727, 1771, 2104, 3364, 3411, 3429, 4311, 4781]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:463: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_211 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_212 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_105 (Concatenate)    (None, 400)           0           lstm_211[0][0]                   \n",
      "                                                                   lstm_212[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_209 (Dense)                (None, 400)           160400      concatenate_105[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_209 (Dropout)            (None, 400)           0           dense_209[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_210 (Dense)                (None, 100)           40100       dropout_209[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_210 (Dropout)            (None, 100)           0           dense_210[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_210[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.5674 - val_loss: 0.4937\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:114: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:156: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  # Set the weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 26\n",
      "TOP5 132\n",
      "TOP10 263\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.2923 - val_loss: 0.4278\n",
      "TOP1 52\n",
      "TOP5 222\n",
      "TOP10 344\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.1922 - val_loss: 0.4376\n",
      "TOP1 53\n",
      "TOP5 233\n",
      "TOP10 372\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.1387 - val_loss: 0.4168\n",
      "TOP1 43\n",
      "TOP5 227\n",
      "TOP10 376\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.0956 - val_loss: 0.4114\n",
      "TOP1 54\n",
      "TOP5 241\n",
      "TOP10 387\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.0698 - val_loss: 0.4133\n",
      "TOP1 50\n",
      "TOP5 261\n",
      "TOP10 393\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.0511 - val_loss: 0.3932\n",
      "TOP1 58\n",
      "TOP5 287\n",
      "TOP10 438\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0340 - val_loss: 0.3962\n",
      "TOP1 63\n",
      "TOP5 299\n",
      "TOP10 441\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0296 - val_loss: 0.4326\n",
      "TOP1 57\n",
      "TOP5 268\n",
      "TOP10 410\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0322 - val_loss: 0.4443\n",
      "TOP1 63\n",
      "TOP5 259\n",
      "TOP10 409\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 11/11\n",
      "8000/8000 [==============================] - 6s - loss: 0.0287 - val_loss: 0.4806\n",
      "TOP1 63\n",
      "TOP5 266\n",
      "TOP10 412\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 12/12\n",
      "8000/8000 [==============================] - 6s - loss: 0.0340 - val_loss: 0.4287\n",
      "TOP1 56\n",
      "TOP5 259\n",
      "TOP10 420\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 13/13\n",
      "8000/8000 [==============================] - 6s - loss: 0.0322 - val_loss: 0.4628\n",
      "TOP1 50\n",
      "TOP5 247\n",
      "TOP10 389\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 14/14\n",
      "8000/8000 [==============================] - 6s - loss: 0.0281 - val_loss: 0.4209\n",
      "TOP1 45\n",
      "TOP5 223\n",
      "TOP10 376\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 15/15\n",
      "8000/8000 [==============================] - 6s - loss: 0.0198 - val_loss: 0.4559\n",
      "TOP1 49\n",
      "TOP5 269\n",
      "TOP10 417\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 16/16\n",
      "8000/8000 [==============================] - 6s - loss: 0.0189 - val_loss: 0.4559\n",
      "TOP1 43\n",
      "TOP5 248\n",
      "TOP10 390\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 17/17\n",
      "8000/8000 [==============================] - 6s - loss: 0.0193 - val_loss: 0.4437\n",
      "TOP1 32\n",
      "TOP5 237\n",
      "TOP10 379\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 18/18\n",
      "8000/8000 [==============================] - 6s - loss: 0.0179 - val_loss: 0.4238\n",
      "TOP1 36\n",
      "TOP5 205\n",
      "TOP10 349\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 19/19\n",
      "8000/8000 [==============================] - 6s - loss: 0.0137 - val_loss: 0.4865\n",
      "TOP1 38\n",
      "TOP5 200\n",
      "TOP10 349\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 20/20\n",
      "8000/8000 [==============================] - 6s - loss: 0.0181 - val_loss: 0.4721\n",
      "TOP1 27\n",
      "TOP5 216\n",
      "TOP10 375\n",
      "TOP1 27\n",
      "TOP5 216\n",
      "TOP10 375\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:378: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:379: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Generate training data (similarity = 0)\n",
      "keras_title.py:383: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # check the duplicated amount\n",
      "keras_title.py:384: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  c = np.all(features_en_1 == features_en_0, axis=(1,2))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 4 position: [1171, 2254, 2613, 3694]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:463: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  print \"TOP5\", (pd.Series(rank_results_test) <= 5).sum()\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_255 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_256 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_127 (Concatenate)    (None, 400)           0           lstm_255[0][0]                   \n",
      "                                                                   lstm_256[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_253 (Dense)                (None, 400)           160400      concatenate_127[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_253 (Dropout)            (None, 400)           0           dense_253[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_254 (Dense)                (None, 100)           40100       dropout_253[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_254 (Dropout)            (None, 100)           0           dense_254[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_254[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.5953 - val_loss: 0.4599\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:114: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:156: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  # Set the weights\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 24\n",
      "TOP5 156\n",
      "TOP10 278\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.2963 - val_loss: 0.4100\n",
      "TOP1 54\n",
      "TOP5 198\n",
      "TOP10 345\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.1885 - val_loss: 0.3468\n",
      "TOP1 39\n",
      "TOP5 254\n",
      "TOP10 418\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1192 - val_loss: 0.3351\n",
      "TOP1 51\n",
      "TOP5 260\n",
      "TOP10 419\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0815 - val_loss: 0.3702\n",
      "TOP1 62\n",
      "TOP5 278\n",
      "TOP10 455\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0505 - val_loss: 0.4121\n",
      "TOP1 67\n",
      "TOP5 261\n",
      "TOP10 406\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0333 - val_loss: 0.3952\n",
      "TOP1 50\n",
      "TOP5 274\n",
      "TOP10 429\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0268 - val_loss: 0.3815\n",
      "TOP1 58\n",
      "TOP5 276\n",
      "TOP10 434\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0298 - val_loss: 0.4033\n",
      "TOP1 41\n",
      "TOP5 229\n",
      "TOP10 392\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0466 - val_loss: 0.4381\n",
      "TOP1 33\n",
      "TOP5 207\n",
      "TOP10 335\n",
      "TOP1 33\n",
      "TOP5 207\n",
      "TOP10 335\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function mae\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:380: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_en_0 = np.array(features_en_1)\n",
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:385: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  print \"C value =\", c.sum(), \"position:\", np.where(c== True)[0].tolist()\n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 4 position: [700, 1195, 1966, 3409]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:465: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_279 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_280 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_139 (Concatenate)    (None, 400)           0           lstm_279[0][0]                   \n",
      "                                                                   lstm_280[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_277 (Dense)                (None, 400)           160400      concatenate_139[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_277 (Dropout)            (None, 400)           0           dense_277[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_278 (Dense)                (None, 100)           40100       dropout_277[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_278 (Dropout)            (None, 100)           0           dense_278[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_278[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.6385 - val_loss: 0.5542\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:116: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # print model_1.summary()\n",
      "keras_title.py:158: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[5].set_weights(dlmodel.layers[7].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 14\n",
      "TOP5 79\n",
      "TOP10 175\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.4048 - val_loss: 0.4460\n",
      "TOP1 24\n",
      "TOP5 128\n",
      "TOP10 244\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.2997 - val_loss: 0.3881\n",
      "TOP1 28\n",
      "TOP5 139\n",
      "TOP10 263\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.2296 - val_loss: 0.3409\n",
      "TOP1 27\n",
      "TOP5 181\n",
      "TOP10 306\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.1653 - val_loss: 0.3304\n",
      "TOP1 34\n",
      "TOP5 159\n",
      "TOP10 300\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.1313 - val_loss: 0.3527\n",
      "TOP1 22\n",
      "TOP5 119\n",
      "TOP10 262\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.1082 - val_loss: 0.3280\n",
      "TOP1 14\n",
      "TOP5 123\n",
      "TOP10 252\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0851 - val_loss: 0.3041\n",
      "TOP1 21\n",
      "TOP5 118\n",
      "TOP10 235\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0733 - val_loss: 0.3093\n",
      "TOP1 15\n",
      "TOP5 136\n",
      "TOP10 284\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0661 - val_loss: 0.3127\n",
      "TOP1 19\n",
      "TOP5 156\n",
      "TOP10 284\n",
      "TOP1 19\n",
      "TOP5 156\n",
      "TOP10 284\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: mape\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:380: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_en_0 = np.array(features_en_1)\n",
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:385: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  print \"C value =\", c.sum(), \"position:\", np.where(c== True)[0].tolist()\n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [689, 1012, 2135, 3360, 3679]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:465: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_303 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_304 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_151 (Concatenate)    (None, 400)           0           lstm_303[0][0]                   \n",
      "                                                                   lstm_304[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_301 (Dense)                (None, 400)           160400      concatenate_151[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_301 (Dropout)            (None, 400)           0           dense_301[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_302 (Dense)                (None, 100)           40100       dropout_301[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_302 (Dropout)            (None, 100)           0           dense_302[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_302[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 62.4895 - val_loss: 56.7956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:116: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # print model_1.summary()\n",
      "keras_title.py:158: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[5].set_weights(dlmodel.layers[7].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 13\n",
      "TOP5 71\n",
      "TOP10 133\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 39.0212 - val_loss: 47.6958\n",
      "TOP1 19\n",
      "TOP5 109\n",
      "TOP10 205\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 28.8734 - val_loss: 39.1982\n",
      "TOP1 21\n",
      "TOP5 114\n",
      "TOP10 218\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 22.3215 - val_loss: 38.5172\n",
      "TOP1 28\n",
      "TOP5 150\n",
      "TOP10 253\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 16.9896 - val_loss: 33.3989\n",
      "TOP1 30\n",
      "TOP5 162\n",
      "TOP10 287\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 13.3258 - val_loss: 36.0581\n",
      "TOP1 20\n",
      "TOP5 126\n",
      "TOP10 239\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 10.2630 - val_loss: 34.0624\n",
      "TOP1 28\n",
      "TOP5 135\n",
      "TOP10 243\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 8.2817 - val_loss: 32.3967\n",
      "TOP1 22\n",
      "TOP5 113\n",
      "TOP10 239\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 7.7660 - val_loss: 32.1305\n",
      "TOP1 19\n",
      "TOP5 122\n",
      "TOP10 238\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 6.1904 - val_loss: 35.0883\n",
      "TOP1 23\n",
      "TOP5 127\n",
      "TOP10 250\n",
      "TOP1 23\n",
      "TOP5 127\n",
      "TOP10 250\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: msle\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:380: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_en_0 = np.array(features_en_1)\n",
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:385: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  print \"C value =\", c.sum(), \"position:\", np.where(c== True)[0].tolist()\n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [445, 3375, 3927, 4261, 4292]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:465: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_327 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_328 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_163 (Concatenate)    (None, 400)           0           lstm_327[0][0]                   \n",
      "                                                                   lstm_328[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_325 (Dense)                (None, 400)           160400      concatenate_163[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_325 (Dropout)            (None, 400)           0           dense_325[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_326 (Dense)                (None, 100)           40100       dropout_325[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_326 (Dropout)            (None, 100)           0           dense_326[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_326[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 5s - loss: 0.0766 - val_loss: 0.0610\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:116: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # print model_1.summary()\n",
      "keras_title.py:158: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[5].set_weights(dlmodel.layers[7].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 28\n",
      "TOP5 157\n",
      "TOP10 254\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.0357 - val_loss: 0.0524\n",
      "TOP1 41\n",
      "TOP5 209\n",
      "TOP10 353\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.0229 - val_loss: 0.0480\n",
      "TOP1 48\n",
      "TOP5 224\n",
      "TOP10 347\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.0155 - val_loss: 0.0500\n",
      "TOP1 43\n",
      "TOP5 226\n",
      "TOP10 358\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0117 - val_loss: 0.0499\n",
      "TOP1 44\n",
      "TOP5 233\n",
      "TOP10 386\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0100 - val_loss: 0.0494\n",
      "TOP1 52\n",
      "TOP5 238\n",
      "TOP10 375\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0072 - val_loss: 0.0560\n",
      "TOP1 37\n",
      "TOP5 214\n",
      "TOP10 381\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0069 - val_loss: 0.0551\n",
      "TOP1 27\n",
      "TOP5 191\n",
      "TOP10 323\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0069 - val_loss: 0.0567\n",
      "TOP1 31\n",
      "TOP5 198\n",
      "TOP10 336\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0082 - val_loss: 0.0722\n",
      "TOP1 38\n",
      "TOP5 194\n",
      "TOP10 312\n",
      "TOP1 38\n",
      "TOP5 194\n",
      "TOP10 312\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: squared_hinge\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:380: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_en_0 = np.array(features_en_1)\n",
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:385: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  print \"C value =\", c.sum(), \"position:\", np.where(c== True)[0].tolist()\n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [993, 1033, 1405, 3431, 4068]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:465: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_351 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_352 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_175 (Concatenate)    (None, 400)           0           lstm_351[0][0]                   \n",
      "                                                                   lstm_352[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_349 (Dense)                (None, 400)           160400      concatenate_175[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_349 (Dropout)            (None, 400)           0           dense_349[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_350 (Dense)                (None, 100)           40100       dropout_349[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_350 (Dropout)            (None, 100)           0           dense_350[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_350[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.5970 - val_loss: 0.4742\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:116: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # print model_1.summary()\n",
      "keras_title.py:158: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[5].set_weights(dlmodel.layers[7].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 17\n",
      "TOP5 92\n",
      "TOP10 214\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.3018 - val_loss: 0.5024\n",
      "TOP1 28\n",
      "TOP5 185\n",
      "TOP10 323\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.1966 - val_loss: 0.4943\n",
      "TOP1 28\n",
      "TOP5 179\n",
      "TOP10 301\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1379 - val_loss: 0.4641\n",
      "TOP1 34\n",
      "TOP5 192\n",
      "TOP10 345\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.1100 - val_loss: 0.5129\n",
      "TOP1 27\n",
      "TOP5 196\n",
      "TOP10 325\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0773 - val_loss: 0.4459\n",
      "TOP1 26\n",
      "TOP5 208\n",
      "TOP10 351\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0714 - val_loss: 0.5221\n",
      "TOP1 38\n",
      "TOP5 212\n",
      "TOP10 375\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0552 - val_loss: 0.5450\n",
      "TOP1 28\n",
      "TOP5 213\n",
      "TOP10 345\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0422 - val_loss: 0.6546\n",
      "TOP1 39\n",
      "TOP5 216\n",
      "TOP10 356\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0353 - val_loss: 0.6507\n",
      "TOP1 42\n",
      "TOP5 212\n",
      "TOP10 347\n",
      "TOP1 42\n",
      "TOP5 212\n",
      "TOP10 347\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: hinge\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:380: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  features_en_0 = np.array(features_en_1)\n",
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:385: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  print \"C value =\", c.sum(), \"position:\", np.where(c== True)[0].tolist()\n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 5 position: [1015, 1514, 3367, 3523, 4273]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:465: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_375 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_376 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_187 (Concatenate)    (None, 400)           0           lstm_375[0][0]                   \n",
      "                                                                   lstm_376[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_373 (Dense)                (None, 400)           160400      concatenate_187[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_373 (Dropout)            (None, 400)           0           dense_373[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_374 (Dense)                (None, 100)           40100       dropout_373[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_374 (Dropout)            (None, 100)           0           dense_374[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_374[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.5378 - val_loss: 0.4085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:116: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  # print model_1.summary()\n",
      "keras_title.py:158: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[5].set_weights(dlmodel.layers[7].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 15\n",
      "TOP5 69\n",
      "TOP10 139\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.2834 - val_loss: 0.4083\n",
      "TOP1 15\n",
      "TOP5 100\n",
      "TOP10 185\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.2103 - val_loss: 0.3325\n",
      "TOP1 16\n",
      "TOP5 103\n",
      "TOP10 227\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1532 - val_loss: 0.3295\n",
      "TOP1 26\n",
      "TOP5 142\n",
      "TOP10 225\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.1209 - val_loss: 0.3904\n",
      "TOP1 25\n",
      "TOP5 149\n",
      "TOP10 234\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 5s - loss: 0.1060 - val_loss: 0.3527\n",
      "TOP1 28\n",
      "TOP5 163\n",
      "TOP10 251\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0837 - val_loss: 0.4181\n",
      "TOP1 31\n",
      "TOP5 170\n",
      "TOP10 296\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0773 - val_loss: 0.4182\n",
      "TOP1 22\n",
      "TOP5 148\n",
      "TOP10 272\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "2912/8000 [=========>....................] - ETA: 2s - loss: 0.0514"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    479\u001b[0m                            \u001b[0mvalidation_data\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mX1_test\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mX2_test\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my_test\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    480\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 481\u001b[0;31m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    482\u001b[0m                            initial_epoch=i)\n\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    992\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mallow_gc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 993\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfree\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    994\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    995\u001b[0m         \u001b[0mrval\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0minputs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnode_input_storage\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36mfree\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    997\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstorage_map\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    998\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mConstant\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 999\u001b[0;31m                     \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstorage_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1000\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1001\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mnode\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnodes_with_inner_function\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: 0\n",
      "loss_function: binary_crossentropy\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 3 position: [3368, 3390, 3399]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:466: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # # Save the history and the model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_393 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_394 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_196 (Concatenate)    (None, 400)           0           lstm_393[0][0]                   \n",
      "                                                                   lstm_394[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_391 (Dense)                (None, 400)           160400      concatenate_196[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_391 (Dropout)            (None, 400)           0           dense_391[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_392 (Dense)                (None, 100)           40100       dropout_391[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_392 (Dropout)            (None, 100)           0           dense_392[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_392[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.6299 - val_loss: 0.5929\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:159: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[7].set_weights(dlmodel.layers[9].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 7\n",
      "TOP5 43\n",
      "TOP10 101\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.5317 - val_loss: 0.8467\n",
      "TOP1 16\n",
      "TOP5 85\n",
      "TOP10 161\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.5861 - val_loss: 0.6428\n",
      "TOP1 5\n",
      "TOP5 45\n",
      "TOP10 86\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 1.0363 - val_loss: 2.6850\n",
      "TOP1 0\n",
      "TOP5 8\n",
      "TOP10 12\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.9057 - val_loss: 0.5675\n",
      "TOP1 7\n",
      "TOP5 37\n",
      "TOP10 76\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.6224 - val_loss: 0.6143\n",
      "TOP1 4\n",
      "TOP5 43\n",
      "TOP10 82\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "4640/8000 [================>.............] - ETA: 1s - loss: 0.3873"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    480\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n, allow_gc)\u001b[0m\n\u001b[1;32m    987\u001b[0m         def rval(p=p, i=node_input_storage, o=node_output_storage, n=node,\n\u001b[1;32m    988\u001b[0m                  allow_gc=allow_gc):\n\u001b[0;32m--> 989\u001b[0;31m             \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    990\u001b[0m             \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    991\u001b[0m                 \u001b[0mcompute_map\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mo\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTrue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/scan_module/scan_op.pyc\u001b[0m in \u001b[0;36mp\u001b[0;34m(node, args, outs)\u001b[0m\n\u001b[1;32m    976\u001b[0m                                                 \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    977\u001b[0m                                                 \u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 978\u001b[0;31m                                                 self, node)\n\u001b[0m\u001b[1;32m    979\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mImportError\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtheano\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgof\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcmodule\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mMissingGXX\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    980\u001b[0m             \u001b[0mp\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: 0\n",
      "loss_function: kld\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [691, 2603, 3367, 3397, 3431, 4264, 4474]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:466: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # # Save the history and the model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_413 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_414 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_206 (Concatenate)    (None, 400)           0           lstm_413[0][0]                   \n",
      "                                                                   lstm_414[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_411 (Dense)                (None, 400)           160400      concatenate_206[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_411 (Dropout)            (None, 400)           0           dense_411[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_412 (Dense)                (None, 100)           40100       dropout_411[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_412 (Dropout)            (None, 100)           0           dense_412[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_412[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.0231 - val_loss: -8.0590e-07\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:159: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[7].set_weights(dlmodel.layers[9].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 1\n",
      "TOP5 5\n",
      "TOP10 9\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "7328/8000 [==========================>...] - ETA: 0s - loss: -8.0634e-07"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m/home/M2015eliu/cas/2017.1.1~LiuSTM/keras_title.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m    480\u001b[0m                            \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mi\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    481\u001b[0m                            \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 482\u001b[0;31m                            initial_epoch=i)\n\u001b[0m\u001b[1;32m    483\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    484\u001b[0m         sim_results_test, rank_results_test = find_ranking_quick(X1_test_1,\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, **kwargs)\u001b[0m\n\u001b[1;32m   1505\u001b[0m                               \u001b[0mval_f\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_f\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_ins\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mval_ins\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mshuffle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1506\u001b[0m                               \u001b[0mcallback_metrics\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcallback_metrics\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1507\u001b[0;31m                               initial_epoch=initial_epoch)\n\u001b[0m\u001b[1;32m   1508\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mevaluate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m32\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msample_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/engine/training.pyc\u001b[0m in \u001b[0;36m_fit_loop\u001b[0;34m(self, f, ins, out_labels, batch_size, epochs, verbose, callbacks, val_f, val_ins, shuffle, callback_metrics, initial_epoch)\u001b[0m\n\u001b[1;32m   1154\u001b[0m                 \u001b[0mbatch_logs\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'size'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1155\u001b[0m                 \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_index\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_logs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1156\u001b[0;31m                 \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mins_batch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1157\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1158\u001b[0m                     \u001b[0mouts\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mouts\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/keras/backend/theano_backend.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, inputs)\u001b[0m\n\u001b[1;32m   1194\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__call__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0misinstance\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mlist\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1196\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1197\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1198\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/compile/function_module.pyc\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    882\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    883\u001b[0m             \u001b[0moutputs\u001b[0m \u001b[0;34m=\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 884\u001b[0;31m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0moutput_subset\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    885\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0moutput_subset\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    886\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mException\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/M2015eliu/.pyenv/versions/anaconda2-4.1.1/lib/python2.7/site-packages/theano/gof/op.pyc\u001b[0m in \u001b[0;36mrval\u001b[0;34m(p, i, o, n)\u001b[0m\n\u001b[1;32m    869\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mparams\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mgraph\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mNoParams\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    870\u001b[0m             \u001b[0;31m# default arguments are stored in the closure of `rval`\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 871\u001b[0;31m             \u001b[0;32mdef\u001b[0m \u001b[0mrval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mp\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode_input_storage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode_output_storage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mn\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mnode\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    872\u001b[0m                 \u001b[0mr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mp\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mo\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    873\u001b[0m                 \u001b[0;32mfor\u001b[0m \u001b[0mo\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mnode\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutputs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: 0\n",
      "loss_function: mae\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 4 position: [2111, 2425, 3932, 4156]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:466: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # # Save the history and the model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_417 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_418 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_208 (Concatenate)    (None, 400)           0           lstm_417[0][0]                   \n",
      "                                                                   lstm_418[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_415 (Dense)                (None, 400)           160400      concatenate_208[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_415 (Dropout)            (None, 400)           0           dense_415[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_416 (Dense)                (None, 100)           40100       dropout_415[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_416 (Dropout)            (None, 100)           0           dense_416[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_416[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.3216 - val_loss: 0.3036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:159: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[7].set_weights(dlmodel.layers[9].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 12\n",
      "TOP5 74\n",
      "TOP10 136\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.1949 - val_loss: 0.2341\n",
      "TOP1 27\n",
      "TOP5 125\n",
      "TOP10 212\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.1424 - val_loss: 0.1864\n",
      "TOP1 35\n",
      "TOP5 180\n",
      "TOP10 313\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1052 - val_loss: 0.1869\n",
      "TOP1 37\n",
      "TOP5 176\n",
      "TOP10 290\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0748 - val_loss: 0.1682\n",
      "TOP1 24\n",
      "TOP5 175\n",
      "TOP10 264\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0510 - val_loss: 0.1481\n",
      "TOP1 29\n",
      "TOP5 179\n",
      "TOP10 337\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0356 - val_loss: 0.1629\n",
      "TOP1 20\n",
      "TOP5 142\n",
      "TOP10 293\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0275 - val_loss: 0.1593\n",
      "TOP1 16\n",
      "TOP5 122\n",
      "TOP10 251\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0234 - val_loss: 0.1568\n",
      "TOP1 17\n",
      "TOP5 130\n",
      "TOP10 274\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0192 - val_loss: 0.1752\n",
      "TOP1 13\n",
      "TOP5 135\n",
      "TOP10 273\n",
      "TOP1 13\n",
      "TOP5 135\n",
      "TOP10 273\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: 0\n",
      "loss_function: mse\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 7 position: [439, 1288, 3106, 3391, 3992, 4309, 4892]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:466: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # # Save the history and the model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_441 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_442 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_220 (Concatenate)    (None, 400)           0           lstm_441[0][0]                   \n",
      "                                                                   lstm_442[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_439 (Dense)                (None, 400)           160400      concatenate_220[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_439 (Dropout)            (None, 400)           0           dense_439[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_440 (Dense)                (None, 100)           40100       dropout_439[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_440 (Dropout)            (None, 100)           0           dense_440[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_440[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.1549 - val_loss: 0.1366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:159: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[7].set_weights(dlmodel.layers[9].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 30\n",
      "TOP5 137\n",
      "TOP10 239\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.0720 - val_loss: 0.1271\n",
      "TOP1 33\n",
      "TOP5 191\n",
      "TOP10 305\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.0463 - val_loss: 0.1153\n",
      "TOP1 52\n",
      "TOP5 218\n",
      "TOP10 370\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.0297 - val_loss: 0.1125\n",
      "TOP1 36\n",
      "TOP5 231\n",
      "TOP10 378\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0205 - val_loss: 0.1191\n",
      "TOP1 48\n",
      "TOP5 248\n",
      "TOP10 395\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0144 - val_loss: 0.1170\n",
      "TOP1 40\n",
      "TOP5 244\n",
      "TOP10 393\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0109 - val_loss: 0.1090\n",
      "TOP1 54\n",
      "TOP5 263\n",
      "TOP10 422\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0067 - val_loss: 0.1162\n",
      "TOP1 62\n",
      "TOP5 250\n",
      "TOP10 399\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0057 - val_loss: 0.1073\n",
      "TOP1 63\n",
      "TOP5 260\n",
      "TOP10 415\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 5s - loss: 0.0074 - val_loss: 0.1130\n",
      "TOP1 40\n",
      "TOP5 225\n",
      "TOP10 381\n",
      "TOP1 40\n",
      "TOP5 225\n",
      "TOP10 381\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 10 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: mse\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:381: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  np.random.shuffle((features_en_0))\n",
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:386: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 14 position: [1830, 2938, 3284, 3367, 3402, 3405, 3421, 3427, 4065, 4066, 4145, 4322, 4730, 4906]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:466: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/main_inp...)`\n",
      "  # # Save the history and the model\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 10, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_465 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_466 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_232 (Concatenate)    (None, 400)           0           lstm_465[0][0]                   \n",
      "                                                                   lstm_466[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_463 (Dense)                (None, 400)           160400      concatenate_232[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_463 (Dropout)            (None, 400)           0           dense_463[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_464 (Dense)                (None, 100)           40100       dropout_463[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_464 (Dropout)            (None, 100)           0           dense_464[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_464[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 4s - loss: 0.6055 - val_loss: 0.5377\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[/main_inp...)`\n",
      "  \n",
      "keras_title.py:159: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[/lstm1, /...)`\n",
      "  model_2.layers[7].set_weights(dlmodel.layers[9].get_weights())\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 21\n",
      "TOP5 144\n",
      "TOP10 255\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 4s - loss: 0.3301 - val_loss: 0.4091\n",
      "TOP1 34\n",
      "TOP5 189\n",
      "TOP10 326\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 4s - loss: 0.2052 - val_loss: 0.3911\n",
      "TOP1 47\n",
      "TOP5 212\n",
      "TOP10 340\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 4s - loss: 0.1382 - val_loss: 0.3807\n",
      "TOP1 47\n",
      "TOP5 219\n",
      "TOP10 360\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 4s - loss: 0.0916 - val_loss: 0.3780\n",
      "TOP1 47\n",
      "TOP5 230\n",
      "TOP10 372\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 4s - loss: 0.0595 - val_loss: 0.4250\n",
      "TOP1 46\n",
      "TOP5 252\n",
      "TOP10 385\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 4s - loss: 0.0405 - val_loss: 0.3783\n",
      "TOP1 55\n",
      "TOP5 247\n",
      "TOP10 373\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 4s - loss: 0.0369 - val_loss: 0.3991\n",
      "TOP1 45\n",
      "TOP5 224\n",
      "TOP10 358\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 4s - loss: 0.0395 - val_loss: 0.4391\n",
      "TOP1 37\n",
      "TOP5 211\n",
      "TOP10 345\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 4s - loss: 0.0476 - val_loss: 0.4423\n",
      "TOP1 26\n",
      "TOP5 163\n",
      "TOP10 300\n",
      "TOP1 26\n",
      "TOP5 163\n",
      "TOP10 300\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 15 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: mse\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:383: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # check the duplicated amount\n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n",
      "keras_title.py:388: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  X_1 = np.concatenate((features_en_1, features_en_0), axis = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 3 position: [1035, 3364, 4300]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:468: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[dense_503...)`\n",
      "  # path_model_lstm2 = \"model_lstm2_\" + code\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 15, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_507 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_508 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_253 (Concatenate)    (None, 400)           0           lstm_507[0][0]                   \n",
      "                                                                   lstm_508[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_504 (Dense)                (None, 400)           160400      concatenate_253[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_503 (Dropout)            (None, 400)           0           dense_504[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_505 (Dense)                (None, 100)           40100       dropout_503[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_504 (Dropout)            (None, 100)           0           dense_505[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_504[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 6s - loss: 0.5779 - val_loss: 0.4416\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[dense_503...)`\n",
      "  \n",
      "keras_title.py:160: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[dense_503...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 21\n",
      "TOP5 138\n",
      "TOP10 230\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 6s - loss: 0.3056 - val_loss: 0.3684\n",
      "TOP1 45\n",
      "TOP5 237\n",
      "TOP10 352\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 6s - loss: 0.1984 - val_loss: 0.3473\n",
      "TOP1 68\n",
      "TOP5 245\n",
      "TOP10 397\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 6s - loss: 0.1354 - val_loss: 0.3973\n",
      "TOP1 57\n",
      "TOP5 257\n",
      "TOP10 429\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 6s - loss: 0.0913 - val_loss: 0.3481\n",
      "TOP1 68\n",
      "TOP5 296\n",
      "TOP10 435\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 6s - loss: 0.0623 - val_loss: 0.3613\n",
      "TOP1 69\n",
      "TOP5 287\n",
      "TOP10 451\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 6s - loss: 0.0454 - val_loss: 0.3557\n",
      "TOP1 70\n",
      "TOP5 284\n",
      "TOP10 444\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 6s - loss: 0.0245 - val_loss: 0.3607\n",
      "TOP1 70\n",
      "TOP5 313\n",
      "TOP10 465\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 6s - loss: 0.0165 - val_loss: 0.3763\n",
      "TOP1 70\n",
      "TOP5 299\n",
      "TOP10 445\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 6s - loss: 0.0157 - val_loss: 0.3701\n",
      "TOP1 60\n",
      "TOP5 316\n",
      "TOP10 441\n",
      "TOP1 60\n",
      "TOP5 316\n",
      "TOP10 441\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false,
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "maxlen 20 epoch 10 dim_lstm 200 dim_Dense 400 100\n",
      "dropout_rate 0.0\n",
      "Activation function: ['relu', 'relu']\n",
      "bias of y: -1\n",
      "loss_function: mse\n",
      "------------------------------\n",
      "('Reading english Data:', 5000)\n",
      "('Reading english Data:', 5000)\n",
      "Merging the English and Japanes news dataframe...\n",
      "Drop the null line...\n",
      "False    4995\n",
      "True        5\n",
      "Name: en_article, dtype: int64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:382: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  \n",
      "keras_title.py:383: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # check the duplicated amount\n",
      "keras_title.py:387: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  # Prepare the final training and test data\n",
      "keras_title.py:388: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  X_1 = np.concatenate((features_en_1, features_en_0), axis = 0)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "C value = 4 position: [163, 1004, 3082, 4310]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:468: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[dense_503...)`\n",
      "  # path_model_lstm2 = \"model_lstm2_\" + code\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "____________________________________________________________________________________________________\n",
      "Layer (type)                     Output Shape          Param #     Connected to                     \n",
      "====================================================================================================\n",
      "main_input_1 (InputLayer)        (None, 20, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "main_input_2 (InputLayer)        (None, 20, 200)       0                                            \n",
      "____________________________________________________________________________________________________\n",
      "lstm_531 (LSTM)                  (None, 200)           320800      main_input_1[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "lstm_532 (LSTM)                  (None, 200)           320800      main_input_2[0][0]               \n",
      "____________________________________________________________________________________________________\n",
      "concatenate_265 (Concatenate)    (None, 400)           0           lstm_531[0][0]                   \n",
      "                                                                   lstm_532[0][0]                   \n",
      "____________________________________________________________________________________________________\n",
      "dense_528 (Dense)                (None, 400)           160400      concatenate_265[0][0]            \n",
      "____________________________________________________________________________________________________\n",
      "dropout_527 (Dropout)            (None, 400)           0           dense_528[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "dense_529 (Dense)                (None, 100)           40100       dropout_527[0][0]                \n",
      "____________________________________________________________________________________________________\n",
      "dropout_528 (Dropout)            (None, 100)           0           dense_529[0][0]                  \n",
      "____________________________________________________________________________________________________\n",
      "main_output (Dense)              (None, 1)             101         dropout_528[0][0]                \n",
      "====================================================================================================\n",
      "Total params: 842,201\n",
      "Trainable params: 842,201\n",
      "Non-trainable params: 0\n",
      "____________________________________________________________________________________________________\n",
      "None\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 1/1\n",
      "8000/8000 [==============================] - 8s - loss: 0.5821 - val_loss: 0.5336\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "keras_title.py:117: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=[Subtensor..., inputs=[dense_503...)`\n",
      "  \n",
      "keras_title.py:160: UserWarning: Update your `Model` call to the Keras 2 API: `Model(outputs=Elemwise{a..., inputs=[dense_503...)`\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "TOP1 21\n",
      "TOP5 122\n",
      "TOP10 214\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 2/2\n",
      "8000/8000 [==============================] - 8s - loss: 0.3019 - val_loss: 0.4293\n",
      "TOP1 32\n",
      "TOP5 195\n",
      "TOP10 318\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 3/3\n",
      "8000/8000 [==============================] - 8s - loss: 0.1836 - val_loss: 0.4235\n",
      "TOP1 67\n",
      "TOP5 271\n",
      "TOP10 435\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 4/4\n",
      "8000/8000 [==============================] - 8s - loss: 0.1195 - val_loss: 0.4071\n",
      "TOP1 53\n",
      "TOP5 273\n",
      "TOP10 438\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 5/5\n",
      "8000/8000 [==============================] - 8s - loss: 0.0775 - val_loss: 0.3688\n",
      "TOP1 59\n",
      "TOP5 276\n",
      "TOP10 435\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 6/6\n",
      "8000/8000 [==============================] - 8s - loss: 0.0539 - val_loss: 0.4182\n",
      "TOP1 56\n",
      "TOP5 260\n",
      "TOP10 419\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 7/7\n",
      "8000/8000 [==============================] - 8s - loss: 0.0373 - val_loss: 0.4112\n",
      "TOP1 60\n",
      "TOP5 271\n",
      "TOP10 411\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 8/8\n",
      "8000/8000 [==============================] - 8s - loss: 0.0271 - val_loss: 0.4168\n",
      "TOP1 58\n",
      "TOP5 299\n",
      "TOP10 445\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 9/9\n",
      "8000/8000 [==============================] - 8s - loss: 0.0213 - val_loss: 0.4108\n",
      "TOP1 44\n",
      "TOP5 257\n",
      "TOP10 432\n",
      "Train on 8000 samples, validate on 2000 samples\n",
      "Epoch 10/10\n",
      "8000/8000 [==============================] - 8s - loss: 0.0216 - val_loss: 0.4329\n",
      "TOP1 55\n",
      "TOP5 263\n",
      "TOP10 430\n",
      "TOP1 55\n",
      "TOP5 263\n",
      "TOP10 430\n"
     ]
    }
   ],
   "source": [
    "run keras_title.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}