train_seq.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From DQN.py:122: arg_max (from tensorflow.python.ops.gen_math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use `argmax` instead\n",
      "('loss', 100, 259.82812)\n",
      "('open loss', 100, 10.015593)\n",
      "('loss', 200, 159.98347)\n",
      "('open loss', 200, 3.6290262)\n",
      "('loss', 300, 147.96391)\n",
      "('open loss', 300, 3.5185876)\n",
      "('loss', 400, 151.48378)\n",
      "('open loss', 400, 0.81601524)\n",
      "('loss', 500, 220.49083)\n",
      "('open loss', 500, 1.141274)\n",
      "('loss', 600, 225.09843)\n",
      "('open loss', 600, 0.7413332)\n",
      "('loss', 700, 177.30547)\n",
      "('open loss', 700, 5.181875)\n",
      "('loss', 800, 138.07266)\n",
      "('open loss', 800, 1.5934232)\n",
      "('loss', 900, 185.9479)\n",
      "('open loss', 900, 3.5892887)\n",
      "('loss', 1000, 146.33969)\n",
      "('open loss', 1000, 0.7066427)\n",
      "('loss', 1100, 176.68382)\n",
      "('open loss', 1100, 2.7272987)\n",
      "('loss', 1200, 189.12881)\n",
      "('open loss', 1200, 1.6229471)\n",
      "('loss', 1300, 141.0231)\n",
      "('open loss', 1300, 1.9380943)\n",
      "('loss', 1400, 1810.0466)\n",
      "('open loss', 1400, 3.1920426)\n",
      "episode 20000\n",
      "('loss', 1500, 275.607)\n",
      "('open loss', 1500, 3.037034)\n",
      "('loss', 1600, 190.90926)\n",
      "('open loss', 1600, 1.1163877)\n",
      "('loss', 1700, 207.14317)\n",
      "('open loss', 1700, 2.527204)\n",
      "('loss', 1800, 214.79643)\n",
      "('open loss', 1800, 2.3749945)\n",
      "('loss', 1900, 245.29288)\n",
      "('open loss', 1900, 1.8842126)\n",
      "('loss', 2000, 220.22144)\n",
      "('open loss', 2000, 1.3708404)\n",
      "('loss', 2100, 401.52682)\n",
      "('open loss', 2100, 8.448906)\n",
      "('loss', 2200, 223.01334)\n",
      "('open loss', 2200, 2.5524461)\n",
      "('loss', 2300, 225.64027)\n",
      "('open loss', 2300, 1.2628608)\n",
      "('loss', 2400, 216.13107)\n",
      "('open loss', 2400, 2.8022947)\n",
      "('loss', 2500, 189.66292)\n",
      "('open loss', 2500, 6.0966187)\n",
      "('loss', 2600, 185.28865)\n",
      "('open loss', 2600, 2.5490308)\n",
      "('loss', 2700, 181.40164)\n",
      "('open loss', 2700, 1.1710699)\n",
      "('loss', 2800, 175.73506)\n",
      "('open loss', 2800, 1.8622937)\n",
      "episode 30000\n",
      "episode 30000\n",
      "episode 30000\n",
      "episode 30000\n",
      "('loss', 2900, 215.22594)\n",
      "('open loss', 2900, 9.188995)\n",
      "('loss', 3000, 14073.104)\n",
      "('open loss', 3000, 7.9561977)\n",
      "('loss', 3100, 234.60764)\n",
      "('open loss', 3100, 3.182091)\n",
      "('loss', 3200, 466.1937)\n",
      "('open loss', 3200, 4.5579677)\n",
      "('loss', 3300, 199.37535)\n",
      "('open loss', 3300, 24.558428)\n",
      "('loss', 3400, 198.5814)\n",
      "('open loss', 3400, 41.060276)\n",
      "('loss', 3500, 208.30757)\n",
      "('open loss', 3500, 5.23853)\n",
      "('loss', 3600, 180.63603)\n",
      "('open loss', 3600, 33.96158)\n",
      "('loss', 3700, 166.50113)\n",
      "('open loss', 3700, 4.1299114)\n",
      "('loss', 3800, 217.16866)\n",
      "('open loss', 3800, 1.3782799)\n",
      "('loss', 3900, 172.67584)\n",
      "('open loss', 3900, 1.5932797)\n",
      "('loss', 4000, 212.09425)\n",
      "('open loss', 4000, 3.3583682)\n",
      "('loss', 4100, 204.41081)\n",
      "('open loss', 4100, 4.744397)\n",
      "('loss', 4200, 207.20969)\n",
      "('open loss', 4200, 2.7924619)\n",
      "('loss', 4300, 170.93886)\n",
      "('open loss', 4300, 5.3631916)\n",
      "('loss', 4400, 229.10533)\n",
      "('open loss', 4400, 305.58353)\n",
      "('loss', 4500, 222.08203)\n",
      "('open loss', 4500, 13.325277)\n",
      "episode 40000\n",
      "('loss', 4600, 229.55595)\n",
      "('open loss', 4600, 6.278552)\n",
      "('loss', 4700, 236.20226)\n",
      "('open loss', 4700, 3.136284)\n",
      "('loss', 4800, 230.5588)\n",
      "('open loss', 4800, 14.899469)\n",
      "('loss', 4900, 242.75522)\n",
      "('open loss', 4900, 8.274839)\n",
      "('loss', 5000, 219.58281)\n",
      "('open loss', 5000, 7.7843566)\n",
      "('loss', 5100, 237.76834)\n",
      "('open loss', 5100, 3.3533025)\n",
      "('loss', 5200, 251.47359)\n",
      "('open loss', 5200, 10.163898)\n",
      "('loss', 5300, 242.85869)\n",
      "('open loss', 5300, 20.470922)\n",
      "('loss', 5400, 204.25308)\n",
      "('open loss', 5400, 3.5790415)\n",
      "('loss', 5500, 317.6468)\n",
      "('open loss', 5500, 4.8888297)\n",
      "('loss', 5600, 233.245)\n",
      "('open loss', 5600, 7.11079)\n",
      "('loss', 5700, 248.56)\n",
      "('open loss', 5700, 3.6272073)\n",
      "('loss', 5800, 215.44464)\n",
      "('open loss', 5800, 4.68339)\n",
      "('loss', 5900, 229.9065)\n",
      "('open loss', 5900, 17.682306)\n",
      "('loss', 6000, 231.17554)\n",
      "('open loss', 6000, 35.8117)\n",
      "('loss', 6100, 349.61154)\n",
      "('open loss', 6100, 587.23865)\n",
      "('loss', 6200, 274.9484)\n",
      "('open loss', 6200, 37.950188)\n",
      "('loss', 6300, 252.88974)\n",
      "('open loss', 6300, 14.396567)\n",
      "episode 50000\n",
      "episode 50000\n",
      "('loss', 6400, 229.53902)\n",
      "('open loss', 6400, 6.416561)\n",
      "('loss', 6500, 298.1199)\n",
      "('open loss', 6500, 5.113886)\n",
      "('loss', 6600, 215.50783)\n",
      "('open loss', 6600, 11.423488)\n",
      "('loss', 6700, 261.34665)\n",
      "('open loss', 6700, 13.097281)\n",
      "('loss', 6800, 228.7043)\n",
      "('open loss', 6800, 6.183641)\n",
      "('loss', 6900, 233.5747)\n",
      "('open loss', 6900, 118.29948)\n",
      "('loss', 7000, 312.70157)\n",
      "('open loss', 7000, 82.61)\n",
      "('loss', 7100, 250.45876)\n",
      "('open loss', 7100, 13.623709)\n",
      "('loss', 7200, 250.62268)\n",
      "('open loss', 7200, 10.685129)\n",
      "('loss', 7300, 280.4889)\n",
      "('open loss', 7300, 14.8540745)\n",
      "('loss', 7400, 261.25864)\n",
      "('open loss', 7400, 8.877039)\n",
      "('loss', 7500, 256.48117)\n",
      "('open loss', 7500, 4.5351043)\n",
      "('loss', 7600, 297.0223)\n",
      "('open loss', 7600, 13.097722)\n",
      "('loss', 7700, 206.74222)\n",
      "('open loss', 7700, 28.199541)\n",
      "('loss', 7800, 252.00119)\n",
      "('open loss', 7800, 9.288696)\n",
      "('loss', 7900, 244.45993)\n",
      "('open loss', 7900, 89.418495)\n",
      "('loss', 8000, 261.57635)\n",
      "('open loss', 8000, 20.544037)\n",
      "episode 60000\n",
      "episode 60000\n",
      "episode 60000\n",
      "episode 60000\n",
      "('loss', 8100, 236.6061)\n",
      "('open loss', 8100, 8.100156)\n",
      "('loss', 8200, 1789.6683)\n",
      "('open loss', 8200, 18.62873)\n",
      "('loss', 8300, 721.5027)\n",
      "('open loss', 8300, 165.5035)\n",
      "('loss', 8400, 240.21768)\n",
      "('open loss', 8400, 8.373261)\n",
      "('loss', 8500, 289.67563)\n",
      "('open loss', 8500, 23.44542)\n",
      "('loss', 8600, 235.23558)\n",
      "('open loss', 8600, 28.384466)\n",
      "('loss', 8700, 292.79428)\n",
      "('open loss', 8700, 12.165112)\n",
      "('loss', 8800, 361.28043)\n",
      "('open loss', 8800, 8.902716)\n",
      "('loss', 8900, 272.01624)\n",
      "('open loss', 8900, 5.499675)\n",
      "('loss', 9000, 299.0344)\n",
      "('open loss', 9000, 30.176247)\n",
      "('loss', 9100, 346.97742)\n",
      "('open loss', 9100, 366.82825)\n",
      "('loss', 9200, 336.1241)\n",
      "('open loss', 9200, 67.29559)\n",
      "('loss', 9300, 1209.8182)\n",
      "('open loss', 9300, 22.367615)\n",
      "('loss', 9400, 279.70874)\n",
      "('open loss', 9400, 19.255552)\n",
      "('loss', 9500, 265.7253)\n",
      "('open loss', 9500, 11.425558)\n",
      "('loss', 9600, 284.22372)\n",
      "('open loss', 9600, 94.98892)\n",
      "('loss', 9700, 263.6327)\n",
      "('open loss', 9700, 20.220732)\n",
      "('loss', 9800, 254.74843)\n",
      "('open loss', 9800, 30.45598)\n",
      "('loss', 9900, 293.44614)\n",
      "('open loss', 9900, 20.622047)\n",
      "('loss', 10000, 275.1597)\n",
      "('open loss', 10000, 32.35486)\n",
      "('loss', 10100, 322.97025)\n",
      "('open loss', 10100, 22.102556)\n",
      "('loss', 10200, 257.98105)\n",
      "('open loss', 10200, 31.8956)\n",
      "episode 70000\n",
      "episode 70000\n",
      "episode 70000\n",
      "('loss', 10300, 309.117)\n",
      "('open loss', 10300, 132.52303)\n",
      "('loss', 10400, 1146.6133)\n",
      "('open loss', 10400, 866.8282)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0mTraceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-83d5f6cbbc7f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     34\u001b[0m         \u001b[0;31m# DQN 根据观测值选择行为\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m         \u001b[0mavailble_actions\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgameEnv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchooseAvailbleAction\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mplayerI\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 36\u001b[0;31m         \u001b[0maction\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0m_\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mRLModel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mchoose_action\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0marray\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mobservation_this\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mavailble_actions\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mgameEnv\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstepNum\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     37\u001b[0m         \u001b[0;31m# 环境根据行为给出下一个 state, reward, 是否终止\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0;31m#print (playerI, action, gameEnv.deskMoney, gameEnv.nowPrice,gameEnv.personStatus[playerI])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/data/opt/jupyter/fanyu.zhang/GoldenFlower/DQN.py\u001b[0m in \u001b[0;36mchoose_action\u001b[0;34m(self, status, availble_actions, step, debug)\u001b[0m\n\u001b[1;32m    192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m         \u001b[0m_feed_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_feed_dict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstatus\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 194\u001b[0;31m         \u001b[0mprob_all\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msess\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredictions\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0m_feed_dict\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    195\u001b[0m         \u001b[0mseeFlag\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"闷\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    196\u001b[0m         \u001b[0;31m# 状态为闷的，确定是否要看\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m    903\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[0;32m--> 905\u001b[0;31m                          run_metadata_ptr)\n\u001b[0m\u001b[1;32m    906\u001b[0m       \u001b[0;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run\u001b[0;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1135\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[0;32m-> 1137\u001b[0;31m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[1;32m   1138\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_run\u001b[0;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[1;32m   1353\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[0;32m-> 1355\u001b[0;31m                            options, run_metadata)\n\u001b[0m\u001b[1;32m   1356\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1357\u001b[0m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_do_call\u001b[0;34m(self, fn, *args)\u001b[0m\n\u001b[1;32m   1359\u001b[0m   \u001b[0;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1360\u001b[0m     \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1361\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1362\u001b[0m     \u001b[0;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/anaconda2/lib/python2.7/site-packages/tensorflow/python/client/session.pyc\u001b[0m in \u001b[0;36m_run_fn\u001b[0;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[1;32m   1338\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[0;32m-> 1340\u001b[0;31m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[1;32m   1341\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1342\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msession\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "#-*- coding:utf-8\n",
    "import sys\n",
    "import os\n",
    "os.environ['CUDA_VISIBLE_DEVICES'] = \"\"\n",
    "stdi, stdo, stde = sys.stdin, sys.stdout, sys.stderr\n",
    "reload(sys)\n",
    "sys.setdefaultencoding('utf-8')\n",
    "sys.stdin, sys.stdout, sys.stderr = stdi, stdo, stde\n",
    "\n",
    "import GameEnv\n",
    "import random\n",
    "import numpy as np\n",
    "import DQN\n",
    "\n",
    "gameEnv = GameEnv.GlodenFlower([100000,100000])\n",
    "RLModel = DQN.DQN(embedding_size=50,sequence_length=20,learning_rate=0.01,batch_size=1000)\n",
    "gameEnv.debug = False\n",
    "for episode in range(500000):\n",
    "    # 初始化环境\n",
    "    gameEnv.reset()\n",
    "\n",
    "    playerI = gameEnv.getStartTurn()\n",
    "    observation_this = [[],gameEnv.playerCards[\"A\"],gameEnv.personStatus[\"A\"]]\n",
    "    if playerI == \"B\":\n",
    "        availble_actions = gameEnv.chooseAvailbleAction(playerI)\n",
    "        action,_ = RLModel.choose_action(np.array([gameEnv.transformObservationForB(observation_this)]),availble_actions,gameEnv.stepNum)\n",
    "        #print (playerI, action, gameEnv.deskMoney, gameEnv.nowPrice)\n",
    "        observation_next, reward, done = gameEnv.step(action, \"B\")\n",
    "        playerI = \"A\"\n",
    "        if done:\n",
    "            continue\n",
    "\n",
    "    while True:\n",
    "        # DQN 根据观测值选择行为\n",
    "        availble_actions = gameEnv.chooseAvailbleAction(playerI)\n",
    "        action,_ = RLModel.choose_action(np.array([observation_this]),availble_actions,gameEnv.stepNum)\n",
    "        # 环境根据行为给出下一个 state, reward, 是否终止\n",
    "        #print (playerI, action, gameEnv.deskMoney, gameEnv.nowPrice,gameEnv.personStatus[playerI])\n",
    "        observation_next, reward, done = gameEnv.stepA(action,RLModel)\n",
    "\n",
    "\n",
    "        # DQN 存储记忆\n",
    "        # RL.store_transition(observation, action, reward)\n",
    "        RLModel.store_transition(observation_this, action, reward,done,observation_next,gameEnv.playerCards[\"B\"],gameEnv.personStatus[\"A\"],gameEnv.nowPrice)\n",
    "\n",
    "        # 控制学习起始时间和频率 (先累积一些记忆再开始学习)\n",
    "        if (episode > 10000):\n",
    "            if episode % 10000 == 0 :print \"episode\",episode\n",
    "            if episode % 10 == 0:\n",
    "                RLModel.train(RLModel.experience_see_replay())\n",
    "                RLModel.train_open(RLModel.experience_open_replay())\n",
    "\n",
    "        # 将下一个 state_ 变为 下次循环的 state\n",
    "        observation_this = observation_next\n",
    "\n",
    "        # 如果终止, 就跳出循环\n",
    "        if done:\n",
    "            break\n",
    "\n",
    "        # end of game\n",
    "print('game over')\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 验证终结点是否有效"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "l = RLModel.card_index_dicts.keys()\n",
    "l.remove(\" \")\n",
    "card_list = []\n",
    "input_data = []\n",
    "for i in range(100000):\n",
    "    cards = random.sample(l,3)\n",
    "    card_list.append(cards)\n",
    "    input_data.append([[],cards ,\"看\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "res = RLModel.sess.run(RLModel.predictions,feed_dict=RLModel._feed_dict(np.array(input_data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(data={\"card\":card_list,\"score\":res[:,8]})\n",
    "\n",
    "import GameEnv\n",
    "gameEnv = GameEnv.GlodenFlower([2000,2000])\n",
    "d = {\"豹子\":10,\"同花顺\":9,\"金花\":8,\"顺子\":7,\"对子\":6,\"单\":5}\n",
    "reverse_dict = {v: k for k, v in d.iteritems()}\n",
    "\n",
    "def f1(x):\n",
    "    return reverse_dict[gameEnv.score(x)]\n",
    "    \n",
    "df['card_level'] = df['card'].apply(lambda x:f1(x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card</th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_level</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>单</th>\n",
       "      <td>74764</td>\n",
       "      <td>74764</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>同花顺</th>\n",
       "      <td>220</td>\n",
       "      <td>220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>对子</th>\n",
       "      <td>16798</td>\n",
       "      <td>16798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>豹子</th>\n",
       "      <td>232</td>\n",
       "      <td>232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>金花</th>\n",
       "      <td>5025</td>\n",
       "      <td>5025</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>顺子</th>\n",
       "      <td>2961</td>\n",
       "      <td>2961</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             card  score\n",
       "card_level              \n",
       "单           74764  74764\n",
       "同花顺           220    220\n",
       "对子          16798  16798\n",
       "豹子            232    232\n",
       "金花           5025   5025\n",
       "顺子           2961   2961"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(\"card_level\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>score</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>card_level</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>单</th>\n",
       "      <td>-5.581727</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>同花顺</th>\n",
       "      <td>62.862709</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>对子</th>\n",
       "      <td>7.465661</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>豹子</th>\n",
       "      <td>11.503154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>金花</th>\n",
       "      <td>8.099229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>顺子</th>\n",
       "      <td>8.476376</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                score\n",
       "card_level           \n",
       "单           -5.581727\n",
       "同花顺         62.862709\n",
       "对子           7.465661\n",
       "豹子          11.503154\n",
       "金花           8.099229\n",
       "顺子           8.476376"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.groupby(\"card_level\").mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>card</th>\n",
       "      <th>score</th>\n",
       "      <th>card_level</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>40328</th>\n",
       "      <td>[spade_K, spade_1, heart_Q]</td>\n",
       "      <td>16.200001</td>\n",
       "      <td>顺子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>40869</th>\n",
       "      <td>[heart_Q, spade_1, spade_K]</td>\n",
       "      <td>16.200001</td>\n",
       "      <td>顺子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23969</th>\n",
       "      <td>[heart_7, heart_1, heart_2]</td>\n",
       "      <td>16.177023</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>81334</th>\n",
       "      <td>[heart_7, heart_1, heart_2]</td>\n",
       "      <td>16.177023</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>30457</th>\n",
       "      <td>[heart_2, heart_1, heart_7]</td>\n",
       "      <td>16.177023</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>22452</th>\n",
       "      <td>[heart_2, heart_7, heart_1]</td>\n",
       "      <td>16.177023</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74130</th>\n",
       "      <td>[heart_1, club_5, spade_1]</td>\n",
       "      <td>16.147739</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25479</th>\n",
       "      <td>[heart_1, club_5, spade_1]</td>\n",
       "      <td>16.147739</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9855</th>\n",
       "      <td>[heart_1, club_5, spade_1]</td>\n",
       "      <td>16.147739</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>45411</th>\n",
       "      <td>[club_5, spade_1, club_1]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89207</th>\n",
       "      <td>[club_5, club_1, spade_1]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>41680</th>\n",
       "      <td>[club_5, spade_1, club_1]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>83576</th>\n",
       "      <td>[club_1, club_5, spade_1]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11952</th>\n",
       "      <td>[spade_1, club_1, club_5]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13724</th>\n",
       "      <td>[club_5, spade_1, club_1]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17442</th>\n",
       "      <td>[club_5, spade_1, club_1]</td>\n",
       "      <td>16.088219</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62800</th>\n",
       "      <td>[heart_3, heart_4, spade_3]</td>\n",
       "      <td>16.005739</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>69487</th>\n",
       "      <td>[heart_4, spade_3, heart_3]</td>\n",
       "      <td>16.005739</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2991</th>\n",
       "      <td>[diamond_K, club_1, spade_1]</td>\n",
       "      <td>15.925213</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88228</th>\n",
       "      <td>[club_1, spade_1, diamond_K]</td>\n",
       "      <td>15.925213</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20865</th>\n",
       "      <td>[club_1, diamond_K, spade_1]</td>\n",
       "      <td>15.925213</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98162</th>\n",
       "      <td>[spade_1, club_1, diamond_K]</td>\n",
       "      <td>15.925213</td>\n",
       "      <td>对子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11191</th>\n",
       "      <td>[heart_7, heart_1, heart_3]</td>\n",
       "      <td>15.830370</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97324</th>\n",
       "      <td>[heart_3, heart_1, heart_7]</td>\n",
       "      <td>15.830370</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70848</th>\n",
       "      <td>[heart_1, heart_3, heart_7]</td>\n",
       "      <td>15.830370</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92704</th>\n",
       "      <td>[heart_7, heart_1, heart_3]</td>\n",
       "      <td>15.830370</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>77011</th>\n",
       "      <td>[heart_1, heart_7, heart_3]</td>\n",
       "      <td>15.830370</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88711</th>\n",
       "      <td>[diamond_K, spade_K, club_K]</td>\n",
       "      <td>15.828056</td>\n",
       "      <td>豹子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80652</th>\n",
       "      <td>[diamond_K, spade_K, club_K]</td>\n",
       "      <td>15.828056</td>\n",
       "      <td>豹子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>63458</th>\n",
       "      <td>[club_K, spade_K, diamond_K]</td>\n",
       "      <td>15.828056</td>\n",
       "      <td>豹子</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>42840</th>\n",
       "      <td>[spade_4, spade_6, spade_2]</td>\n",
       "      <td>-120.027451</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43187</th>\n",
       "      <td>[club_J, club_10, club_Q]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95940</th>\n",
       "      <td>[club_J, club_Q, club_10]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20919</th>\n",
       "      <td>[club_J, club_Q, club_10]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65007</th>\n",
       "      <td>[club_Q, club_10, club_J]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8927</th>\n",
       "      <td>[club_10, club_Q, club_J]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70658</th>\n",
       "      <td>[club_10, club_J, club_Q]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>65256</th>\n",
       "      <td>[club_J, club_Q, club_10]</td>\n",
       "      <td>-165.373199</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9748</th>\n",
       "      <td>[diamond_10, diamond_J, diamond_Q]</td>\n",
       "      <td>-185.105087</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54556</th>\n",
       "      <td>[diamond_J, diamond_Q, diamond_10]</td>\n",
       "      <td>-185.105087</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>43197</th>\n",
       "      <td>[diamond_Q, diamond_10, diamond_J]</td>\n",
       "      <td>-185.105087</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25445</th>\n",
       "      <td>[diamond_Q, diamond_J, diamond_10]</td>\n",
       "      <td>-185.105087</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74574</th>\n",
       "      <td>[diamond_J, diamond_10, diamond_Q]</td>\n",
       "      <td>-185.105087</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>32125</th>\n",
       "      <td>[diamond_Q, diamond_J, diamond_10]</td>\n",
       "      <td>-185.105087</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21640</th>\n",
       "      <td>[club_10, club_9, club_J]</td>\n",
       "      <td>-216.493866</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>38018</th>\n",
       "      <td>[club_J, club_10, club_9]</td>\n",
       "      <td>-216.493866</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56060</th>\n",
       "      <td>[club_10, club_J, club_9]</td>\n",
       "      <td>-216.493866</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>66191</th>\n",
       "      <td>[club_J, club_10, club_9]</td>\n",
       "      <td>-216.493866</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>85395</th>\n",
       "      <td>[club_10, club_9, club_J]</td>\n",
       "      <td>-216.493866</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99650</th>\n",
       "      <td>[club_J, club_10, club_9]</td>\n",
       "      <td>-216.493866</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>53174</th>\n",
       "      <td>[spade_1, spade_4, spade_6]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>92954</th>\n",
       "      <td>[spade_1, spade_6, spade_4]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>33798</th>\n",
       "      <td>[spade_4, spade_6, spade_1]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18693</th>\n",
       "      <td>[spade_4, spade_6, spade_1]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59785</th>\n",
       "      <td>[spade_6, spade_1, spade_4]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89471</th>\n",
       "      <td>[spade_4, spade_6, spade_1]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15801</th>\n",
       "      <td>[spade_4, spade_1, spade_6]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>29793</th>\n",
       "      <td>[spade_4, spade_6, spade_1]</td>\n",
       "      <td>-306.815063</td>\n",
       "      <td>金花</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80119</th>\n",
       "      <td>[spade_4, spade_2, spade_3]</td>\n",
       "      <td>-331.663483</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>54240</th>\n",
       "      <td>[spade_2, spade_4, spade_3]</td>\n",
       "      <td>-331.663483</td>\n",
       "      <td>同花顺</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>99900 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     card       score card_level\n",
       "40328         [spade_K, spade_1, heart_Q]   16.200001         顺子\n",
       "40869         [heart_Q, spade_1, spade_K]   16.200001         顺子\n",
       "23969         [heart_7, heart_1, heart_2]   16.177023         金花\n",
       "81334         [heart_7, heart_1, heart_2]   16.177023         金花\n",
       "30457         [heart_2, heart_1, heart_7]   16.177023         金花\n",
       "22452         [heart_2, heart_7, heart_1]   16.177023         金花\n",
       "74130          [heart_1, club_5, spade_1]   16.147739         对子\n",
       "25479          [heart_1, club_5, spade_1]   16.147739         对子\n",
       "9855           [heart_1, club_5, spade_1]   16.147739         对子\n",
       "45411           [club_5, spade_1, club_1]   16.088219         对子\n",
       "89207           [club_5, club_1, spade_1]   16.088219         对子\n",
       "41680           [club_5, spade_1, club_1]   16.088219         对子\n",
       "83576           [club_1, club_5, spade_1]   16.088219         对子\n",
       "11952           [spade_1, club_1, club_5]   16.088219         对子\n",
       "13724           [club_5, spade_1, club_1]   16.088219         对子\n",
       "17442           [club_5, spade_1, club_1]   16.088219         对子\n",
       "62800         [heart_3, heart_4, spade_3]   16.005739         对子\n",
       "69487         [heart_4, spade_3, heart_3]   16.005739         对子\n",
       "2991         [diamond_K, club_1, spade_1]   15.925213         对子\n",
       "88228        [club_1, spade_1, diamond_K]   15.925213         对子\n",
       "20865        [club_1, diamond_K, spade_1]   15.925213         对子\n",
       "98162        [spade_1, club_1, diamond_K]   15.925213         对子\n",
       "11191         [heart_7, heart_1, heart_3]   15.830370         金花\n",
       "97324         [heart_3, heart_1, heart_7]   15.830370         金花\n",
       "70848         [heart_1, heart_3, heart_7]   15.830370         金花\n",
       "92704         [heart_7, heart_1, heart_3]   15.830370         金花\n",
       "77011         [heart_1, heart_7, heart_3]   15.830370         金花\n",
       "88711        [diamond_K, spade_K, club_K]   15.828056         豹子\n",
       "80652        [diamond_K, spade_K, club_K]   15.828056         豹子\n",
       "63458        [club_K, spade_K, diamond_K]   15.828056         豹子\n",
       "...                                   ...         ...        ...\n",
       "42840         [spade_4, spade_6, spade_2] -120.027451         金花\n",
       "43187           [club_J, club_10, club_Q] -165.373199        同花顺\n",
       "95940           [club_J, club_Q, club_10] -165.373199        同花顺\n",
       "20919           [club_J, club_Q, club_10] -165.373199        同花顺\n",
       "65007           [club_Q, club_10, club_J] -165.373199        同花顺\n",
       "8927            [club_10, club_Q, club_J] -165.373199        同花顺\n",
       "70658           [club_10, club_J, club_Q] -165.373199        同花顺\n",
       "65256           [club_J, club_Q, club_10] -165.373199        同花顺\n",
       "9748   [diamond_10, diamond_J, diamond_Q] -185.105087        同花顺\n",
       "54556  [diamond_J, diamond_Q, diamond_10] -185.105087        同花顺\n",
       "43197  [diamond_Q, diamond_10, diamond_J] -185.105087        同花顺\n",
       "25445  [diamond_Q, diamond_J, diamond_10] -185.105087        同花顺\n",
       "74574  [diamond_J, diamond_10, diamond_Q] -185.105087        同花顺\n",
       "32125  [diamond_Q, diamond_J, diamond_10] -185.105087        同花顺\n",
       "21640           [club_10, club_9, club_J] -216.493866        同花顺\n",
       "38018           [club_J, club_10, club_9] -216.493866        同花顺\n",
       "56060           [club_10, club_J, club_9] -216.493866        同花顺\n",
       "66191           [club_J, club_10, club_9] -216.493866        同花顺\n",
       "85395           [club_10, club_9, club_J] -216.493866        同花顺\n",
       "99650           [club_J, club_10, club_9] -216.493866        同花顺\n",
       "53174         [spade_1, spade_4, spade_6] -306.815063         金花\n",
       "92954         [spade_1, spade_6, spade_4] -306.815063         金花\n",
       "33798         [spade_4, spade_6, spade_1] -306.815063         金花\n",
       "18693         [spade_4, spade_6, spade_1] -306.815063         金花\n",
       "59785         [spade_6, spade_1, spade_4] -306.815063         金花\n",
       "89471         [spade_4, spade_6, spade_1] -306.815063         金花\n",
       "15801         [spade_4, spade_1, spade_6] -306.815063         金花\n",
       "29793         [spade_4, spade_6, spade_1] -306.815063         金花\n",
       "80119         [spade_4, spade_2, spade_3] -331.663483        同花顺\n",
       "54240         [spade_2, spade_4, spade_3] -331.663483        同花顺\n",
       "\n",
       "[99900 rows x 3 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.sort_values(\"score\",ascending=False).iloc[100:]\n",
    "# df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 单点检查\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 闷_2 1.1362231\n",
      "1 闷_4 4.0263476\n",
      "2 闷_8 1.6519315\n",
      "3 闷开_0 -0.0860311\n",
      "4 看 -2.184517\n"
     ]
    }
   ],
   "source": [
    "# RLModel.train(RLModel.exerience_replay_final_step())\n",
    "res = RLModel.sess.run(RLModel.predictionsNotSee,feed_dict=RLModel._feed_dict(np.array([[[],  ['spade_6', 'spade_10', 'spade_8'],\"闷\"]])))\n",
    "for i,v in  zip(RLModel.action_notsee_reverse_index_dicts.keys(),res[0]):\n",
    "    print i, RLModel.action_notsee_reverse_index_dicts[i],v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[-11.574262    1.5971797  -3.8307111   0.3447262  -2.2230055]\n",
      "看\n"
     ]
    }
   ],
   "source": [
    "print res[0]\n",
    "print RLModel._random_pick([\"闷_2\",\"闷_4\",\"闷_8\",\"闷开_0\",\"看\"],res[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7_4', 0.24453352)\n",
      "('\\xe9\\x97\\xb7_2', 0.28051785)\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.2544603)\n",
      "('\\xe9\\x97\\xb7_8', 0.22048834)\n",
      "闷_2\n"
     ]
    }
   ],
   "source": [
    "# RLModel.train(RLModel.exerience_replay_final_step())\n",
    "print RLModel.choose_action_max(np.array([[[],  ['heart_4', 'heart_2', 'spade_8'],\"闷\"]]),RLModel.actions_index_dicts.keys(),1,True)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[False]\n"
     ]
    }
   ],
   "source": [
    "# RLModel.train(RLModel.exerience_replay_final_step())\n",
    "import tensorflow as tf\n",
    "res = RLModel.sess.run(tf.equal(tf.arg_max(RLModel.predictionsNotSee,1),len(RLModel.action_notsee_index_dicts)),feed_dict=RLModel._feed_dict(np.array([[[\"B_看_20\",\"A_闷_8\",\"B_看_20\"],  ['heart_4', 'heart_2', 'spade_8'],\"闷\"]])))\n",
    "print res\n",
    "#     "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 闷_2 -1.5942369\n",
      "1 闷_4 6.0963144\n",
      "2 闷_8 27.949627\n",
      "3 闷开_0 3.9862342\n",
      "4 看_2 0.0\n",
      "5 看_5 0.0\n",
      "6 看_10 0.0\n",
      "7 看_20 0.0\n",
      "8 开_0 0.0\n",
      "9 丢_0 -0.0\n"
     ]
    }
   ],
   "source": [
    "# RLModel.train(RLModel.exerience_replay_final_step())\n",
    "res = RLModel.sess.run(RLModel.predictions,feed_dict=RLModel._feed_dict(np.array([[[\"B_看_20\",\"A_闷_8\",\"B_看_20\"],  ['heart_4', 'heart_2', 'heart_3'],\"闷\"]])))\n",
    "for i,v in  zip(RLModel.action_reverse_index_dicts.keys(),res[0]):\n",
    "    print i, RLModel.action_reverse_index_dicts[i],v"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检查完整的一把过程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['club_2', 'heart_8', 'spade_10'] ['club_3', 'heart_2', 'spade_9'] \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.2544603)\n",
      "('\\xe9\\x97\\xb7_4', 0.24453352)\n",
      "('\\xe9\\x97\\xb7_2', 0.28051785)\n",
      "('\\xe9\\x97\\xb7_8', 0.22048834)\n",
      "0 闷_2 0.862558\n",
      "1 闷_4 -0.51029044\n",
      "2 闷_8 -1.5453668\n",
      "3 闷开_0 -0.11236637\n",
      "4 看_2 1.1037825\n",
      "5 看_5 -2.4782772\n",
      "6 看_10 -7.3136\n",
      "7 看_20 -15.57754\n",
      "8 开_0 -23.355452\n",
      "9 丢_0 -0.97227585\n",
      "______ A 闷_2 0.28051785 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.2548142)\n",
      "('\\xe9\\x97\\xb7_4', 0.22328515)\n",
      "('\\xe9\\x97\\xb7_2', 0.3097111)\n",
      "('\\xe9\\x97\\xb7_8', 0.21218948)\n",
      "0 闷_2 1.801272\n",
      "1 闷_4 -1.4706305\n",
      "2 闷_8 -1.9803306\n",
      "3 闷开_0 -0.14978021\n",
      "4 看_2 2.3119493\n",
      "5 看_5 -1.6553155\n",
      "6 看_10 -6.2624264\n",
      "7 看_20 -21.73891\n",
      "8 开_0 -24.083447\n",
      "9 丢_0 -1.9734172\n",
      "______ A 闷_2 0.3097111 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.25183108)\n",
      "('\\xe9\\x97\\xb7_4', 0.64660376)\n",
      "('\\xe9\\x97\\xb7_8', 0.101565115)\n",
      "0 闷_2 -8.072526\n",
      "1 闷_4 11.872682\n",
      "2 闷_8 -6.6376553\n",
      "3 闷开_0 2.4429295\n",
      "4 看_2 -18.83345\n",
      "5 看_5 -22.659836\n",
      "6 看_10 -30.738573\n",
      "7 看_20 -63.97066\n",
      "8 开_0 -46.255383\n",
      "9 丢_0 -2.109784\n",
      "______ A 闷_4 0.64660376 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.0040273056)\n",
      "('\\xe9\\x97\\xb7_4', 0.02384547)\n",
      "('\\xe9\\x97\\xb7_8', 0.97212726)\n",
      "0 闷_2 -16.67851\n",
      "1 闷_4 1.2265867\n",
      "2 闷_8 38.30551\n",
      "3 闷开_0 -16.55838\n",
      "4 看_2 -28.627768\n",
      "5 看_5 -21.19807\n",
      "6 看_10 -22.2282\n",
      "7 看_20 -71.99234\n",
      "8 开_0 -46.879906\n",
      "9 丢_0 -10.543123\n",
      "______ A 闷_8 0.97212726 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.0016064966)\n",
      "('\\xe9\\x97\\xb7_8', 0.99839354)\n",
      "0 闷_2 -6.311447\n",
      "1 闷_4 -0.38634294\n",
      "2 闷_8 38.95027\n",
      "3 闷开_0 -25.370646\n",
      "4 看_2 -42.66687\n",
      "5 看_5 -18.466206\n",
      "6 看_10 -12.900815\n",
      "7 看_20 -83.59524\n",
      "8 开_0 -56.078697\n",
      "9 丢_0 -22.105833\n",
      "______ A 闷_8 0.99839354 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.006974899)\n",
      "('\\xe9\\x97\\xb7_8', 0.9930251)\n",
      "0 闷_2 -26.140728\n",
      "1 闷_4 -39.768528\n",
      "2 闷_8 38.744244\n",
      "3 闷开_0 -10.840136\n",
      "4 看_2 -41.275436\n",
      "5 看_5 19.275276\n",
      "6 看_10 -1.5753677\n",
      "7 看_20 -62.32558\n",
      "8 开_0 -15.192328\n",
      "9 丢_0 -32.658855\n",
      "______ A 闷_8 0.9930251 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.00622304)\n",
      "('\\xe9\\x97\\xb7_8', 0.9937769)\n",
      "0 闷_2 -24.861755\n",
      "1 闷_4 -39.656967\n",
      "2 闷_8 42.09819\n",
      "3 闷开_0 -8.634349\n",
      "4 看_2 -31.94622\n",
      "5 看_5 29.756298\n",
      "6 看_10 5.5765553\n",
      "7 看_20 -49.590645\n",
      "8 开_0 -2.618636\n",
      "9 丢_0 -32.7766\n",
      "______ A 闷_8 0.9937769 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.009499022)\n",
      "('\\xe9\\x97\\xb7_8', 0.99050105)\n",
      "0 闷_2 -24.240797\n",
      "1 闷_4 -31.943022\n",
      "2 闷_8 42.142956\n",
      "3 闷开_0 -4.327262\n",
      "4 看_2 -28.464146\n",
      "5 看_5 32.3446\n",
      "6 看_10 7.016967\n",
      "7 看_20 -45.24919\n",
      "8 开_0 -1.2587744\n",
      "9 丢_0 -32.42145\n",
      "______ A 闷_8 0.99050105 \n",
      "\n",
      "('\\xe9\\x97\\xb7', '\\xe9\\x97\\xb7')\n",
      "('\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.1915916)\n",
      "('\\xe9\\x97\\xb7_8', 0.8084083)\n",
      "0 闷_2 -16.976604\n",
      "1 闷_4 -22.90924\n",
      "2 闷_8 28.900192\n",
      "3 闷开_0 14.50318\n",
      "4 看_2 -30.573269\n",
      "5 看_5 37.723774\n",
      "6 看_10 10.706404\n",
      "7 看_20 -38.671055\n",
      "8 开_0 -7.434466\n",
      "9 丢_0 -39.945797\n",
      "('\\xe5\\x8d\\x95', '\\xe5\\x8d\\x95', ['club_2', 'heart_8', 'spade_10'], ['club_3', 'heart_2', 'spade_9'])\n",
      "______ A 闷_8 0.8084083 \n",
      "\n",
      "A_闷_2\n",
      "B_闷_2\n",
      "A_闷_2\n",
      "B_闷_4\n",
      "A_闷_4\n",
      "B_闷_4\n",
      "A_闷_8\n",
      "B_闷_8\n",
      "A_闷_8\n",
      "B_闷_8\n",
      "A_闷_8\n",
      "B_闷_8\n",
      "A_闷_8\n",
      "B_闷_8\n",
      "A_闷_8\n",
      "B_闷_8\n",
      "A_闷_8\n",
      "B_闷开_0\n"
     ]
    }
   ],
   "source": [
    "def print_prob(status):\n",
    "    res = RLModel.sess.run(RLModel.predictions,feed_dict=RLModel._feed_dict(status))\n",
    "    for i,v in  zip(RLModel.action_reverse_index_dicts.keys(),res[0]):\n",
    "        print i, RLModel.action_reverse_index_dicts[i],v\n",
    "import GameEnv\n",
    "gameEnv = GameEnv.GlodenFlower([20000,20000])\n",
    "gameEnv.reset()\n",
    "gameEnv.playerCards[\"A\"] = ['club_2', 'heart_8', 'spade_10']\n",
    "gameEnv.playerCards[\"B\"] = ['club_3', 'heart_2', 'spade_9']\n",
    "print gameEnv.playerCards[\"A\"],gameEnv.playerCards[\"B\"],\"\\n\"\n",
    "playerI = \"A\"\n",
    "observation_this = [[],gameEnv.playerCards[\"A\"],gameEnv.personStatus[\"A\"]]\n",
    "while True:\n",
    "        # DQN 根据观测值选择行为\n",
    "        availble_actions = gameEnv.chooseAvailbleAction(playerI)\n",
    "        action,action_value = RLModel.choose_action_max(np.array([observation_this]),availble_actions,gameEnv.stepNum,True)\n",
    "        print_prob(np.array([observation_this]))\n",
    "        # 环境根据行为给出下一个 state, reward, 是否终止\n",
    "        #print (playerI, action, gameEnv.deskMoney, gameEnv.nowPrice,gameEnv.personStatus[playerI])\n",
    "        observation_next, reward, done = gameEnv.stepA(action,RLModel)\n",
    "\n",
    "        # 将下一个 state_ 变为 下次循环的 state\n",
    "        observation_this = observation_next\n",
    "        print \"______\",playerI,action,action_value,\"\\n\"\n",
    "        # 如果终止, 就跳出循环\n",
    "        if done:\n",
    "            break\n",
    "for i in observation_this[0]:\n",
    "    print i"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 检查闷分布"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "B_闷_2,A_闷_2 (4633.0, 1952) 2.37346311475\n",
      "B_闷_2,A_闷_4 (3926.0, 1860) 2.11075268817\n",
      "B_闷_2,A_闷_8 (226.0, 1242) 0.181964573269\n",
      "B_闷_2,A_闷开_0 (-51.0, 1103) -0.0462375339982\n",
      "B_闷_2,A_看_5 (-938.0, 380) -2.46842105263\n",
      "B_闷_2,A_看_20 (-1864.0, 216) -8.62962962963\n",
      "B_闷_2,A_看_10 (-2061.0, 241) -8.55186721992\n"
     ]
    }
   ],
   "source": [
    "for i in sorted(RLModel.memory_open.items(),key = lambda x:x[1],reverse = True):\n",
    "    if len(i[0].split(\",\")) == 2 and i[0].split(\",\")[0] == \"B_闷_2\":\n",
    "        print i[0],i[1],i[1][0] / float(i[1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A_闷_2,B_看_20,A_闷_8 (1163.0, 71) 16.3802816901\n",
      "A_闷_2,B_看_5,A_闷_4 (700.0, 47) 14.8936170213\n",
      "A_闷_2,B_看_10,A_看_10 (687, 27) 25.4444444444\n",
      "A_闷_2,B_看_10,A_闷_4 (359.0, 35) 10.2571428571\n",
      "A_闷_2,B_闷_8,A_闷_8 (307.0, 123) 2.49593495935\n",
      "A_闷_2,B_看_10,A_闷_8 (259.0, 29) 8.93103448276\n",
      "A_闷_2,B_闷_4,A_闷_4 (148.0, 194) 0.762886597938\n",
      "A_闷_2,B_看_20,A_闷开_0 (114.0, 42) 2.71428571429\n",
      "A_闷_2,B_闷_2,A_闷_2 (99.0, 74) 1.33783783784\n",
      "A_闷_2,B_看_5,A_闷_8 (64.0, 15) 4.26666666667\n",
      "A_闷_2,B_看_10,A_闷开_0 (53.0, 31) 1.70967741935\n",
      "A_闷_2,B_看_20,A_看_20 (37, 5) 7.4\n",
      "A_闷_2,B_看_5,A_看_10 (29, 3) 9.66666666667\n",
      "A_闷_2,B_看_5,A_闷_2 (-2.0, 14) -0.142857142857\n",
      "A_闷_2,B_看_5,A_闷开_0 (-4.0, 14) -0.285714285714\n",
      "A_闷_2,B_看_5,A_看_20 (-5, 4) -1.25\n",
      "A_闷_2,B_看_5,A_看_5 (-10, 3) -3.33333333333\n",
      "A_闷_2,B_闷_2,A_看_10 (-68.0, 14) -4.85714285714\n",
      "A_闷_2,B_闷_8,A_看_20 (-83.0, 21) -3.95238095238\n",
      "A_闷_2,B_闷_2,A_闷开_0 (-111.0, 83) -1.33734939759\n",
      "A_闷_2,B_闷_4,A_看_10 (-112.0, 26) -4.30769230769\n",
      "A_闷_2,B_闷_2,A_闷_8 (-113.0, 77) -1.46753246753\n",
      "A_闷_2,B_闷_2,A_看_20 (-123.0, 15) -8.2\n",
      "A_闷_2,B_闷_8,A_闷开_0 (-136.0, 96) -1.41666666667\n",
      "A_闷_2,B_看_10,A_看_20 (-170, 6) -28.3333333333\n",
      "A_闷_2,B_闷_4,A_闷开_0 (-193.0, 151) -1.27814569536\n",
      "A_闷_2,B_闷_2,A_看_5 (-225.0, 15) -15.0\n",
      "A_闷_2,B_闷_4,A_闷_8 (-288.0, 140) -2.05714285714\n",
      "A_闷_2,B_闷_2,A_闷_4 (-291.0, 81) -3.59259259259\n",
      "A_闷_2,B_闷_4,A_看_20 (-310.0, 18) -17.2222222222\n"
     ]
    }
   ],
   "source": [
    "for i in sorted(RLModel.memory_open.items(),key = lambda x:x[1],reverse = True):\n",
    "    if len(i[0].split(\",\")) == 3 and i[0].split(\",\")[0] == \"A_闷_2\":\n",
    "        print i[0],i[1],i[1][0] / float(i[1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A_闷_2 (5190.0, 7418) 0.699649501213\n",
      "A_闷_4 (3279.0, 11613) 0.282355980367\n",
      "A_闷_8 (-548.0, 6332) -0.0865445356917\n",
      "A_闷开_0 (-2728, 5351) -0.509811250234\n",
      "A_看_5 (-4637.0, 972) -4.77057613169\n",
      "A_看_10 (-5625.0, 805) -6.98757763975\n",
      "A_看_2 (-6773.0, 1698) -3.98881036514\n",
      "A_看_20 (-7145.0, 809) -8.83189122373\n"
     ]
    }
   ],
   "source": [
    "for i in sorted(RLModel.memory_open.items(),key = lambda x:x[1],reverse = True):\n",
    "    if len(i[0].split(\",\")) == 1:\n",
    "        print i[0],i[1],i[1][0] / float(i[1][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8', '\\xe9\\x97\\xb7_8', 7.8],\n",
       " ['B_\\xe9\\x97\\xb7_8', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.47368421052631576],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  17.0],\n",
       " ['B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7_2', 2.037037037037037],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8', '\\xe9\\x97\\xb7_8', 1.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10', '', -1.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '', 10.5],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8', '', -3.0],\n",
       " ['B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', -1.105263157894737],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '', 10.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7_4', 14.6],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -17.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  0.7333333333333333],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7_4', 0.5],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '',\n",
       "  -39.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '', 12.5],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4', '', -23.0],\n",
       " ['B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7_8', 0.3939393939393939],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '', -1.5],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  15.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7_2', 6.5],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2',\n",
       "  '',\n",
       "  2.3333333333333335],\n",
       " ['B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7_4', -3.357142857142857],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -11.0],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  51.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -9.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 5.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  8.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '',\n",
       "  28.0],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  0.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8', '', -2.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7_4', -1.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  13.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  25.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '\\xe9\\x97\\xb7_8', 9.75],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -41.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -17.0],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  24.333333333333332],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4', '', -2.4285714285714284],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -11.0],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '', -7.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  13.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  65.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7_8', 23.666666666666668],\n",
       " ['B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7_4', 6.739130434782608],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  1.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 0.25],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7_8', -11.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20', '\\xe9\\x97\\xb7_8', 18.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  -37.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -24.333333333333332],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  25.0],\n",
       " ['B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  28.333333333333332],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '\\xe9\\x97\\xb7_8', 26.636363636363637],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  71.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -17.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  29.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', -2.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -13.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10', '', 33.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10', '', 5.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  17.0],\n",
       " ['B_\\xe7\\x9c\\x8b_20', '', 12.024390243902438],\n",
       " ['B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7_4', 3.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10', '', -33.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  1.9090909090909092],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  -9.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -11.0],\n",
       " ['B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', -0.45454545454545453],\n",
       " ['B_\\xe7\\x9c\\x8b_2', '\\xe9\\x97\\xb7_4', 2.7],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2',\n",
       "  '\\xe9\\x97\\xb7_2',\n",
       "  30.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  15.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '',\n",
       "  31.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2', '\\xe9\\x97\\xb7_2', -0.7333333333333333],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5', '', -23.0],\n",
       " ['B_\\xe7\\x9c\\x8b_10', '', 6.958333333333333],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '',\n",
       "  25.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4', '', -23.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5', '', -6.833333333333333],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  17.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  2.3333333333333335],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '\\xe9\\x97\\xb7_2',\n",
       "  11.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -21.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7_8', 4.352941176470588],\n",
       " ['B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7_8', -2.12],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  -9.0],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  69.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  10.333333333333334],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -11.0],\n",
       " ['', '', -3.4002998500749624],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  21.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2', '\\xe9\\x97\\xb7_8', 0.8],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  11.5],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -4.333333333333333],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -25.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', -0.6],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '',\n",
       "  -51.0],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10', '', -15.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  36.0],\n",
       " ['B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7_8', 14.538461538461538],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -3.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  12.0],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  33.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10', '', 16.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8', '\\xe9\\x97\\xb7_8', 6.2],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  -37.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  6.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5', '', 5.617647058823529],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  -41.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  13.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  13.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -37.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  61.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7_8', 26.0],\n",
       " ['', '\\xe9\\x97\\xb7_4', 0.6813186813186813],\n",
       " ['B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7_4', 1.9375],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  36.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '',\n",
       "  -22.333333333333332],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8', '', 25.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  1.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  46.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '',\n",
       "  -33.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -25.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  11.0],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  33.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10', '', -1.4615384615384615],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7_8', 0.75],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -19.0],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  15.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -11.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -33.0],\n",
       " ['B_\\xe9\\x97\\xb7_4', '', 0.4716981132075472],\n",
       " ['B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 2.823529411764706],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  71.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  41.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2', '', -1.1710526315789473],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '',\n",
       "  13.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  29.666666666666668],\n",
       " ['B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 2.111111111111111],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  -77.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  45.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  33.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  11.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  23.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -19.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5', '', -23.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  33.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  9.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 7.4],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7_2', 11.75],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -21.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7_4', 9.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '', 9.333333333333334],\n",
       " ['', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', -0.48404255319148937],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -17.0],\n",
       " ['B_\\xe9\\x97\\xb7_2', '', -6.064516129032258],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  11.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -17.0],\n",
       " ['B_\\xe7\\x9c\\x8b_20', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 7.551724137931035],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '', -8.142857142857142],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  2.3333333333333335],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  26.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -15.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20', '', 46.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  3.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8', '', -4.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -25.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4', '', -15.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_4',\n",
       "  16.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '',\n",
       "  -5.666666666666667],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  6.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -13.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  26.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8',\n",
       "  '',\n",
       "  17.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '',\n",
       "  15.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -9.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', 1.4],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '', 8.0],\n",
       " ['B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '', 11.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  -29.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4', '\\xe9\\x97\\xb7_8', 5.0],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  13.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7_8', 17.666666666666668],\n",
       " ['B_\\xe7\\x9c\\x8b_10,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20', '', 31.0],\n",
       " ['', '\\xe9\\x97\\xb7_2', 3.25],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -2.3333333333333335],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -21.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7_4', 31.0],\n",
       " ['B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4', '', -30.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  14.0],\n",
       " ['', '\\xe9\\x97\\xb7_8', 2.9884393063583814],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  13.0],\n",
       " ['B_\\xe7\\x9c\\x8b_20', '\\xe9\\x97\\xb7_8', 18.82857142857143],\n",
       " ['B_\\xe9\\x97\\xb7_8', '', 1.697674418604651],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_10', '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0', -7.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_10',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  16.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  49.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5', '\\xe9\\x97\\xb7_2', 3.8823529411764706],\n",
       " ['B_\\xe7\\x9c\\x8b_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '',\n",
       "  14.666666666666666],\n",
       " ['B_\\xe9\\x97\\xb7_8', '\\xe9\\x97\\xb7_8', 6.317073170731708],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '',\n",
       "  -33.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2', '\\xe9\\x97\\xb7_4', -16.333333333333332],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_8', '', 11.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_5',\n",
       "  '\\xe9\\x97\\xb7_2',\n",
       "  13.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20', '', 36.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20', '', -25.0],\n",
       " ['B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  12.0],\n",
       " ['B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -25.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2', '', -0.7142857142857143],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4',\n",
       "  '',\n",
       "  -27.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8,A_\\xe9\\x97\\xb7_8,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -25.0],\n",
       " ['B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_4',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  7.0],\n",
       " ['B_\\xe7\\x9c\\x8b_5,A_\\xe9\\x97\\xb7_2,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  -11.0],\n",
       " ['A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_4,A_\\xe9\\x97\\xb7_4,B_\\xe7\\x9c\\x8b_20',\n",
       "  '\\xe9\\x97\\xb7_8',\n",
       "  25.0],\n",
       " ['A_\\xe9\\x97\\xb7_8,B_\\xe9\\x97\\xb7_8', '', 4.0],\n",
       " ['A_\\xe9\\x97\\xb7_2,B_\\xe9\\x97\\xb7_2,A_\\xe9\\x97\\xb7_4,B_\\xe9\\x97\\xb7_8',\n",
       "  '\\xe9\\x97\\xb7\\xe5\\xbc\\x80_0',\n",
       "  -15.0]]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "RLModel.experience_open_replay()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0397981\n"
     ]
    }
   ],
   "source": [
    "train_data = [[\"\",\"闷_2\",0.34]]\n",
    "playSequenceStr = [i[0].split(\",\") if i[0] != \"\" else [] for i in train_data]\n",
    "playSequenceIndex = [[RLModel.seq_action_index_dicts[i] if i != \"\" else len(RLModel.seq_action_index_dicts) for i in j] + [len(RLModel.seq_action_index_dicts)] * (len(RLModel.seq_action_index_dicts) - len(j)) for j in playSequenceStr]\n",
    "playSequenceLength = [len(i) + 1 for i in playSequenceStr]\n",
    "actionIndex = [RLModel.action_notsee_index_dicts[i[1]] for i in train_data]\n",
    "yInput = [i[2] for i in train_data]\n",
    "feed_dict = {RLModel.playSequenceInput: np.array(playSequenceIndex),\n",
    "             RLModel.actionInputOpen: RLModel._one_hot(actionIndex,len(RLModel.action_notsee_index_dicts)),RLModel.playSequenceLengthInput: np.array(playSequenceLength),\n",
    "             RLModel.yInputOpen: np.array(yInput)}\n",
    "# for i in range(100):\n",
    "rs, global_step, loss = RLModel.sess.run([RLModel.predictionsNotSee, RLModel.global_steps_open, RLModel.loss_open], feed_dict=feed_dict)\n",
    "\n",
    "print loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 闷_2 0.49433714\n",
      "1 闷_4 5.0349956\n",
      "2 闷_8 1.5542467\n",
      "3 闷开_0 -0.15048584\n",
      "4 看 -4.3465657\n"
     ]
    }
   ],
   "source": [
    "# RLModel.train(RLModel.exerience_replay_final_step())\n",
    "res = RLModel.sess.run(RLModel.predictionsNotSee,feed_dict=RLModel._feed_dict(np.array([[[],  ['heart_4', 'heart_2', 'spade_8'],\"闷\"]])))\n",
    "for i,v in  zip(RLModel.action_notsee_reverse_index_dicts.keys(),res[0]):\n",
    "    print i, RLModel.action_notsee_reverse_index_dicts[i],v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 闷_2 11.790662\n",
      "1 闷_4 11.684034\n",
      "2 闷_8 10.927329\n",
      "3 闷开_0 1.2110952\n",
      "4 看 -3.9434104\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# RLModel.train(RLModel.exerience_replay_final_step())\n",
    "res = RLModel.sess.run(RLModel.predictionsNotSee,feed_dict=RLModel._feed_dict(np.array([[[],  ['heart_4', 'heart_2', 'spade_8'],\"闷\"]])))\n",
    "for i,v in  zip(RLModel.action_notsee_reverse_index_dicts.keys(),res[0]):\n",
    "    print i, RLModel.action_notsee_reverse_index_dicts[i],v"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.16"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}