test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "139ef340",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "WARNING:tensorflow:From /home/ec2-user/anaconda3/envs/tensorflow_p36/cpu/lib/python3.6/site-packages/tensorflow_core/__init__.py:1473: The name tf.estimator.inputs is deprecated. Please use tf.compat.v1.estimator.inputs instead.\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# export openblas_num_threads=1; export OMP_NUM_THREADS=1; python simu.py\n",
    "\n",
    "from utils import * \n",
    "from weight import * \n",
    "import simu_funs\n",
    "from main import *\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1df77eb1",
   "metadata": {},
   "source": [
    "## Initialize Env"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a9ffa044",
   "metadata": {},
   "outputs": [],
   "source": [
    "a = now()\n",
    "##########################################################################################################################################################\n",
    "\n",
    "rep_times = 100\n",
    "region_parallel = False\n",
    "full_parallel = False\n",
    "pattern_seed = 2\n",
    "sd_u_O = 25\n",
    "w_O = .5\n",
    "w_A = 1.5\n",
    "u_D_range = [80] \n",
    "thre_range = [100, 101, 105, 110]\n",
    "\n",
    "penalty_range = [[3e-4, 1e-4, 5e-5], [3e-4, 1e-4, 5e-5]]\n",
    "with_IS = True\n",
    "n_layer = 3\n",
    "max_iteration = 1001\n",
    "Learning_rate = 5e-4\n",
    "w_hidden = 30\n",
    "batch_size = 32\n",
    "\n",
    "gpus = tf.config.experimental.list_physical_devices('GPU')\n",
    "if gpus:\n",
    "    try:\n",
    "        for gpu in gpus:\n",
    "            tf.config.experimental.set_memory_growth(gpu, True)\n",
    "    except RuntimeError as e:\n",
    "        print(e)\n",
    "tf.keras.backend.set_floatx('float64')\n",
    "\n",
    "import os\n",
    "os.environ[\"OMP_NUM_THREADS\"] = \"1\"\n",
    "os.environ[\"OPENBLAS_NUM_THREADS\"] = \"1\"\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "b1fbf03f",
   "metadata": {},
   "outputs": [],
   "source": [
    "with_NO_MARL = False\n",
    "with_MF = False\n",
    "\n",
    "l = 5\n",
    "t_func = None\n",
    "sd_D = None\n",
    "sd_O = None\n",
    "CV_QV = True\n",
    "penalty_NMF = [[1e-3], [1e-3]]\n",
    "dim_S_plus_Ts = 3 + 3\n",
    "epsilon = 1e-6\n",
    "penalty = penalty_range\n",
    "inner_parallel = region_parallel\n",
    "\n",
    "n_gpu = 8\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "28f13cbd",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[thre_range, sd_R_range, day_range, penalty_range]:  [[100, 101, 105, 110], [0, 5, 10, 15, 20, 25, 30], [7], [[0.0003, 0.0001, 5e-05], [0.0003, 0.0001, 5e-05]]]\n",
      "\n",
      "--------------------------------------\n",
      "[pattern_seed, day, sd_R, u_D] = [2, 7, 0, 80]\n",
      "\n",
      "max(u_O) =  157.3 mean(u_O) =  93.7\n",
      "O_threshold = 100\n",
      "means of Order: \n",
      "\n",
      "89.6 98.6 46.6 141.0 55.2 \n",
      "\n",
      "79.0 112.6 68.9 73.6 77.3 \n",
      "\n",
      "113.8 157.3 101.0 72.1 113.5 \n",
      "\n",
      "85.1 99.5 129.4 81.3 100.2 \n",
      "\n",
      "78.0 96.1 106.4 75.3 91.5 \n",
      "\n",
      "target policy: \n",
      "\n",
      "0 0 0 1 0 \n",
      "\n",
      "0 1 0 0 0 \n",
      "\n",
      "1 1 1 0 1 \n",
      "\n",
      "0 0 1 0 1 \n",
      "\n",
      "0 0 1 0 0 \n",
      "\n",
      "number of reward locations:  9\n",
      "O_threshold = 101\n",
      "number of reward locations:  8\n",
      "O_threshold = 105\n",
      "number of reward locations:  7\n",
      "O_threshold = 110\n",
      "number of reward locations:  6\n"
     ]
    }
   ],
   "source": [
    "aim = \"final_sd\"\n",
    "\n",
    "sd_R_range = [0, 5, 10, 15, 20, 25, 30]\n",
    "day_range = [7]\n",
    "print(\"[thre_range, sd_R_range, day_range, penalty_range]: \", [thre_range, sd_R_range, day_range, penalty_range])\n",
    "results, res_real = [], []\n",
    "\n",
    "u_D = 80\n",
    "day = 7\n",
    "T = day * 48\n",
    "\n",
    "for sd_R in sd_R_range: # vary\n",
    "    print_setting = DASH + \"[pattern_seed, day, sd_R, u_D] = \" + str([pattern_seed, day, sd_R, u_D]) + \"\\n\"\n",
    "    print(print_setting)\n",
    "    \n",
    "    ####################################################################################################################################################################################\n",
    "    np.random.seed(pattern_seed)\n",
    "    u_O = rnorm(100, sd_u_O, l**2) \n",
    "\n",
    "    print(\"max(u_O) = \", np.round(max(u_O), 1), \"mean(u_O) = \", np.round(np.mean(u_O), 1))\n",
    "\n",
    "    # generate the corresponding target plicy\n",
    "    target_policys = []\n",
    "    n_tp = len(thre_range)\n",
    "    for i in range(n_tp):\n",
    "        O_thre = thre_range[i]\n",
    "        print(\"O_threshold = \" + str(thre_range[i]))\n",
    "        if i == 0: \n",
    "            target_policy = simu_DGP.simu_target_policy_pattern(l = l, u_O = u_O, threshold =  O_thre, print_flag = \"all\") # \"all\"\n",
    "        else:\n",
    "            target_policy = simu_DGP.simu_target_policy_pattern(l = l, u_O = u_O, threshold =  O_thre, print_flag = \"None\") # \"policy_only\"\n",
    "        target_policys.append(target_policy)\n",
    "\n",
    "    # generate the adj for the grid\n",
    "    neigh = adj2neigh(getAdjGrid(l))\n",
    "    break"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c5fa65e5",
   "metadata": {},
   "source": [
    "## Simulate Data (for a given region i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d64a33e6",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/100 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  1%|          | 1/100 [00:00<01:18,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 2/100 [00:01<01:18,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|▎         | 3/100 [00:02<01:25,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 3\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  4%|▍         | 4/100 [00:03<01:21,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 4\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|▌         | 5/100 [00:04<01:19,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 5\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|▌         | 6/100 [00:04<01:17,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 6\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 7/100 [00:05<01:15,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 7\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|▊         | 8/100 [00:06<01:19,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 8\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|▉         | 9/100 [00:07<01:16,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 9\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 10%|█         | 10/100 [00:08<01:14,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 10\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|█         | 11/100 [00:09<01:12,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 11\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|█▏        | 12/100 [00:09<01:10,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 12\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 13/100 [00:10<01:09,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 13\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█▍        | 14/100 [00:11<01:13,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 14\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|█▌        | 15/100 [00:12<01:11,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 15\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 16%|█▌        | 16/100 [00:13<01:09,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 16\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|█▋        | 17/100 [00:14<01:07,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 17\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█▊        | 18/100 [00:14<01:06,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 18\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 19%|█▉        | 19/100 [00:15<01:08,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 19\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 20/100 [00:16<01:06,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 20\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|██        | 21/100 [00:17<01:04,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 21\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 22%|██▏       | 22/100 [00:18<01:02,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 22\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|██▎       | 23/100 [00:18<01:01,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 23\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|██▍       | 24/100 [00:19<01:03,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 24\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 25%|██▌       | 25/100 [00:20<01:01,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 25\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██▌       | 26/100 [00:21<00:59,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 26\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 27%|██▋       | 27/100 [00:22<00:58,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 27\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|██▊       | 28/100 [00:23<00:57,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 28\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|██▉       | 29/100 [00:23<00:58,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 29\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 30%|███       | 30/100 [00:24<00:56,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 30\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|███       | 31/100 [00:25<00:55,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 31\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|███▏      | 32/100 [00:26<00:53,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 32\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 33%|███▎      | 33/100 [00:26<00:52,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 33\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███▍      | 34/100 [00:27<00:54,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 34\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▌      | 35/100 [00:28<00:52,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 35\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 36%|███▌      | 36/100 [00:29<00:50,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 36\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|███▋      | 37/100 [00:30<00:50,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 37\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|███▊      | 38/100 [00:30<00:48,  1.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 38\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 39%|███▉      | 39/100 [00:31<00:51,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 39\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 40/100 [00:32<00:49,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 40\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████      | 41/100 [00:33<00:48,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 41\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 42%|████▏     | 42/100 [00:34<00:46,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 42\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|████▎     | 43/100 [00:35<00:45,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 43\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|████▍     | 44/100 [00:36<00:47,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 44\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 45%|████▌     | 45/100 [00:36<00:45,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 45\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████▌     | 46/100 [00:37<00:44,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 46\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|████▋     | 47/100 [00:38<00:42,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 47\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 48%|████▊     | 48/100 [00:39<00:41,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 48\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|████▉     | 49/100 [00:40<00:43,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 49\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 50/100 [00:41<00:42,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 50\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 51%|█████     | 51/100 [00:41<00:40,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 51\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|█████▏    | 52/100 [00:42<00:38,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 52\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 53/100 [00:43<00:37,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 53\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████▍    | 54/100 [00:44<00:38,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 54\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|█████▌    | 55/100 [00:45<00:37,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 55\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 56%|█████▌    | 56/100 [00:45<00:36,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 56\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|█████▋    | 57/100 [00:46<00:35,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 57\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|█████▊    | 58/100 [00:47<00:33,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 58\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 59%|█████▉    | 59/100 [00:48<00:35,  1.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 59\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 60/100 [00:49<00:33,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 60\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████    | 61/100 [00:50<00:32,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 61\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 62%|██████▏   | 62/100 [00:50<00:31,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 62\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 63/100 [00:51<00:30,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 63\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|██████▍   | 64/100 [00:52<00:30,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 64\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 65%|██████▌   | 65/100 [00:53<00:29,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 65\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████▌   | 66/100 [00:54<00:27,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 66\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|██████▋   | 67/100 [00:54<00:26,  1.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 67\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 68%|██████▊   | 68/100 [00:55<00:25,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 68\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████▉   | 69/100 [00:56<00:28,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 69\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|███████   | 70/100 [00:57<00:26,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 70\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 71%|███████   | 71/100 [00:58<00:25,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 71\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|███████▏  | 72/100 [00:59<00:23,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 72\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████▎  | 73/100 [01:00<00:22,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 73\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|███████▍  | 74/100 [01:01<00:22,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 74\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|███████▌  | 75/100 [01:01<00:21,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 75\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████▌  | 76/100 [01:02<00:20,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 76\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 77%|███████▋  | 77/100 [01:03<00:18,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 77\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|███████▊  | 78/100 [01:04<00:17,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 78\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████▉  | 79/100 [01:05<00:17,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 79\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 80/100 [01:06<00:16,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 80\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████  | 81/100 [01:06<00:15,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 81\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|████████▏ | 82/100 [01:07<00:14,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████▎ | 83/100 [01:08<00:13,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|████████▍ | 84/100 [01:09<00:13,  1.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████▌ | 85/100 [01:10<00:13,  1.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████▌ | 86/100 [01:11<00:11,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 86\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|████████▋ | 87/100 [01:11<00:10,  1.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 87\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████▊ | 88/100 [01:12<00:09,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 88\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|████████▉ | 89/100 [01:13<00:08,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 89\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|█████████ | 90/100 [01:14<00:08,  1.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 90\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|█████████ | 91/100 [01:15<00:07,  1.17it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 91\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████▏| 92/100 [01:16<00:06,  1.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 92\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████▎| 93/100 [01:16<00:05,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 93\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████▍| 94/100 [01:17<00:04,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 94\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|█████████▌| 95/100 [01:18<00:04,  1.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 95\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|█████████▌| 96/100 [01:19<00:03,  1.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 96\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████▋| 97/100 [01:20<00:02,  1.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 97\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|█████████▊| 98/100 [01:20<00:01,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 98\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████▉| 99/100 [01:21<00:00,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "seed = 99\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [01:22<00:00,  1.21it/s]\n"
     ]
    }
   ],
   "source": [
    "# reg_data = n_region * n_days * [12 components]\n",
    "# [S_it, A_it, R_it, Tsit, Tait,  # 0 - 4\n",
    "#                     S_i(t+1), Tsi(t+1), pi_Sit_1, T_ait_1_pi, # 5 - 8 : pi_Sit_1 = \\pi(S_{i, t + 1}), T_ait_1_pi = T_{a, i, t + 1}(\\pi)\n",
    "#                     A_it1, pi_Sit, T_ait_pi] # 9 - 11\n",
    "data_test_reps = []\n",
    "for seed in tqdm(range(100)):\n",
    "    reload(simu_funs)\n",
    "    data, adj_mat, details, reg_data = simu_funs.simu_once(seed = seed, l = l, T = T, t_func = t_func,  u_D = u_D,\n",
    "                             u_O = u_O,  \n",
    "                             target_policys = target_policys, w_A = w_A, w_O = w_O, dim_S_plus_Ts = dim_S_plus_Ts,  \n",
    "                              penalty = penalty, penalty_NMF = penalty_NMF, \n",
    "                             n_layer = n_layer, sd_D = sd_D, sd_R = sd_R, sd_O = sd_O, \n",
    "                              w_hidden = w_hidden, Learning_rate = Learning_rate,  CV_QV = CV_QV, \n",
    "                              batch_size = batch_size, max_iteration = max_iteration, epsilon = epsilon, \n",
    "                             with_MF = with_MF, with_NO_MARL = with_NO_MARL, with_IS = with_IS, \n",
    "                               inner_parallel = inner_parallel, gpu_number = seed % 8\n",
    "                                , return_datail_only = True)\n",
    "    i = 1 # region id\n",
    "    X = []\n",
    "    A = []\n",
    "    R = []\n",
    "    for t in range(len(reg_data[i])):\n",
    "        s = np.concatenate([reg_data[i][t][0], reg_data[i][t][3], [reg_data[i][t][4]]])\n",
    "        a = reg_data[i][t][1]\n",
    "        r = reg_data[i][t][2]\n",
    "        X.append(s)\n",
    "        R.append(r)\n",
    "        A.append(a)\n",
    "    X = arr(X)\n",
    "    A = arr(A)\n",
    "    R = arr(R)\n",
    "    R = np.expand_dims(R, 1)\n",
    "    A = np.expand_dims(A, 1)\n",
    "    if include_reward:\n",
    "        data_test_reps.append([[X[:(335 // 2)], A[:(335 // 2)], R[:(335 // 2)]]\n",
    "                           , [X[(335 // 2 + 1):], A[(335 // 2 + 1):], R[(335 // 2 + 1):]]])\n",
    "    else:\n",
    "        data_test_reps.append([[X[:(335 // 2)], A[:(335 // 2)]], [X[(335 // 2 + 1):], A[(335 // 2 + 1):]]])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efc9fd16",
   "metadata": {},
   "source": [
    "## Run the test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f9d9b6d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "include_reward = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b41e2f7c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import _core_test_fun\n",
    "reload(_core_test_fun)\n",
    "def get_one_p_value(seed):\n",
    "    test1 = _core_test_fun.MarkovTester(data_test_reps[seed], J = 1,\n",
    "            B = 100, Q = 2, L = 2, \n",
    "            paras= [3, 10], n_trees = 50, \n",
    "            print_time = False,\n",
    "            include_reward = include_reward)\n",
    "    return test1.pValues"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "conda_tensorflow_p36",
   "language": "python",
   "name": "conda_tensorflow_p36"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}