batch_eval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2022-05-31 18:43:39,756  29 MainThread INFO     numexpr.utils:147 - _init_num_threads: Note: NumExpr detected 12 cores but \"NUMEXPR_MAX_THREADS\" not set, so enforcing safe limit of 8.\n",
      "2022-05-31 18:43:39,757  29 MainThread INFO     numexpr.utils:159 - _init_num_threads: NumExpr defaulting to 8 threads.\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "#import scipy.io\n",
    "import argparse\n",
    "from pinn.neural_net import MLP\n",
    "from pinn.get_points import normalize\n",
    "from pinn.util import log\n",
    "from pathlib import Path\n",
    "import matplotlib.pyplot as plt\n",
    "from pinn.evaluate import get_pars, evaluate, load_data, get_residuals, get_loss\n",
    "import pandas as pd\n",
    "import glob\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#cases = ['model_nf1000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle0.pt',\n",
    "#         'model_nf1000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle1.pt',\n",
    "#         'model_nf3000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle0.pt',\n",
    "#         'model_nf3000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle1.pt',\n",
    "#         'model_nf10000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle0.pt',\n",
    "#         'model_nf10000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle1.pt',\n",
    "#         'model_nf30000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle0.pt',\n",
    "#         'model_nf30000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle1.pt',\n",
    "#         'model_nf100000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle0.pt',\n",
    "#         'model_nf100000_nb1000_ns0_fn1-0-0_fs0-1-1_shuffle1.pt',\n",
    "#         'model_nf10000_nb1000_ns0_fn1-10-5_fs0-2-1_shuffle0.pt',\n",
    "#         'model_nf10000_nb1000_ns0_fn1-20-10_fs0-2-1_shuffle0.pt'\n",
    "#         ]\n",
    "\n",
    "cases = [f for f in glob.glob(\"models/*.pt\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def summarize(case):\n",
    "    nt = 51\n",
    "    nx = 101\n",
    "    ny = 11\n",
    "\n",
    "    t = np.linspace(0,12*3600,nt)\n",
    "    x = np.linspace(0,10000,nx)\n",
    "    y = np.linspace(0,250,ny)\n",
    "\n",
    "\n",
    "    [E, U, V, t_grid, x_grid, y_grid, X] = evaluate(t,x,y,case)\n",
    "\n",
    "    [r1, r2, r3, t_grid, x_grid, y_grid, X] = get_residuals(t,x,y,case)\n",
    "    R_rms = np.sqrt(np.sum(r1**2) + np.sum(r2**2) + np.sum(r3**2))/(12*3600*np.sqrt(nt*nx*ny))\n",
    "    R_abs = (np.sum(np.abs(r1)) + np.sum(np.abs(r2)) + np.sum(np.abs(r3)))/(12*3600*nt*nx*ny)\n",
    "\n",
    "    m = torch.load(case)\n",
    "\n",
    "    log_loss = m['loss']\n",
    "    time = m['time']\n",
    "    memory = m['memory']\n",
    "    loss = log_loss[-1,0]\n",
    "    iter = log_loss.shape[0]\n",
    "    U_max = np.max(U)\n",
    "\n",
    "    #return {'case':case,'t':t,'x':x,'y':y,'E':E,'U':U,'V':V,'R_rms':R_rms,'R_abs':R_abs,'loss':log_loss,'time':time,'memory':memory}\n",
    "    return {'case':case,'R_rms':R_rms,'R_abs':R_abs,'iter':iter,'loss':loss,'log_loss':log_loss,'time':time,'memory':memory,'U_max':U_max}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "S = pd.DataFrame.from_dict([summarize(case) for case in cases])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>case</th>\n",
       "      <th>memory</th>\n",
       "      <th>time</th>\n",
       "      <th>R_rms</th>\n",
       "      <th>R_abs</th>\n",
       "      <th>loss</th>\n",
       "      <th>iter</th>\n",
       "      <th>U_max</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>models/model_nf3000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>102.726074</td>\n",
       "      <td>8427.299298</td>\n",
       "      <td>0.000112</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.082461</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>models/model_nf30000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>982.700195</td>\n",
       "      <td>9773.714826</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000056</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.124976</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>models/model_nf3000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed2_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>102.726074</td>\n",
       "      <td>8415.405435</td>\n",
       "      <td>0.000093</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000069</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.074140</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>models/model_nf1000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed2_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>37.545898</td>\n",
       "      <td>8419.702273</td>\n",
       "      <td>0.000213</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>0.000067</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.053187</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>models/model_nf300_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>14.750000</td>\n",
       "      <td>6999.211867</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>0.000139</td>\n",
       "      <td>0.000051</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.025798</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>models/model_nf3000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>102.726074</td>\n",
       "      <td>8780.512274</td>\n",
       "      <td>0.000090</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000066</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.089951</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>models/model_nf1000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>37.545898</td>\n",
       "      <td>8858.075607</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.000086</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.064354</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>models/model_nf300_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>14.750000</td>\n",
       "      <td>7536.853683</td>\n",
       "      <td>0.000128</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>0.000093</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.034265</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>models/model_nf1000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>37.545898</td>\n",
       "      <td>8411.347518</td>\n",
       "      <td>0.000190</td>\n",
       "      <td>0.000113</td>\n",
       "      <td>0.000072</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.045581</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>models/model_nf300_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed2_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>14.750000</td>\n",
       "      <td>6984.302615</td>\n",
       "      <td>0.000280</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>0.000060</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.024662</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>models/model_nf100000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>3275.965332</td>\n",
       "      <td>16811.405370</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000086</td>\n",
       "      <td>0.000048</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.123549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>models/model_nf30000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt</td>\n",
       "      <td>982.700195</td>\n",
       "      <td>9693.894631</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.000054</td>\n",
       "      <td>49999</td>\n",
       "      <td>0.125932</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                  case  \\\n",
       "0     models/model_nf3000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt   \n",
       "1    models/model_nf30000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt   \n",
       "2     models/model_nf3000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed2_adam_lr0.001_loss_l1.pt   \n",
       "3     models/model_nf1000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed2_adam_lr0.001_loss_l1.pt   \n",
       "4      models/model_nf300_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt   \n",
       "5     models/model_nf3000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt   \n",
       "6     models/model_nf1000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt   \n",
       "7      models/model_nf300_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle1_seed1_adam_lr0.001_loss_l1.pt   \n",
       "8     models/model_nf1000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt   \n",
       "9      models/model_nf300_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed2_adam_lr0.001_loss_l1.pt   \n",
       "10  models/model_nf100000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt   \n",
       "11   models/model_nf30000_nb1000_ns0nf_delta10000_fn1-0-0_fs0-1-1_MLPRes_2x2x20_shuffle0_seed1_adam_lr0.001_loss_l1.pt   \n",
       "\n",
       "         memory          time     R_rms     R_abs      loss   iter     U_max  \n",
       "0    102.726074   8427.299298  0.000112  0.000090  0.000067  49999  0.082461  \n",
       "1    982.700195   9773.714826  0.000084  0.000091  0.000056  49999  0.124976  \n",
       "2    102.726074   8415.405435  0.000093  0.000088  0.000069  49999  0.074140  \n",
       "3     37.545898   8419.702273  0.000213  0.000118  0.000067  49999  0.053187  \n",
       "4     14.750000   6999.211867  0.000283  0.000139  0.000051  49999  0.025798  \n",
       "5    102.726074   8780.512274  0.000090  0.000088  0.000066  49999  0.089951  \n",
       "6     37.545898   8858.075607  0.000131  0.000103  0.000086  49999  0.064354  \n",
       "7     14.750000   7536.853683  0.000128  0.000099  0.000093  49999  0.034265  \n",
       "8     37.545898   8411.347518  0.000190  0.000113  0.000072  49999  0.045581  \n",
       "9     14.750000   6984.302615  0.000280  0.000145  0.000060  49999  0.024662  \n",
       "10  3275.965332  16811.405370  0.000081  0.000086  0.000048  49999  0.123549  \n",
       "11   982.700195   9693.894631  0.000083  0.000089  0.000054  49999  0.125932  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', 120)\n",
    "S[['case','memory','time','R_rms','R_abs','loss','iter','U_max']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n",
      "/tmp/ipykernel_29110/921812561.py:13: FutureWarning: The frame.append method is deprecated and will be removed from pandas in a future version. Use pandas.concat instead.\n",
      "  R = R.append({'nf':nf,'shuffle':shuffle,\n"
     ]
    }
   ],
   "source": [
    "R = pd.DataFrame(columns=['nf','shuffle','R_rms','R_abs','R_rms_min','R_rms_max','R_abs_min','R_abs_max','time','time_min','time_max','memory','n_runs'])\n",
    "\n",
    "for ind, l in S.iterrows():\n",
    "    nf = int(re.search('nf(\\d+)', l['case']).group(1))\n",
    "    shuffle = int(re.search('shuffle(\\d+)', l['case']).group(1))\n",
    "\n",
    "    known = -1\n",
    "    for i, (NF, SHUFFLE) in enumerate(zip(R.nf.tolist(), R.shuffle.tolist())):\n",
    "        if nf == NF and shuffle == SHUFFLE:\n",
    "            known = i\n",
    "    \n",
    "    if known < 0:\n",
    "        R = R.append({'nf':nf,'shuffle':shuffle,\n",
    "                    'R_rms':l['R_rms'],'R_abs':l['R_abs'],\n",
    "                    'R_rms_min':l['R_rms'],'R_rms_max':l['R_rms'],\n",
    "                    'R_abs_min':l['R_abs'],'R_abs_max':l['R_abs'],\n",
    "                    'time':l['time'],'time_min':l['time'],'time_max':l['time'],\n",
    "                    'memory':l['memory'],'n_runs':1},ignore_index=True)\n",
    "    else:\n",
    "        current = R.loc[known]\n",
    "        current.R_rms = (current.n_runs * current.R_rms + l.R_rms)/(current.n_runs+1)\n",
    "        nr = R.loc[known,'n_runs']\n",
    "        R.loc[known,'n_runs'] = nr + 1\n",
    "        R.loc[known,'R_rms'] = (R.loc[known,'R_rms']*nr + l['R_rms'])/(nr+1)\n",
    "        R.loc[known,'R_rms_min'] = min(R.loc[known,'R_rms_min'], l['R_rms'])\n",
    "        R.loc[known,'R_rms_max'] = max(R.loc[known,'R_rms_max'], l['R_rms'])\n",
    "        R.loc[known,'R_abs'] = (R.loc[known,'R_abs']*nr + l['R_abs'])/(nr+1)\n",
    "        R.loc[known,'R_abs_min'] = min(R.loc[known,'R_abs_min'], l['R_abs'])\n",
    "        R.loc[known,'R_abs_max'] = max(R.loc[known,'R_abs_max'], l['R_abs'])\n",
    "        R.loc[known,'time'] = (R.loc[known,'time']*nr + l['time'])/(nr+1)\n",
    "        R.loc[known,'time_min'] = min(R.loc[known,'time_min'], l['time'])\n",
    "        R.loc[known,'time_max'] = max(R.loc[known,'time_max'], l['time'])\n",
    "\n",
    "R =R.sort_values(['nf','shuffle'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nf</th>\n",
       "      <th>shuffle</th>\n",
       "      <th>R_rms</th>\n",
       "      <th>R_abs</th>\n",
       "      <th>R_rms_min</th>\n",
       "      <th>R_rms_max</th>\n",
       "      <th>R_abs_min</th>\n",
       "      <th>R_abs_max</th>\n",
       "      <th>time</th>\n",
       "      <th>time_min</th>\n",
       "      <th>time_max</th>\n",
       "      <th>memory</th>\n",
       "      <th>n_runs</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>300.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000281</td>\n",
       "      <td>0.000142</td>\n",
       "      <td>0.00028</td>\n",
       "      <td>0.000283</td>\n",
       "      <td>0.000139</td>\n",
       "      <td>0.000145</td>\n",
       "      <td>6991.757241</td>\n",
       "      <td>6984.302615</td>\n",
       "      <td>6999.211867</td>\n",
       "      <td>14.75</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>300.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000128</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>0.000128</td>\n",
       "      <td>0.000128</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>0.000099</td>\n",
       "      <td>7536.853683</td>\n",
       "      <td>7536.853683</td>\n",
       "      <td>7536.853683</td>\n",
       "      <td>14.75</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000196</td>\n",
       "      <td>0.000116</td>\n",
       "      <td>0.00019</td>\n",
       "      <td>0.000213</td>\n",
       "      <td>0.000113</td>\n",
       "      <td>0.000118</td>\n",
       "      <td>8415.524896</td>\n",
       "      <td>8411.347518</td>\n",
       "      <td>8419.702273</td>\n",
       "      <td>37.545898</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>1000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0.000131</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>0.000103</td>\n",
       "      <td>8858.075607</td>\n",
       "      <td>8858.075607</td>\n",
       "      <td>8858.075607</td>\n",
       "      <td>37.545898</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000098</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.000093</td>\n",
       "      <td>0.000112</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.00009</td>\n",
       "      <td>8421.352367</td>\n",
       "      <td>8415.405435</td>\n",
       "      <td>8427.299298</td>\n",
       "      <td>102.726074</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.00009</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.00009</td>\n",
       "      <td>0.00009</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>0.000088</td>\n",
       "      <td>8780.512274</td>\n",
       "      <td>8780.512274</td>\n",
       "      <td>8780.512274</td>\n",
       "      <td>102.726074</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>30000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000083</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>0.000089</td>\n",
       "      <td>9693.894631</td>\n",
       "      <td>9693.894631</td>\n",
       "      <td>9693.894631</td>\n",
       "      <td>982.700195</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>30000.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000084</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>0.000091</td>\n",
       "      <td>9773.714826</td>\n",
       "      <td>9773.714826</td>\n",
       "      <td>9773.714826</td>\n",
       "      <td>982.700195</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>100000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000086</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000081</td>\n",
       "      <td>0.000086</td>\n",
       "      <td>0.000086</td>\n",
       "      <td>16811.40537</td>\n",
       "      <td>16811.40537</td>\n",
       "      <td>16811.40537</td>\n",
       "      <td>3275.965332</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         nf shuffle     R_rms     R_abs R_rms_min R_rms_max R_abs_min  \\\n",
       "3     300.0     0.0  0.000281  0.000142   0.00028  0.000283  0.000139   \n",
       "6     300.0     1.0  0.000128  0.000099  0.000128  0.000128  0.000099   \n",
       "2    1000.0     0.0  0.000196  0.000116   0.00019  0.000213  0.000113   \n",
       "5    1000.0     1.0  0.000131  0.000103  0.000131  0.000131  0.000103   \n",
       "0    3000.0     0.0  0.000098  0.000089  0.000093  0.000112  0.000088   \n",
       "4    3000.0     1.0   0.00009  0.000088   0.00009   0.00009  0.000088   \n",
       "8   30000.0     0.0  0.000083  0.000089  0.000083  0.000083  0.000089   \n",
       "1   30000.0     1.0  0.000084  0.000091  0.000084  0.000084  0.000091   \n",
       "7  100000.0     0.0  0.000081  0.000086  0.000081  0.000081  0.000086   \n",
       "\n",
       "  R_abs_max         time     time_min     time_max       memory n_runs  \n",
       "3  0.000145  6991.757241  6984.302615  6999.211867        14.75    2.0  \n",
       "6  0.000099  7536.853683  7536.853683  7536.853683        14.75    1.0  \n",
       "2  0.000118  8415.524896  8411.347518  8419.702273    37.545898    2.0  \n",
       "5  0.000103  8858.075607  8858.075607  8858.075607    37.545898    1.0  \n",
       "0   0.00009  8421.352367  8415.405435  8427.299298   102.726074    2.0  \n",
       "4  0.000088  8780.512274  8780.512274  8780.512274   102.726074    1.0  \n",
       "8  0.000089  9693.894631  9693.894631  9693.894631   982.700195    1.0  \n",
       "1  0.000091  9773.714826  9773.714826  9773.714826   982.700195    1.0  \n",
       "7  0.000086  16811.40537  16811.40537  16811.40537  3275.965332    1.0  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "R"
   ]
  }
 ],
 "metadata": {
  "interpreter": {
   "hash": "1b9cb75973010bcd25b76ff5d71e3fbf8897c5f3014732e61f061ced28fe6b32"
  },
  "kernelspec": {
   "display_name": "Python 3.9.12 ('torch39')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}