Module16/16_simple_batchnorm_eval.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "15a4c593",
   "metadata": {},
   "source": [
    "# Playing with Batchnorm without any training...\n",
    "\n",
    "How do you explain these figures?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "ca7ca78d",
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import torch\n",
    "from torch import nn\n",
    "\n",
    "torch.manual_seed(42);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "88ceb159",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAYZElEQVR4nO3df3BUZZ7v8ffXgER+XH5mKSR4g7OU6AAJECOKP7hy9SpOCSo6jowSlxlqtpytdWrXWRyrvNGypphbrA4Wt7QyowNMqRNlV02ptS7+oJRBccLIKAx6id5YCYPAhB+rI4jod//oJ9kG86OTdPfpPHxeVV055zmnu785dD48/fTp55i7IyIicTkl6QJERCT7FO4iIhFSuIuIREjhLiISIYW7iEiEBiRdAMCYMWO8rKws6TJERPqVLVu2/NndSzraVhDhXlZWRkNDQ9JliIj0K2b2UWfbNCwjIhIhhbuISIQU7iIiESqIMXcR6Z0vvviClpYWjhw5knQpkkPFxcWUlpYycODAjO+jcBfpx1paWhg2bBhlZWWYWdLlSA64O62trbS0tDBx4sSM76dhGZF+7MiRI4wePVrBHjEzY/To0T1+d6ZwF+nnFOzx682/scJdRCRCGnMXiUjZsuez+nhNy6/qdp8LLriATZs2Zfd5m5rYtGkTN910U4fb77jjDl544QXmzZvHN77xDQYPHswtt9yS0WNv3bqVP/3pT8ybN+9r21avXk1DQwOrVq3KuNaf/vSn/OQnP+lyn+rqar71rW+xcOHCjB+3rxTuMaoZDjWHkq6iIHUWfpmEmHQs28EOqXB//PHHOw332tpa9u/fT1FRUaePcezYMQYM+HrEbd26lYaGhg7DvTcyCfckaFhGRPpk6NChAGzYsIE5c+awcOFCJk+ezKJFi2i70ltZWRk//vGPmTp1KlVVVTQ2NgKpHu26deu+9ljLli3j9ddfp6KiggceeOC457v66qv59NNPmTlzJnV1ddTU1LBixQoA5syZw+23305lZSUrV67kqaeeYsqUKZSXl3PxxRdz9OhR7r77burq6qioqKCuru5rv09zczNz5sxh0qRJ3HPPPe3tCxYsYObMmXzzm9+ktra2vc7Dhw9TUVHBokWLAFi7di3Tpk2jvLycm2++uf3+r732GhdccAFnnnnmcb9zrqjnLiJZ8/bbb7N9+3ZOP/10Zs+ezW9/+1suvPBCAIYPH867777L2rVruf3223nuuec6fZzly5ezYsWKDvepr69n6NChbN26FYCamprjth89erR9rqqpU6fy4osvMn78eA4ePMipp57Kvffe2+XQy1tvvcW2bdsYPHgw5557LldddRWVlZU8+uijjBo1isOHD3Puuedy3XXXsXz5clatWtVey/bt27nvvvvYtGkTY8aMYf/+/e2Pu3v3bjZu3Mh7773H1VdfnfMhGvXcRSRrqqqqKC0t5ZRTTqGiooKmpqb2bd/5znfaf77xxhs5q+Hb3/52+/Ls2bOprq7mF7/4BV9++WVG97/ssssYPXo0p512Gtdeey0bN24E4MEHH6S8vJxZs2bR3NzMzp07v3bfV155heuvv54xY8YAMGrUqPZtCxYs4JRTTuGcc85hz549ffkVM6Keu4hkzaBBg9qXi4qKOHbsWPt6+ul8bcsDBgzgq6++AuCrr77i6NGjfa5hyJAh7csPP/wwmzdv5vnnn2fmzJls2bKl2/ufeNqhmbFhwwZeeukl3njjDQYPHsycOXN6fN55+rFpG67KJfXcY1YzPOkK+r2yZc93epOeaRvfrqur4/zzzwdSY/FtgVtfX88XX3wBwLBhw/jkk0/6/JwffPAB5513Hvfeey8lJSU0Nzd3+9jr169n//79HD58mGeeeYbZs2dz6NAhRo4cyeDBg3nvvfd488032/cfOHBge92XXnopTz31FK2trQDHDcvkm3rusVKw90gsYV3IZ/0cOHCAadOmMWjQIJ544gkAvv/97zN//nzKy8u54oor2nvd06ZNo6ioiPLycqqrq/nRj37Uq+e844472LlzJ+7O3LlzKS8v54wzzmD58uVUVFRw5513HjeMA6mhpeuuu46Wlha++93vUllZydSpU3n44Yc5++yzOeuss5g1a1b7/kuXLmXatGnMmDGDxx57jLvuuotLLrmEoqIipk+fzurVq3t3wPrIMnl7YGYjgF8CUwAH/gZ4H6gDyoAm4AZ3P2Cp9zQrgXnAZ0C1u/++q8evrKx0Xawji9KDXadEHiebIV4IQbpjxw7OPvvspMvoVtsFedrGoqXnOvq3NrMt7l7Z0f6ZDsusBP7N3ScD5cAOYBnwsrtPAl4O6wBXApPCbSnwUE9/CRER6Ztuw93MhgMXA48AuPtRdz8IzAfWhN3WAAvC8nxgrae8CYwws3FZrltE+pGmpib12vMsk577RGAf8Csze9vMfmlmQ4Cx7r477PMxMDYsjwea0+7fEtpERCRPMvlAdQAwA/g7d99sZiv5ryEYANzdzaxH5/aY2VJSwzacccYZPbmrdEYforaL5QNSkd7KJNxbgBZ33xzW15EK9z1mNs7dd4dhl71h+y5gQtr9S0Pbcdy9FqiF1Aeqvaxf2pykwa4QF+lYt8My7v4x0GxmZ4WmucAfgXpgcWhbDDwbluuBWyxlFnAobfhGRETyINPz3P8OeMzMTgU+BG4l9R/Dk2a2BPgIuCHs+wKp0yAbSZ0KeWtWKxaRzmX7HVwOTqVdvXo1l19+Oaeffnqv7q/pgDOTUbi7+1ago3Mp53awrwO39a0sEYnV6tWrmTJlSp/CXdMBd0/TD4hIn9x///1MmTKFKVOm8POf/xxIBfCUKVPa91mxYgU1NTWsW7eOhoYGFi1aREVFBYcPH9Z0wORmOmBNPyAivbZlyxZ+9atfsXnzZtyd8847j0suuYSRI0d2uP/ChQtZtWoVK1asoLLyvwYDNB1w9qcDVs9dRHpt48aNXHPNNQwZMoShQ4dy7bXX8vrrr/f4cTQdcPanA1bPPXa65F7O6JJ9nUufyhfodnpcTQec/emA1XMXkV676KKLeOaZZ/jss8/4y1/+wtNPP81FF13E2LFj2bt3L62trXz++efHDaF0NOWupgPOPvXcRWKS53dpM2bMoLq6mqqqKgC+973vMX36dADuvvtuqqqqGD9+PJMnT26/T3V1NT/4wQ847bTT2odgNB1w9mU05W+uacrfLOjq/OaIh2UK8Ruq+RyW6S9T/nZF0wFnpqdT/qrnHoOTYOqBQgxxkUKmcBeRRKVfRFuyRx+oivRzhTC0KrnVm39jhbtIP1ZcXExra6sCPmLuTmtrK8XFxT26n4ZlRPqx0tJSWlpa2LdvX9KlSA4VFxdTWlrao/so3EX6sYEDBzJx4sSky5ACpHDv7yI7U0ZnxYhkh8bcRUQipHAXEYmQwl1EJEIKdxGRCCncRUQipHAXEYmQwl1EJEIKdxGRCCncRUQipHAXEYlQRuFuZk1m9q6ZbTWzhtA2yszWm9nO8HNkaDcze9DMGs3sHTObkctfQEREvq4nc8v8D3f/c9r6MuBld19uZsvC+j8BVwKTwu084KHwU5JWMzzqS+4Vus7mzcnnZfnk5NGXYZn5wJqwvAZYkNa+1lPeBEaY2bg+PI+IiPRQpuHuwL+b2RYzWxraxrr77rD8MTA2LI8HmtPu2xLajmNmS82swcwaNBe1iEh2ZTosc6G77zKzvwLWm9l76Rvd3c2sR5eCcfdaoBagsrJSl5EREcmijMLd3XeFn3vN7GmgCthjZuPcfXcYdtkbdt8FTEi7e2lok6RENud7odOc9FIIuh2WMbMhZjasbRm4HNgG1AOLw26LgWfDcj1wSzhrZhZwKG34RkRE8iCTnvtY4Gkza9v/cXf/NzP7HfCkmS0BPgJuCPu/AMwDGoHPgFuzXrWIiHSp23B39w+B8g7aW4G5HbQ7cFtWqhMRkV7RN1RFRCKkcBcRiZDCXUQkQgp3EZEIKdxFRCLUk4nDJAaaPKzgaEIxyQX13EVEIqRwFxGJkMJdRCRCCncRkQgp3EVEIqRwFxGJkMJdRCRCCncRkQjpS0z9VT+/upKuViSSW+q5i4hESOEuIhIhhbuISIQU7iIiEVK4i4hESOEuIhIhhbuISIQU7iIiEVK4i4hEKONwN7MiM3vbzJ4L6xPNbLOZNZpZnZmdGtoHhfXGsL0sR7WLiEgnetJz/3tgR9r6z4AH3P2vgQPAktC+BDgQ2h8I+4mISB5lFO5mVgpcBfwyrBtwKbAu7LIGWBCW54d1wva5YX8REcmTTHvuPwd+DHwV1kcDB939WFhvAcaH5fFAM0DYfijsfxwzW2pmDWbWsG/fvt5VLyIiHeo23M3sW8Bed9+SzSd291p3r3T3ypKSkmw+tIjISS+TKX9nA1eb2TygGPhvwEpghJkNCL3zUmBX2H8XMAFoMbMBwHCgNeuVi4hIp7rtubv7ne5e6u5lwI3AK+6+CHgVWBh2Www8G5brwzph+yvu7lmtWkREutSXi3X8E/AbM7sPeBt4JLQ/AvzazBqB/aT+Q5Bs6kcX6tBFOUSS0aNwd/cNwIaw/CFQ1cE+R4Drs1CbiIj0kr6hKiISIYW7iEiEFO4iIhFSuIuIREjhLiISIYW7iEiEFO4iIhFSuIuIREjhLiISIYW7iEiEFO4iIhFSuIuIREjhLiISIYW7iEiE+jKfu/RXNcOh5lDSVUg3OpsLv2n5VXmuRPoj9dxFRCKkcBcRiZDCXUQkQgp3EZEIKdxFRCKkcBcRiZDCXUQkQjrPvb+pGZ50BR3q7JxsEUlGtz13Mys2s7fM7A9mtt3M7gntE81ss5k1mlmdmZ0a2geF9cawvSzHv4OIiJwgk2GZz4FL3b0cqACuMLNZwM+AB9z9r4EDwJKw/xLgQGh/IOwnIiJ51O2wjLs78GlYHRhuDlwK3BTa1wA1wEPA/LAMsA5YZWYWHkcKhaYg6Lc0LYFkIqMPVM2syMy2AnuB9cAHwEF3PxZ2aQHGh+XxQDNA2H4IGJ3FmkVEpBsZhbu7f+nuFUApUAVM7usTm9lSM2sws4Z9+/b19eFERCRNj06FdPeDwKvA+cAIM2sb1ikFdoXlXcAEgLB9ONDawWPVunulu1eWlJT0rnoREelQJmfLlJjZiLB8GnAZsINUyC8Muy0Gng3L9WGdsP0VjbeLiORXJue5jwPWmFkRqf8MnnT358zsj8BvzOw+4G3gkbD/I8CvzawR2A/cmIO6RUSkC5mcLfMOML2D9g9Jjb+f2H4EuD4r1YmISK9o+gERkQgp3EVEIqRwFxGJkCYO608KYNIwTRAm0j+o5y4iEiGFu4hIhBTuIiIRUriLiERI4S4iEiGFu4hIhBTuIiIRUriLiERI4S4iEiGFu4hIhDT9wMmsbToDXSg7CrpwtqRTz11EJEIKdxGRCCncRUQipHAXEYmQwl1EJEIKdxGRCCncRUQipHAXEYmQwl1EJELdfkPVzCYAa4GxgAO17r7SzEYBdUAZ0ATc4O4HzMyAlcA84DOg2t1/n5vyTyJ5vji2LoQt0r9l0nM/BvyDu58DzAJuM7NzgGXAy+4+CXg5rANcCUwKt6XAQ1mvWkREutRtuLv77raet7t/AuwAxgPzgTVhtzXAgrA8H1jrKW8CI8xsXLYLFxGRzvVozN3MyoDpwGZgrLvvDps+JjVsA6ngb067W0toO/GxlppZg5k17Nu3r6d1i4hIFzIOdzMbCvwLcLu7/0f6Nnd3UuPxGXP3WnevdPfKkpKSntxVRES6kVG4m9lAUsH+mLv/a2je0zbcEn7uDe27gAlpdy8NbSIikifdhns4++URYIe735+2qR5YHJYXA8+mtd9iKbOAQ2nDNyIikgeZXKxjNnAz8K6ZbQ1tPwGWA0+a2RLgI+CGsO0FUqdBNpI6FfLWbBYsOVAzXBfsiJgu4nFy6jbc3X0jYJ1sntvB/g7c1se6RESkD/QNVRGRCCncRUQipHAXEYmQwl1EJEIKdxGRCCncRUQipHAXEYmQwl1EJEIKdxGRCGUy/YAkKU9XYNKVl0Tiop67iEiE1HMXAJqKb6LsyONJlyF5pAnF4qaeu4hIhBTuIiIRUriLiERI4S4iEiGFu4hIhBTuIiIR0qmQ0k6nQwroFMlYqOcuIhIhhbuISIQU7iIiEVK4i4hESOFeyPI0I6SIxKfbcDezR81sr5ltS2sbZWbrzWxn+DkytJuZPWhmjWb2jpnNyGXxIiLSsUx67quBK05oWwa87O6TgJfDOsCVwKRwWwo8lJ0yRUSkJ7oNd3d/Ddh/QvN8YE1YXgMsSGtf6ylvAiPMbFyWaj25aEhGRPqgt2PuY919d1j+GBgblscDzWn7tYS2rzGzpWbWYGYN+/bt62UZIiLSkT5/oOruDngv7lfr7pXuXllSUtLXMkREJE1vpx/YY2bj3H13GHbZG9p3ARPS9isNbdJPNBXfBKBpCORrurrOrqYmKDy97bnXA4vD8mLg2bT2W8JZM7OAQ2nDNyIikifd9tzN7AlgDjDGzFqA/w0sB540syXAR8ANYfcXgHlAI/AZcGsOahYRkW50G+7u/p1ONs3tYF8HbutrUSIi0jf6hqqISIQU7iIiEVK4i4hESOEuIhIhhbuISIQU7iIiEVK4i4hEqLfTD0jkmopv0hQEkrHOpibQtATJUc+9EGm6XxHpI4W7iEiEFO4iIhFSuIuIREjhLiISIYV7odGHqSKSBToVsgCULXu+/QpIIiLZoJ67iEiEFO4iIhHSsIx0St9Slb7SN1eTo567iEiEFO4iIhHSsIx0qe0sHg3PSDZ1NlzTGQ3j9JzCPQd68sJtKr6JpuIcFiMiJyWFu2REPXhJkj6Y7TmNuYuIRCgn4W5mV5jZ+2bWaGbLcvEcIiLSuawPy5hZEfB/gcuAFuB3Zlbv7n/M9nP1Rlfj4Z29xevphz+dPn7aeeP9dboBnfsuhUTDNZ3LxZh7FdDo7h8CmNlvgPlATsI9W8GbS/01yDuT/vucGPQKfykEWeuQZfE/iXz/R2Tunt0HNFsIXOHu3wvrNwPnufsPT9hvKbA0rJ4FvN/NQ48B/pzVYrNDdWWuEGsC1dVTqitzua7pv7t7SUcbEjtbxt1rgdpM9zezBnevzGFJvaK6MleINYHq6inVlbkka8rFB6q7gAlp66WhTURE8iQX4f47YJKZTTSzU4EbgfocPI+IiHQi68My7n7MzH4IvAgUAY+6+/YsPHTGQzh5proyV4g1gerqKdWVucRqyvoHqiIikjx9Q1VEJEIKdxGRCBV8uJvZ9Wa23cy+MrPKE7bdGaY4eN/M/ldC9dWY2S4z2xpu85KoI62egpz6wcyazOzdcIwaEqzjUTPba2bb0tpGmdl6M9sZfo4skLoSfW2Z2QQze9XM/hj+Bv8+tCd6vLqoK+njVWxmb5nZH0Jd94T2iWa2OfxN1oUTTXLP3Qv6BpxN6ktOG4DKtPZzgD8Ag4CJwAdAUQL11QD/mPRxCrUUheNwJnBqOD7nJF1XqK0JGFMAdVwMzAC2pbX9H2BZWF4G/KxA6kr0tQWMA2aE5WHA/wt/d4kery7qSvp4GTA0LA8ENgOzgCeBG0P7w8Df5qOegu+5u/sOd+/o26vzgd+4++fu/v+BRlJTH5zM2qd+cPejQNvUDxK4+2vA/hOa5wNrwvIaYEE+a4JO60qUu+9299+H5U+AHcB4Ej5eXdSVKE/5NKwODDcHLgXWhfa8Ha+CD/cujAea09ZbSO4f+Idm9k54a533t/RpCumYnMiBfzezLWHqiUIy1t13h+WPgbFJFnOCgnhtmVkZMJ1Ub7RgjtcJdUHCx8vMisxsK7AXWE/qnfRBdz8Wdsnb32RBhLuZvWRm2zq4FUSvs5v6HgK+AVQAu4F/TrLWAnahu88ArgRuM7OLky6oI55671wo5wcXxGvLzIYC/wLc7u7/kb4tyePVQV2JHy93/9LdK0h9M78KmJzvGtoUxJWY3P1/9uJueZvmINP6zOwXwHO5qCFDBTv1g7vvCj/3mtnTpF74ryVbVbs9ZjbO3Xeb2ThSva7EufuetuWkXltmNpBUgD7m7v8amhM/Xh3VVQjHq427HzSzV4HzgRFmNiD03vP2N1kQPfdeqgduNLNBZjYRmAS8le8iwou7zTXAts72zYOCnPrBzIaY2bC2ZeBykj1OJ6oHFoflxcCzCdbSLunXlpkZ8Aiww93vT9uU6PHqrK4COF4lZjYiLJ9G6poWO4BXgYVht/wdr6Q+We7BJ9DXkBqn+hzYA7yYtu0uUmNa7wNXJlTfr4F3gXdIvejHJXy85pE6e+AD4K6k//1CTWeSOnPnD8D2JOsCniD1lv2L8LpaAowGXgZ2Ai8BowqkrkRfW8CFpIZc3gG2htu8pI9XF3UlfbymAW+H598G3B3azyTV8WwEngIG5aMeTT8gIhKh/jwsIyIinVC4i4hESOEuIhIhhbuISIQU7iIiEVK4i4hESOEuIhKh/wQAixDxpyEsLwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "m = torch.nn.BatchNorm1d(1, momentum=None)\n",
    "\n",
    "# input data (mean 10, var 25)\n",
    "x = 10+5*torch.randn(10000, 1)\n",
    "\n",
    "# first batch\n",
    "with torch.no_grad():\n",
    "    y = m(x)\n",
    "    \n",
    "plt.hist(x.numpy(),bins=50,label='input first batch');\n",
    "plt.hist(y.numpy(),bins=50,label='output first batch');\n",
    "plt.legend(loc='upper right');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d3270f2d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAhA0lEQVR4nO3df3RU1fnv8fcDAYKigPwqAhp6C9iWQIwhgigFKb8tqEWRWgWLUqtytb1qqV39mq5lu/ArV2q1UmkVgpUK8lWhaitiVVArmiCCigJ6QwmiBBTkh/x+7h9zkg4hIZNkQuacfF5rZc05++wzs/dM8mTPnn2eMXdHRESipVF9N0BERJJPwV1EJIIU3EVEIkjBXUQkghTcRUQiKK2+GwDQtm1bz8jIqO9miIiESmFh4TZ3b1fRsZQI7hkZGRQUFNR3M0REQsXMNlZ2TNMyIiIRpOAuIhJBCu4iIhGUEnPuFTl48CDFxcXs27evvpsiQnp6Op07d6ZJkyb13RSRhKRscC8uLuaUU04hIyMDM6vv5kgD5u5s376d4uJiunbtWt/NEUlIyk7L7Nu3jzZt2iiwS70zM9q0aaN3kRIqKRvcAQV2SRn6XZSwSengLiIiNZPQnLuZ/RS4FnBgDXAN0BF4HGgDFAJXufsBM2sGzAXOAbYD49y9qLYNzZj6bG3v4ihF00Yl9f4A5syZw9ChQzn99NNrdH5RURGvv/46P/jBD5LcsuSYM2cOBQUFPPDAA0eV5+Xl0aJFC2699daE7mfHjh3MmzePG2644bj1Bg4cyPTp08nJyalxm6ujRYsW7N69+4Q8lkhdq3LkbmadgP8N5Lh7T6AxcAVwNzDD3b8BfAFMCk6ZBHwRlM8I6jUIc+bM4ZNPPqnx+UVFRcybNy+JLUpNO3bs4MEHH6zvZohEWqLTMmlAczNLA04CtgAXAguD4/nAxcH2mGCf4PhgC+mE5b333kvPnj3p2bMnv/vd74BYAO7Zs2dZnenTp5OXl8fChQspKCjgyiuvJCsri6+++oqMjAxuv/12MjMzyc3NZcOGDQBMnDiRhQsXlt1HixYtAJg6dSrLly8nKyuLGTNmHNWWLVu2MGDAALKysujZsyfLly8HYMmSJfTr14/s7Gwuu+yyspHnW2+9xXnnnUfv3r3Jzc1l165d7Nu3j2uuuYbMzEzOPvtsXnrpJSD2T+nSSy9l+PDhdOvWjdtvv73scWfPnk337t3Jzc3ltddeq/S5euedd+jXrx/dunXjT3/6EwC7d+9m8ODBZGdnk5mZyaJFi8r6+dFHH5GVlcVtt90GwN13301mZia9e/dm6tSpZff7xBNPkJubS/fu3cv6XN4999xDnz596NWrF3feeWfZY/zhD38oq5OXl8f06dMrbZNI1FQ5LePum81sOvBv4CtgCbFpmB3ufiioVgx0CrY7AZuCcw+Z2U5iUzfb4u/XzCYDkwHOOOOM2vckyQoLC5k9ezYrVqzA3Tn33HP5zne+Q+vWrSusP3bsWB544IFjphFatmzJmjVrmDt3LrfccgvPPPNMpY85bdo0pk+fXmGdefPmMWzYMH75y19y+PBh9u7dy7Zt27jrrrtYunQpJ598MnfffTf33nsvU6dOZdy4ccyfP58+ffrw5Zdf0rx5c+677z7MjDVr1vDBBx8wdOhQ1q1bB8CqVat4++23adasGT169GDKlCmkpaVx5513UlhYSMuWLRk0aBBnn312hW1fvXo1b7zxBnv27OHss89m1KhRtG/fnqeeeopTTz2Vbdu20bdvX0aPHs20adN49913WbVqFQB///vfWbRoEStWrOCkk07i888/L7vfQ4cO8eabb/Lcc8/x61//mqVLlx71uEuWLGH9+vW8+eabuDujR49m2bJljBs3jltuuYUbb7wRgAULFvD888+Tnp5eYZtCOv4QqVSVwd3MWhMbjXcFdgBPAMNr+8DuPguYBZCTk5NyX+T66quvcskll3DyyScDcOmll7J8+XJGjx5drfsZP3582e1Pf/rTGrenT58+/OhHP+LgwYNcfPHFZGVl8corr/D+++/Tv39/AA4cOEC/fv348MMP6dixI3369AHg1FNPLevTlClTADjrrLM488wzy4L74MGDadmyJQDf+ta32LhxI9u2bWPgwIG0axdLOjdu3Liy+uWNGTOG5s2b07x5cwYNGsSbb77JqFGjuOOOO1i2bBmNGjVi8+bNfPbZZ8ecu3TpUq655hpOOukkAE477bSyY5deeikA55xzDkVFRcecu2TJEpYsWVL2T2f37t2sX7+eSZMmsXXrVj755BNKSkpo3bo1Xbp04eDBgxW26Wtf+1o1Xg2R1JfIB6rfBf6fu5cAmNmTQH+glZmlBaP3zsDmoP5moAtQHEzjtCT2wWokpKWlceTIkbL9qtY+x48IS7fj7+PIkSMcOHCgyscdMGAAy5Yt49lnn2XixIn87Gc/o3Xr1gwZMoS//vWvR9Vds2ZNwv0p1axZs7Ltxo0bc+jQoePUPlb5ka+Z8dhjj1FSUkJhYSFNmjQhIyOj2mvFS9tVWZvcnV/84hf8+Mc/PubYZZddxsKFC/n0008ZN24cQFLaJBIGicy5/xvoa2YnBXPng4H3gZeAsUGdCUDp5OXiYJ/g+D/dPeVG5lW54IILePrpp9m7dy979uzhqaee4oILLqBDhw5s3bqV7du3s3///qOmUE455RR27dp11P3Mnz+/7LZfv35ALMVxYWEhAIsXL+bgwYOVnl9q48aNdOjQgeuuu45rr72WlStX0rdvX1577bWyufw9e/awbt06evTowZYtW3jrrbcA2LVrF4cOHeKCCy7gscceA2DdunX8+9//pkePHpU+B+eeey6vvPIK27dv5+DBgzzxxBOV1l20aBH79u1j+/btvPzyy/Tp04edO3fSvn17mjRpwksvvcTGjRsr7OeQIUOYPXs2e/fuBThqWqYqw4YN45FHHin7rGHz5s1s3boViL3TePzxx1m4cCGXXXYZQKVtEomaRObcV5jZQmAlcAh4m9h0yrPA42Z2V1D2cHDKw8CjZrYB+JzYyppaq4uli8eTnZ3NxIkTyc3NBeDaa68te+v/X//1X+Tm5tKpUyfOOuussnMmTpzI9ddfT/PmzfnXv/4FwBdffEGvXr1o1qxZ2Qj7uuuuY8yYMfTu3Zvhw4eXTf306tWLxo0b07t3byZOnHjUNM7LL7/MPffcQ5MmTWjRogVz586lXbt2zJkzh/Hjx7N//34A7rrrLrp37878+fOZMmUKX331Fc2bN2fp0qXccMMN/OQnPyEzM5O0tDTmzJlz1Ii9vI4dO5KXl0e/fv1o1aoVWVlZldbt1asXgwYNYtu2bfzqV7/i9NNP58orr+R73/semZmZ5OTklD1Xbdq0oX///vTs2ZMRI0Zwzz33sGrVKnJycmjatCkjR47kt7/9bUKv09ChQ1m7dm3ZP84WLVrwl7/8hfbt2/Ptb3+bXbt20alTJzp27AhQaZtEosZSYVCdk5Pj5b+sY+3atXzzm9+spxYlR+mXkLRt27a+myJJEIXfSYkWMyt09wovBNEVqiIiEZSyWSGjoKLVHSIiJ4JG7iIiEaTgLiISQQruIiIRpOAuIhJB4flANa9lku9vZ3LvD6X8rY+Uv7V5zs477zxef/31ap8nEgYauSeRUv4mJpkpf4/3nFWVQkGBXaJMwf04lPI39VP+ln/O5syZw+jRo7nwwgsZPHjwcVP8lj7vL7/8MgMHDmTs2LGcddZZXHnllaTCxX0itRGeaZkTTCl/w5Hyt/xzNmfOHFauXMnq1as57bTTOHToUEIpft9++23ee+89Tj/9dPr3789rr73G+eefX+lrJZLqNHKvRHzK3xYtWpSl/K2u+JS/pflmaqJPnz7Mnj2bvLw81qxZwymnnMIbb7xRlvI3KyuL/Px8Nm7cWGHK37S0NF599VV++MMfApWn/E1PTy9L+btixYqylL9NmzYty6xYkdKUv23bti1L+evu3HHHHfTq1Yvvfve7dZLytyJDhgwpu49E25Cbm0vnzp1p1KgRWVlZugBNQk8j92pSyt+K1VfK34qUJmKDxFP81rb/IqlGI/dKKOVvOFL+Hu85A6X4lYYrPCP3Oli6eDxK+RuOlL/ln7Pyn4koxa80VEr5W4eU8jdaovA7KdGilL8iIg1MeKZlQkgrLkSkvlQ5cjezHma2Ku7nSzO7xcxOM7MXzGx9cNs6qG9m9nsz22Bmq80su+67ISIi8aoM7u7+obtnuXsWcA6wF3gKmAq86O7dgBeDfYARQLfgZzIwsw7aLSIix1HdOffBwEfuvhEYA+QH5fnAxcH2GGCux7wBtDKzjslorIiIJKa6wf0KoPSKmQ7uviXY/hToEGx3AjbFnVMclB3FzCabWYGZFZSUlFSzGSIicjwJf6BqZk2B0cAvyh9zdzezaq2pdPdZwCyILYWsqn5mfmZ17r5KayZU/yrOqtRlyt/yxypLvwswcuRI5s2bR6tWrY77eJWl6p04cSIXXXQRY8eOrXW7453IpaFFRUVcdNFFvPvuu3X+WCKpqDoj9xHASncvTczxWel0S3C7NSjfDHSJO69zUBZ5dZnytzrpgJ977rljAru7H5U2IZkaSqpikTCpTnAfz3+mZAAWAxOC7QnAorjyq4NVM32BnXHTN6GSSil/Kzr2ySefVJimNyMjg23btlFUVESPHj24+uqr6dmzJ5s2beI3v/kN3bt35/zzz+fDDz+stO9Lly4lJyeH7t27l6VYKCoq4oILLiA7O5vs7OyyfOjl23b48GFuvfVWevbsSa9evbj//vvL7vf+++8vS7/7wQcfHPO4hw8f5rbbbqNPnz706tWLhx56CIArrriCZ599tqxe6XNYWZtEGrqEpmXM7GRgCPDjuOJpwAIzmwRsBC4Pyp8DRgIbiK2suSZprT2BUi3lb0WpbStK09ulS5ejzlu/fj35+fn07duXwsJCHn/8cVatWsWhQ4fIzs7mnHPOqbAtRUVFvPnmm3z00UcMGjSIDRs20L59e1544QXS09NZv34948ePp6Cg4Ji2zZw5k6KiIlatWkVaWtpRuWLatm3LypUrefDBB5k+fTp//vOfj3rchx9+mJYtW/LWW2+xf/9++vfvz9ChQxk3bhwLFixg1KhRHDhwgBdffJGZM2fi7hW2SaShSyi4u/seoE25su3EVs+Ur+vAjUlpXT2KT/kLlKX8HT16dLXuJz7lb3yumGQoTdMLlKXpLR/czzzzTPr27QvA8uXLueSSS8pS6x6vL5dffjmNGjWiW7dufP3rX+eDDz6ga9eu3HTTTaxatYrGjRuXpQsub+nSpVx//fWkpcV+vSpL4fvkk08ec+6SJUtYvXp12TubnTt3sn79ekaMGMHNN9/M/v37+cc//sGAAQNo3rw5O3fuTKhNIg2NrlCtpvpK+VuRRNLUxqe/rY6KUvjOmDGDDh068M4773DkyBHS09Orfb9VpfB1d+6//36GDRt2zLGBAwfy/PPPM3/+fK644gqApLRJJIqUW6YSqZbyt6rUtokYMGAATz/9NF999RW7du3ib3/7W6V1n3jiCY4cOcJHH33Exx9/TI8ePdi5cycdO3akUaNGPProoxw+fLjCtg0ZMoSHHnqoLHhXJ4XvsGHDmDlzZtlzsm7dOvbs2QPAuHHjmD17NsuXL2f48OEAlbZJpKELzci9LpYuHk+qpfytKrVton0aN24cvXv3pn379mXf1FSRM844g9zcXL788kv++Mc/kp6ezg033MD3v/995s6de9x2T5kyhXXr1tGrVy+aNGnCddddx0033ZRQG6+99lqKiorIzs7G3WnXrh1PP/00AEOHDuWqq65izJgxNG3aFKDSNok0dEr5W4eU8jdaovA7KdGilL8iIg1MaKZlwkgpf0WkvqT0yD0VpoxEQL+LEj4pG9zT09PZvn27/qik3rk727dv1zJLCZWUnZbp3LkzxcXFKGOkpIL09HQ6d+5c380QSVjKBvcmTZrQtWvX+m6GiEgopey0jIiI1JyCe13Laxn7kdDIzM9M+vcHSN3JmPosGVOfrbpiA6PgLiISQQruIiIRpOAuIhJBKbtaJnLKz7vn7ayfdkjC4ufdT3TiOqm+8vPuRdNG1VNLUkNCI3cza2VmC83sAzNba2b9zOw0M3vBzNYHt62DumZmvzezDWa22syy67YLIiJSXqLTMvcB/3D3s4DewFpgKvCiu3cDXgz2IfZF2t2Cn8nAzKS2WEREqlRlcDezlsAA4GEAdz/g7juAMUB+UC0fuDjYHgPM9Zg3gFZm1jHJ7RYRkeNIZOTeFSgBZpvZ22b25+ALszu4+5agzqdAh2C7E7Ap7vzioOwoZjbZzArMrEApBkREkiuRD1TTgGxgiruvMLP7+M8UDBD7Umwzq1aGL3efBcyC2Jd1VOfclKeLlkJFFyyFjy5aqloiI/dioNjdVwT7C4kF+89Kp1uC263B8c1Al7jzOwdlIiJyglQZ3N39U2CTmfUIigYD7wOLgQlB2QRgUbC9GLg6WDXTF9gZN30jIiInQKLr3KcAj5lZU+Bj4Bpi/xgWmNkkYCNweVD3OWAksAHYG9QVEZETKKHg7u6rgIq+hHVwBXUduLF2zRIRkdpQ+gERkQhScK8vWlETKkoDHD4NfUWNgruISAQpuIuIRJCyQiaTplpCR1Mt4dLQp1qqQyN3EZEIUnAXEYkgBXcRkQhScBcRiSB9oJoM+iA1dPRBarjog9Tq08hdRCSCFNxFRCJIwV1EJIIU3EVEIkjBXUQkghTcRUQiSMFdRCSCEgruZlZkZmvMbJWZFQRlp5nZC2a2PrhtHZSbmf3ezDaY2Wozy67LDoiIyLGqM3If5O5Z7l76dXtTgRfdvRvwYrAPMALoFvxMBmYmq7GRlddSF0KFjC6CCpeMqc82uAuhajMtMwbID7bzgYvjyud6zBtAKzPrWIvHERGRako0uDuwxMwKzWxyUNbB3bcE258CHYLtTsCmuHOLg7KjmNlkMysws4KSkpIaNF1ERCqTaG6Z8919s5m1B14wsw/iD7q7m5lX54HdfRYwCyAnJ6da50aGpmJCR9Mx4dLQpmLiJTRyd/fNwe1W4CkgF/isdLoluN0aVN8MdIk7vXNQJiIiJ0iVwd3MTjazU0q3gaHAu8BiYEJQbQKwKNheDFwdrJrpC+yMm74REZETIJFpmQ7AU2ZWWn+eu//DzN4CFpjZJGAjcHlQ/zlgJLAB2Atck/RWi4jIcVUZ3N39Y6B3BeXbgcEVlDtwY1JaJyIiNaIrVEVEIkjBXUQkghTcRUQiSME9lSgNQahk5mdq3XvINKQ0BAruIiIRpOAuIhJBCu4iIhGk4C4iEkEK7iIiEZRoVkipjFa3hIpWt4RPQ1ndkmwauYuIRJCCu4hIBCm4i4hEkIK7iEgEKbiLiESQgruISAQlHNzNrLGZvW1mzwT7Xc1shZltMLP5ZtY0KG8W7G8IjmfUUdtFRKQS1Rm53wysjdu/G5jh7t8AvgAmBeWTgC+C8hlBPREROYESCu5m1hkYBfw52DfgQmBhUCUfuDjYHhPsExwfHNQXEZETJNGR+++A24EjwX4bYIe7Hwr2i4FOwXYnYBNAcHxnUP8oZjbZzArMrKCkpKRmrRcRkQpVGdzN7CJgq7sXJvOB3X2Wu+e4e067du2SedciIg1eIrll+gOjzWwkkA6cCtwHtDKztGB03hnYHNTfDHQBis0sDWgJbE96y0VEpFJVjtzd/Rfu3tndM4ArgH+6+5XAS8DYoNoEYFGwvTjYJzj+T3f3pLZaRESOqzZZIX8OPG5mdwFvAw8H5Q8Dj5rZBuBzYv8QokfZIENF2SDDR9kga6dawd3dXwZeDrY/BnIrqLMPuCwJbRMRkRrSFaoiIhGk4C4iEkEK7iIiEaTgLiISQQruIiIRpOAuIhJBCu4iIhGk4C4iEkEK7iIiEaTgLiISQQruqUh5a0IlMz9TuWtCpiHkrVFwFxGJIAV3EZEIUnAXEYkgBXcRkQhScBcRiSAFdxGRCKrym5jMLB1YBjQL6i909zvNrCvwONAGKASucvcDZtYMmAucQ+yLsce5e1Edtf/E0zLF0NEyxXBpCMsUT4RERu77gQvdvTeQBQw3s77A3cAMd/8G8AUwKag/CfgiKJ8R1BMRkROoyuDuMbuD3SbBjwMXAguD8nzg4mB7TLBPcHywmVmyGtxg5LXUu4SQ0cVM4ZIx9dlIv0tIaM7dzBqb2SpgK/AC8BGww90PBVWKgU7BdidgE0BwfCexqZvy9znZzArMrKCkpKRWnRARkaMlFNzd/bC7ZwGdgVzgrNo+sLvPcvccd89p165dbe9ORETiVGu1jLvvAF4C+gGtzKz0A9nOwOZgezPQBSA43pLYB6siInKCVBnczaydmbUKtpsDQ4C1xIL82KDaBGBRsL042Cc4/k939yS2WUREqlDlUkigI5BvZo2J/TNY4O7PmNn7wONmdhfwNvBwUP9h4FEz2wB8DlxRB+0WEZHjqDK4u/tq4OwKyj8mNv9evnwfcFlSWiciIjWSyMhdSmlpYqhoWWK4RHlZYn1Q+gERkQhScBcRiSAFdxGRCFJwFxGJIAV3EZEIUnAXEYkgBXcRkQhScBcRiSAF91SnvO6ho4unwiWqed0V3EVEIkjBXUQkghTcRUQiSMFdRCSCFNxFRCJIwV1EJIIS+Zq9Lmb2kpm9b2bvmdnNQflpZvaCma0PblsH5WZmvzezDWa22syy67oTIiJytES+rOMQ8H/cfaWZnQIUmtkLwETgRXefZmZTganAz4ERQLfg51xgZnAbXlpnHipaZx4+UVxnXt+qHLm7+xZ3Xxls7yL25didgDFAflAtH7g42B4DzPWYN4BWZtYx2Q0XEZHKVWvO3cwyiH2f6gqgg7tvCQ59CnQItjsBm+JOKw7KRETkBEk4uJtZC+B/gFvc/cv4Y+7ugFfngc1sspkVmFlBSUlJdU4VEZEqJBTczawJscD+mLs/GRR/VjrdEtxuDco3A13iTu8clB3F3We5e46757Rr166m7RcRkQokslrGgIeBte5+b9yhxcCEYHsCsCiu/Opg1UxfYGfc9I3UlBKIhUpmfqY+2A2ZqCUQS2S1TH/gKmCNma0Kyu4ApgELzGwSsBG4PDj2HDAS2ADsBa5JZoNFRKRqVQZ3d38VsEoOD66gvgM31rJdIiJSC7pCVUQkghTcRUQiSMFdRCSCFNxFRCJIwV1EJIIU3EVEIiiRde4Nly4aCh1dOBQuUbpoKNVo5C4iEkEK7mGjdxOhojQE4ROVdxMK7iIiEaTgLiISQQruIiIRpOAuIhJBCu5hpNzuoaMPVsMlCrndFdxFRCJIwV1EJIIU3EVEIqjK9ANm9ghwEbDV3XsGZacB84EMoAi43N2/CL5v9T5iX7O3F5jo7ivrpul1SPPZoaP57HAJ+3x2GCQycp8DDC9XNhV40d27AS8G+wAjgG7Bz2RgZnKaKSIi1ZHId6guM7OMcsVjgIHBdj7wMvDzoHxu8D2qb5hZKzPr6O5bktbiuqZRe6hoxB4+GrWfGDWdc+8QF7A/BToE252ATXH1ioOyY5jZZDMrMLOCkpKSGjZDREQqUusPVINRutfgvFnunuPuOe3atattM0REJE5Ng/tnZtYRILjdGpRvBrrE1esclEld0MVMoaNppHAJ88VMNQ3ui4EJwfYEYFFc+dUW0xfYGar5dhGRiEhkKeRfiX142tbMioE7gWnAAjObBGwELg+qP0dsGeQGYkshr6mDNouISBUSWS0zvpJDgyuo68CNtW2UiIjUjq5QFRGJIAV3EZEIUnAXEYkgBXcRkQhScI8CrXcPFX1xR/iEcb17latlGgwFx1BRcAyfsAXHsNPIXUQkghTcRUQiSMFdRCSCNOeuufbQ0Xx7uGiuvX403OCuoB46CurhoqBevzQtIyISQQruUaJ3I6Gi9e7hE6Z3IwruIiIRpOAeNbpaNXQ0gg+XsFytarEU7PUrJyfHCwoK6v6BGmLQy9tZ3y2osYYY8NZMWFPfTaiVMAS9ZCuaNqreHtvMCt09p6JjGrlHnUbyoaJRfPik6ki+ToK7mQ03sw/NbIOZTa2LxxARkcolfVrGzBoD64AhQDHwFjDe3d+v7Jw6n5bRyDUmRFM0Gr3GhGWaJhVHrvXhRE/RHG9api4uYsoFNrj7x8GDPw6MASoN7klVGsjzdiqol1f++UiBYF8axEuDmIL60eKfj1QJ9KWBvGjaKAX1cso/H/U6H18HI/exwHB3vzbYvwo4191vKldvMjA52O0BfAi0BbYltUGpI6p9i2q/ILp9i2q/ILp9q6xfZ7p7u4pOqLf0A+4+C5gVX2ZmBZW9xQi7qPYtqv2C6PYtqv2C6PatJv2qiw9UNwNd4vY7B2UiInKC1EVwfwvoZmZdzawpcAWwuA4eR0REKpH0aRl3P2RmNwHPA42BR9z9vQRPn1V1ldCKat+i2i+Ibt+i2i+Ibt+q3a+UuEJVRESSS1eoiohEkIK7iEgEpVxwN7M8M9tsZquCn5H13abaiHIqBjMrMrM1wet0AjK/1R0ze8TMtprZu3Flp5nZC2a2PrhtXZ9trIlK+hX6vzEz62JmL5nZ+2b2npndHJRH4TWrrG/Vet1Sbs7dzPKA3e4+vb7bUls1ScUQJmZWBOS4e+gvGjGzAcBuYK679wzK/hv43N2nBf+YW7v7z+uzndVVSb/yCPnfmJl1BDq6+0ozOwUoBC4GJhL+16yyvl1ONV63lBu5R0xZKgZ3PwCUpmKQFOPuy4DPyxWPAfKD7Xxif2ChUkm/Qs/dt7j7ymB7F7AW6EQ0XrPK+lYtqRrcbzKz1cFbytC9rYrTCdgUt19MDV6kFObAEjMrDNJJRE0Hd98SbH8KdKjPxiRZVP7GMLMM4GxgBRF7zcr1DarxutVLcDezpWb2bgU/Y4CZwP8CsoAtwP+tjzZKQs5392xgBHBjMAUQSR6bv0ytOcyai8zfmJm1AP4HuMXdv4w/FvbXrIK+Vet1q5fcMu7+3UTqmdmfgGfquDl1KdKpGNx9c3C71cyeIjYNtax+W5VUn5lZR3ffEsyDbq3vBiWDu39Wuh3mvzEza0Is+D3m7k8GxZF4zSrqW3Vft5SblglekFKXAO9WVjcEIpuKwcxODj7swcxOBoYS7teqIouBCcH2BGBRPbYlaaLwN2ZmBjwMrHX3e+MOhf41q6xv1X3dUnG1zKPE3nY4UAT8OG4OLXSC5Uq/4z+pGH5Tvy1KDjP7OvBUsJsGzAtz38zsr8BAYqlVPwPuBJ4GFgBnABuBy909VB9OVtKvgYT8b8zMzgeWA2uAI0HxHcTmpsP+mlXWt/FU43VLueAuIiK1l3LTMiIiUnsK7iIiEaTgLiISQQruIiIRpOAuIhJBCu4iIhGk4C4iEkH/H0VwUoddVIf1AAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# second batch (mean 110, var 25)\n",
    "# in eval mode\n",
    "m.eval()\n",
    "with torch.no_grad():\n",
    "    y_eval = m(100+x)\n",
    "# in train mode\n",
    "m.train()\n",
    "with torch.no_grad():\n",
    "    y_train = m(100+x)\n",
    "\n",
    "# third batch (mean 110, var 25)\n",
    "m.eval()\n",
    "with torch.no_grad():\n",
    "    y_eval_3 = m(100+x)\n",
    "    \n",
    "plt.hist(y_eval.numpy(),bins=50,label='output second batch eval');\n",
    "plt.hist(y_train.numpy(),bins=50,label='output second batch train');\n",
    "plt.hist(y_eval_3.numpy(),bins=50,label='output third batch eval');\n",
    "plt.ylim((0,850))\n",
    "plt.legend(loc='upper left');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1b764003",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "37ac74d1",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dldiy",
   "language": "python",
   "name": "dldiy"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}