Capstone_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyM21AV3udNbEc9w4unu+8cD",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Chinmaya-3141/Capstone-Project/blob/Chinmaya/Capstone_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Mount Google Drive\n",
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8lgMPJIltoSZ",
        "outputId": "50906531-b6c8-47f0-8901-11084b99b79b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "import time\n",
        "import numpy as np\n",
        "from numba import cuda"
      ],
      "metadata": {
        "id": "ZrUf-jRRxvmD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "UERrKlQ5snJN"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import torch.optim as optim\n",
        "from torchvision.datasets import ImageFolder\n",
        "from torch.utils.data import DataLoader\n",
        "\n",
        "from torchvision import datasets, transforms"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Check if GPU is available\n",
        "\n",
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "print(\"Using device:\", device)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2VKWswRFstxb",
        "outputId": "cc093863-0ba9-4290-a0ba-a2bc72ae6b6a"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using device: cuda\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Set path here, access file path changes person to person\n",
        "\n",
        "path = '/content/drive/MyDrive/Capstone Data (Shared)/Capstone Data/Resized Data/Resized Data_320x320'"
      ],
      "metadata": {
        "id": "71jv8VMzuhug"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the transform for the data\n",
        "\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((320, 320)),\n",
        "    transforms.ToTensor(),\n",
        "    transforms.Normalize((0.5,), (0.5,))\n",
        "])"
      ],
      "metadata": {
        "id": "hHpqaQTGswu6"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Load the data from the combined folder\n",
        "\n",
        "data = ImageFolder(root=path, transform=transform)"
      ],
      "metadata": {
        "id": "qbN65cAqsz50"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Split the data into training and testing sets\n",
        "\n",
        "train_data, test_data = torch.utils.data.random_split(data, [int(0.8*len(data)), len(data)-int(0.8*len(data))])"
      ],
      "metadata": {
        "id": "Pfol89Zss2mw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the data loaders\n",
        "\n",
        "train_loader = DataLoader(train_data, batch_size=64, shuffle=True)\n",
        "test_loader = DataLoader(test_data, batch_size=64, shuffle=False)"
      ],
      "metadata": {
        "id": "j5Xh3trHs5N9"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the CNN model\n",
        "\n",
        "class CNNModel(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(CNNModel, self).__init__()\n",
        "        self.conv_layer = nn.Sequential(\n",
        "            # input layer\n",
        "            nn.Conv2d(in_channels=3, out_channels=16, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=2),\n",
        "            \n",
        "            # Hidden Layer\n",
        "            nn.Conv2d(in_channels=16, out_channels=32, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=2),\n",
        "\n",
        "            # Hidden Layer\n",
        "            nn.Conv2d(in_channels=32, out_channels=64, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=2),\n",
        "            \n",
        "            # Hidden Layer\n",
        "            nn.Conv2d(in_channels=64, out_channels=128, kernel_size=3, padding=1),\n",
        "            nn.ReLU(inplace=True),\n",
        "            nn.MaxPool2d(kernel_size=2)\n",
        "        )\n",
        "        self.fc_layer = nn.Sequential(\n",
        "            # Hidden Layer\n",
        "            nn.Linear(128 * 20 * 20, 512),\n",
        "            nn.ReLU(inplace=True),\n",
        "\n",
        "            # Hidden Layer\n",
        "            nn.Linear(512, 256),\n",
        "            nn.ReLU(inplace=True),\n",
        "\n",
        "            # Output Layer\n",
        "            nn.Linear(256, 3)\n",
        "        )\n",
        "\n",
        "    def forward(self, x):\n",
        "        x = self.conv_layer(x)\n",
        "        x = x.view(x.size(0), -1)\n",
        "        x = self.fc_layer(x)\n",
        "        return x"
      ],
      "metadata": {
        "id": "McrptB7is79P"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from torchsummary import summary"
      ],
      "metadata": {
        "id": "bpUFTZxyz5o1"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Initialize the model\n",
        "\n",
        "model = CNNModel().to(device)"
      ],
      "metadata": {
        "id": "L6RleLVVs-9D"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Define the loss function and optimizer\n",
        "\n",
        "criterion = nn.CrossEntropyLoss()\n",
        "# Sparse categorical cross entropy can also be used\n",
        "\n",
        "# optimizer = optim.SGD(model.parameters(), lr=0.01, momentum = 0.9)\n",
        "# Adam optimizer\n",
        "# LBFGS optimizer\n",
        "\n",
        "optimizer = optim.AdamW(model.parameters(),lr=0.001)\n",
        "# optimizer = optim.AdamW(model.parameters(),lr=0.001)"
      ],
      "metadata": {
        "id": "oatFElKUtBah"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Time the execution of the program\n",
        "start_time = time.time()\n",
        "loss_change = []\n",
        "\n",
        "# Train the model\n",
        "for epoch in range(20):\n",
        "    running_loss = 0.0\n",
        "    for i, datavar in enumerate(train_loader, 0):\n",
        "        inputs, labels = datavar[0].to(device), datavar[1].to(device)\n",
        "        inputs = inputs.to(device)\n",
        "        labels = labels.to(device)\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(inputs).to(device)\n",
        "        loss = criterion(outputs, labels)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        running_loss += loss.item()\n",
        "        loss_change.append(float(running_loss))\n",
        "        if i % 100 == 99:\n",
        "            print('[%d, %5d] loss: %.3f' % (epoch + 1, i + 1, running_loss / 100))\n",
        "            running_loss = 0.0\n",
        "\n",
        "\n",
        "# Synchronize all parallel processes\n",
        "cuda.synchronize()\n",
        "\n",
        "# Calculate time taken for each part of the parallel processing\n",
        "end_time = time.time()\n",
        "execution_time = end_time - start_time"
      ],
      "metadata": {
        "id": "05prM2kZtD1g"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Print summary of execution time\n",
        "\n",
        "print(f\"Total time: {int(execution_time/60)} minutes {execution_time - (int(execution_time/60)*60)} seconds\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4j1VAWTZxrC8",
        "outputId": "197ddf74-8eee-4d5b-c6fe-8834759976fd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Total time: 45 minutes 18.607905864715576 seconds\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import matplotlib.pyplot as plt"
      ],
      "metadata": {
        "id": "G03s7S0R6MMD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "plt.style.use(\"ggplot\")\n",
        "plt.plot(range(len(loss_change)),loss_change)\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 265
        },
        "id": "1vpyJPQe6IsN",
        "outputId": "62832d49-3cfc-4451-f06a-4f6888145a44"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD4CAYAAAANbUbJAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAv60lEQVR4nO3de3xU9Z3w8c9vJkCu5EJCIFyUCFqhIGqoCioI0XbVbllbXbW2q7Z1fVBp6WOr9mmxz2NZ2SpCL1jbblet3W7VFmjr2lukhFZFwx2D3EHQREJuk8k9M+f3/HEyk0mYyzkzmWRm8n2/XrySTOZ3zpeT5Du/+Z7fRWmtNUIIIVKaY7gDEEIIEX+S7IUQYgSQZC+EECOAJHshhBgBJNkLIcQIIMleCCFGgLShPmFNTU1U7QoLC6mvrx/kaAZPIseXyLGBxBeLRI4NJL5YBMZWUlIS8/GkZy+EECOAJHshhBgBJNkLIcQIIMleCCFGAEn2QggxAkiyF0KIEUCSvRBCjAApn+x17Sn0wX3DHYYQQgyrIZ9UNdSMlfcB4Pzp74Y5EiGEGD4p37MXQgghyV4IIUaElE72ur1tuEMQQoiEkNLJnoa64Y5ACCESQoon+9PDHYEQQiSElE72ur63Z5+RObyBCCHEMEvpZO8v42TlDG8cQggxzFI62ft79koNbyBCCDHMUjrZ+2v2Wg9vHEIIMcwszaC97777SE9Px+Fw4HQ6Wb16Na2traxdu5YzZ85QVFTEihUryM7Ojne89jScGe4IhBAiIVheLuHRRx9l7Nix/q83bdrE7NmzWbp0KZs2bWLTpk3ccccdcQkyGrq7C9pbhzsMIYRICFGXcaqqqli4cCEACxcupKqqatCCGhTNjcMdgRBCJAzLPftVq1YBcO2111JeXo7L5SI/Px+AvLw8XC5X0HYVFRVUVFQAsHr1agoLC6MLNC3NVtvu06doAhz5hSinM+rzWmU3vqGUyLGBxBeLRI4NJL5YDHZslpL9Y489RkFBAS6Xi+985zuUlJT0+75SChVixEt5eTnl5eX+r+vr66MKtLCw0FZb473j5sexedDaEvV5rbIb31BK5NhA4otFIscGEl8sAmMbmHOjYamMU1BQAEBubi7z5s3jyJEj5Obm0tTUBEBTU1O/en5CaG4wP+aPG944hBAiAURM9p2dnXR0dPg/37t3L1OnTqWsrIzKykoAKisrmTdvXnwjtau5EUaPQaXL7FkhhIhYxnG5XDz55JMAeL1errzySubOnct5553H2rVr2bx5s3/oZUJpboS8ApD5VEIIETnZFxcX88QTT5z1eE5ODitXroxLUINBNzdAnpRwhBACUnkGbXMjKq9guKMQQoiEkJLJXmvdW8YZh9RxhBAiRZM9rS3Q0y0jcYQQoldqJvt6cwE0VVg8zIEIIURiSMlkr3uTPUUThjcQIYRIECmZ7H09e8aNNz/KEsdCiBEudZN9Ti4qPUM2LhFCCFI02ev60yD1eiGE8EvJZE/DGVRB0XBHIYQQCSPlkr3WGprqoSAxly0VQojhkHLJnvY26O4asFSC3KAVQoxsqZfs/Usb9/bs5f6sEEKkYLJvMhf7VzJ7Vggh/FIu2eumAT17IYQQqZfsaao3x9bn5vc9JiV7IcQIl3rJvrkRxuah0nxL9UvRXgghUi7Z66Z62bRECCEGSLlkT1OD1OuFEGKAlEr2WmtoPCMjcYQQYoCUSva0uqGjHcYPWNpYVr0UQoxwqZXs62oAUEUT+x6TVS+FECLFkn1zo/lRavZCCNFPSiV77epN9nn54Z8ohBAjTEole1xN4HBAdu5wRyKEEAklxZJ974Qqx8D/ltygFUKMbCmV7LWrCcYOKOHIDVohhEitZE9zU/81cYQQQgCpluxdjai8guGOQgghEk5a5KeYDMPg4YcfpqCggIcffpi6ujrWrVuH2+2mtLSUBx54gLQ0y4cbdNrrhdYW6dkLIUQQlnv2r776KpMmTfJ//Ytf/IIbbriBH/zgB2RlZbF58+a4BGiZu9mcKRss2cv9WSHECGcp2Tc0NLBz506WLFkCmGvQVFdXc/nllwOwaNEiqqqq4helFa4mAFSulHGEEGIgS3WX5557jjvuuIOOjg4A3G43mZmZOJ1OAAoKCmhsbAzatqKigoqKCgBWr15NYWF0s1vT0tLCtu06fpBmIO+caYwKeF5LejpdDkfU5x2s+IZTIscGEl8sEjk2kPhiMdixRUz2O3bsIDc3l9LSUqqrq22foLy8nPLycv/X9fX1to8BUFhYGLatceoEAM3agQp4ntHZiTaMqM87WPENp0SODSS+WCRybCDxxSIwtpKSkpiPFzHZHzx4kO3bt7Nr1y66u7vp6Ojgueeeo729Ha/Xi9PppLGxkYKCYS6f9JZxyM0b1jCEECIRRUz2t99+O7fffjsA1dXV/P73v2f58uU89dRTbNu2jQULFrBlyxbKysriHmxYrkbIzkGljQryTblDK4QY2aIeZ//Zz36WV155hQceeIDW1lYWL148mHHZpl3NZ8+eBZlBK4QQ2BhnDzBr1ixmzZoFQHFxMY8//nhcgopKqwtyZAE0IYQIJnVm0La4UJLshRAiqNRJ9q0uyBkb/HuyLaEQYoRLiWSvPR5obwuxjr3U7IUQIiWSPa0t5kcp4wghRFApkuxdAKhQZRwhhBjhUiPZu309+7xhDUMIIRJVSiR77TZ79nKDVgghgkuJZI8/2Qep2cv9WSGESKFkrxRkZQ93JEIIkZBSJNm3QFYOyuEc7kiEECIhpUSy17JUghBChJUSyR63JHshhAgnRZJ9S+iROLLqpRBCpEiyb5VF0IQQIpykT/ba64VWd4h1cYQQQkAKJHvafLNnZakEIYQIJfmTvdvCImgyg1YIMcKlQLLvXQQtO1TPXm7QCiFE0id77evZj80b1jiEECKRJX2y9y1vLDV7IYQILfmTvW8RtCxJ9kIIEUoKJPvedXGc4dbFkRu0QoiRLemTvXY3hy/hyP1ZIYRI/mSPu0UmVAkhRAQpkOxdMFaSvRBChJP8yb61BRWpZy8leyHECJfUyV4bhrkuTthhl1K0F0KIpE72tLWCNmQteyGEiCAt0hO6u7t59NFH8Xg8eL1eLr/8cm655Rbq6upYt24dbreb0tJSHnjgAdLSIh5ucPkmVIVcKkEIIQRYSPajRo3i0UcfJT09HY/Hw8qVK5k7dy6vvPIKN9xwAwsWLOAnP/kJmzdv5rrrrhuKmPu09K6LIz17IYQIK2IZRylFeno6AF6vF6/Xi1KK6upqLr/8cgAWLVpEVVVVfCMNxr9UQqQbtHKHVggxslmquxiGwUMPPcSHH37Ixz/+cYqLi8nMzMTZO2u1oKCAxsbGoG0rKiqoqKgAYPXq1RQWFkYXaFraWW3btRc3UHDONJwFwY/bkplBp0NFfd5Y4ksUiRwbSHyxSOTYQOKLxWDHZinZOxwOnnjiCdra2njyySepqamxfILy8nLKy8v9X9fX19uPEigsLDyrrVFrxtHY7UGFOK7R0Yk2dNTnjSW+RJHIsYHEF4tEjg0kvlgExlZSUhLz8WyNxsnKymLWrFkcOnSI9vZ2vF4vAI2NjRQUFMQcjG1uF2RmoYb6xrAQQiSZiMm+paWFtrY2wByZs3fvXiZNmsSsWbPYtm0bAFu2bKGsrCy+kQbTKkslCCGEFRG7xE1NTaxfvx7DMNBac8UVV3DppZcyefJk1q1bx69+9SumTZvG4sWLhyLefrTbZXEde7lBK4QY2SIm+3POOYfvfve7Zz1eXFzM448/HpegLHO7oGhC+OcomUErhBBJPoPWjcrKGe4ohBAi4SV3sm9vBUn2QggRUdIme93TDd3dkJU93KEIIUTCS9pkT1ur+THTQrKX+7NCiBEu+ZO9lHGEECKiJE72bgBUVlbcTqF3vol3+W3o7q64nUMIIYZC8ib79vj37I0X1kNHW9+7CCGESFJJm+y1nZp9tFpbzI+9C74JIUSyStpk39ezt5Ls7d+h1R5PTO2FECKRJG+yb3ODckB6ZvjnRTuD9sP3o2snhBAJKImTfau54qUjPv8Ffep4wBdxOYUQQgyZ5E327a3xnVD13pH4HVsIIYZY0iZ73ea2PhInim0J9cmjMbUXQohEkrTJnvY2yLQyxj7Kmn1dbXTthBAiASVvso/jipe6swNcTVBQFJfjCyHEUEviZN8avzH2vl59sW/fRynjCCGSW1Ime20YZhknTjdo9ekPAFDFsW/yK4QQiSApkz2d7aANGzdobR7//ffA4YAJU6JrL4QQCSY5k72dpRKiuD+rPzgBxZNg1Cj7jYUQIgElZ7LvXSpBxWuc/QfvoSafG59jCyHEMEjOZN+7vHE8btDqznaoPw2Tzgl4UOo4QojklpTJXre1mZ/EY+jlBycBUJPOiX5dHSGESDBJmez7Vry0unGJ9Z65/uA985PAnr3coRVCJLnkTPZ2yjh2e+c1J2FMOowbbz8uIYRIUMmZ7NtbYdRo1Ogxg35offIolEztv5qmdOyFEEkuOZN9nGbPasOA946gSi8wH5CavRAiRSRlstfxWt64qR66u6FkyuAfWwghhlFSJnvbPXurQyc/9C2TMHngAayfSwghElBapCfU19ezfv16mpubUUpRXl7O9ddfT2trK2vXruXMmTMUFRWxYsUKsrPjuJlIoLZWGGd1RUrrpRjt24pw4qTeplLGEUKkhojJ3ul08rnPfY7S0lI6Ojp4+OGHmTNnDlu2bGH27NksXbqUTZs2sWnTJu64446hiBnaW1FTpg3+cT9831wjPyev/+MyqUoIkeQilnHy8/MpLS0FICMjg0mTJtHY2EhVVRULFy4EYOHChVRVVcU30kBxqtnrk8fMkTj+Hr307IUQqSFizz5QXV0dx48fZ/r06bhcLvLz8wHIy8vD5XIFbVNRUUFFRQUAq1evprCwMLpA09IoLCxEezzUdXaQWTiebAvHcmdk0K5UxPMaHW2cOXGErJvu8B+3IyebFqCgoABnhPa++BJRIscGEl8sEjk2kPhiMdixWU72nZ2drFmzhjvvvJPMzMx+31NKBfSG+ysvL6e8vNz/dX19fVSBFhYWUl9fj3abLyrtykGnhWMZHR2gjYjn1QffAcNLR8m5/uMabnOmbmNDA8oRfgVMX3yJKJFjA4kvFokcG0h8sQiMraQk9r01LI3G8Xg8rFmzhquuuorLLrsMgNzcXJqamgBoampi7NixMQdjid1F0CxWYnStuSYOk6babiuEEIkuYrLXWvPMM88wadIkbrzxRv/jZWVlVFZWAlBZWcm8efPiF2WgNt/yxoO8CFrNScjIhPzEfEsnhBCxiFjGOXjwIFu3bmXq1Kl87WtfA+C2225j6dKlrF27ls2bN/uHXg6J9t4VLzOtLoJmja45BROnDChHSddeCJEaIib7j3zkI7z00ktBv7dy5cpBDygS7Svj2BmNY2XkZM1J1JwQ705k6KUQIskl3wxa//LGVss4kXvn2t0CbheUTO3/DZlUJYRIEcmb7DMGsYxT07thycBkL4QQKSL5kn1bK4zJQKXZmiIQln8kTsgF0KSMI4RIbsmZ7Ad79myokThSxRFCpIikS/a6PYq17CPcYA0+EifwCfZO13daeUcghEgMSZfsbffsrdxkrTmJmhishBN9116fOo5xz6fQ7+6J+hhCCDFYki/ZD/IiaLrVNxJncDcs0W+ZE8700XcH9bhCCBGN5Ev2ba2owdyS8HQNAKp4UujnRFGO0b0jfDhrIxQhhBh6yZfsB7tnf6bW/GR8kIWGYhln/8F75iGcyXeJhRCpJ6kyke7ugp7uKDYbD9Mzr6s1k3phcXTtQ2k8Y7+NEELESVIle/uzZ4l8j7WuFgqKUKPCL2Fsh+5oD/hi0A4rhBBRS65k3zb4i6DpuloYPzHCk2we850d0TcWQog4SK5k32EmezWYSyWcqUUVhUj20dbs922PPh4hhIiDJEv2veWRjMzwz7NIt7VCqztyz97ucWvf77uvIBOrhBAJIKmSve6MMtmHyrd15kgcFTHZW0/Y+ugBOHEYdbG5o5fkeiFEIkiqZO8r49hb8TJ0KUbXmWPsgw67hKjKOLp6JyiFuvI6222FECJekizZD24ZB98Y+6Jwwy6x1T3X7+6FKaUBN5Glay+EGH7Jlezb20E5YEz64ByvrhbyC1Gjx4R4gr2evXa74OgB1JyyvncFUscRQiSA5Er2ne2QkRF6dUqbLA27BMudc71rG2gDdcl8ZH1kIUQiSa5k39EW5Q5VZ2drrTXUvo8qDlGvx37JXu943XzxmHxu4InsHUQIIeIgqZK97mi3X68PlbHdzeaM3EHaitDbcAYO7EVdusB85yEdeyFEAkmqZE80yT6UmlMAIdaxHyhy77yz8o9gGKj5S3ofiS3bG69XmOP1hRBiECRfsk8fpAlVviWIwyV7G3Wcrre2wjnTURMGLJUczfLILU3o576P8eN/t91WCCGCSbJk3xbdUgnBEu7h/ZA3DvIKomsf+O3mRnoOVaPmXtb3oP+FIopkv3+3+UlOru22QggRTJIl+3bItNuzD94714erURd8NMLIHms9e73nbfPZF19ut2lw1bvMQ0yeFsNBhBCiT9Ike631oNXsdWcHuJpg0jmDEBno3dtwTpgU/Gav3RUztUYf2Gd+ITd5hRCDJGmSPT3d4PUMTs2+oXdjkYIiiw1CZ2xdfxqqdzNmwZIB7xKinFTVVA/NDZFOK4QQtiRNsjfao1kXJ4TGOgBU2N2psNSz1vt2gDbIWHzDgLZRdsvfOxp49OiOIYQQA6RFesLTTz/Nzp07yc3NZc2aNQC0traydu1azpw5Q1FREStWrCA7exA3AQ9Ct/XuUhVNGWdAztT1p81Pxo2Pqn2/bx3YAwVFOCdOhoYGe42DPfvEEXA4zGUhZEKWEGKQROzZL1q0iG984xv9Htu0aROzZ8/m+9//PrNnz2bTpk3xis9PR7txSbAOdu0p80UjNz+KxgExeb3w7l7UrItD3+i1W7M/Um0upBZyvR4hhLAvYrKfOXPmWb32qqoqFi5cCMDChQupqqqKT3QB+so4GTEfS9ecgolTrK+xE6qHfeKwORx05tyzvxfN8sg93XDsEOqCj8rNWSHEoIpYxgnG5XKRn2/2ivPy8nC5XCGfW1FRQUVFBQCrV6+msLAwmlPSc/yAeb4JExll4xitmZm0Kfqd98yH7zO6bAG5EY7TmTsWF5CXnxf0nG2Vx2gFxs1fRFpaWr9zeI0e6oGc7CwyLMbbtXc7zZ4ecssW4Hr9NdLTxzA2yusVaGBsiUbii14ixwYSXywGO7aokn0gpVTYHnJ5eTnl5eX+r+vr66M6T1Zvz765vRNl4xhGezvovvPqtlYMVxNd+YURY9EtbvOcTc2onLOf693+Bkw6h8ZuD4UeT7/j6aYmANxuN20W4zV+9yJk5dAycSpaazrbO+iO8noFKiyM/H8dThJf9BI5NpD4YhEYW0lJ6AUbrYpqNE5ubi5NvcmsqamJsWPHxhxIJLqrw/wkqrXsA8owDebNWVU0Ibr2vke6u+DIftSsS0K0sbkWfmc7+p0dqEvno9IzestAcoNWCDE4okr2ZWVlVFZWAlBZWcm8efMGNahgdGen+ckYmzcuB77r8I/EiTDsEsLn62MHweMx6+vhWB1R88FJ6OpEzb6079yS64UQgyRiGWfdunXs378ft9vNvffeyy233MLSpUtZu3Ytmzdv9g+9jDd/z350bLtU+YddRhpj369RkIcO7jOHR86YFbyN3bXwd79lHm/qeQEHiC7b6+OHQGtU6QVRtRdCpJ6Iyf4rX/lK0MdXrlw52LGEpTs7wZmGSovxNkP9acjIQmVZmRcQZrPyQ9VwznmokOP+rWd73dmBrvwDqmwByjerV6noVszUGuPfHgTA+dPf2W4vhEhNSTODVnd12C/hBDtOfZ31yVShjmEYcOoY6twZFp5sIWFX74SOdtTCTwxoG0Vwh/dH0UgIkeqSJ9l3dkZfwglMuPWn7ZVwzAP0/7L+tLko25Qwq1LaWOLY+PtfIDsHps8c0N5+tjc2vWB+MkiLvAkhUkPyJPuujihH4vSVU7TW0FAXeU0cf9MQyyOfOGx+e0qpldOG5VtITc27GuV09j+3zTKO7mjv69nLDFwhRIDkSfadnbGXcdwu6O6CQptlnIFJd/8uyMqBKedaaBvh2+/sBG2gBi6kZnfopqcH/YsfmV+MGm2rrRAi9SVRsu+IeSSOb9il5Z59qI1Pjh+G0gtQaaMit43UOz951OyFj49t0oR+9WX025WoT90OF8wGw4jpeEKI1JI8yb4r9p59VMMuoV/vXHd2QO37qHOnh29jYW0c3d2F3v466pIrUI4BPwqF5TKOPvQO+tWXUR9biOPGW6NfXlkIkbKSJ9l3RluzD1BXY360OhonWM48edQsu1gZiQOE3fhk1zZzIbX5S4Kf3GKyNzb8HHILULf/a8DBZUaWEKJPUiV7FU0ZJzBh15yCwmJzOYJo4+i9OUukZG9hMI7etsXcLeuC2UHaWxuNo+tq4egB1DXX980dcDgstRVCjBzJk+y7OiE9xtmzbpeFNeyDtuz79PhhGDceNTYvQpsIa+F7euDQO6i5l51dwvE1t5Cv9Yafw6jRqI9d3f8bhiR7IUSf5En2g3GD1u2CnFzrzw9S+9bHD9ko4UDIjL3nbejuQn300lAnj1iK0ccPoXe+iZq/uG/mLcgiakKIsyRFsteGYQ6ZjOEGrdYa3C6UnWTf19j84GqChjooPT9ymwg3SY3KP5r3DmbNDdM+TL3f68X4j6cgfxzqU3cMbCw1eyFEP0mR7OnuMj/GMqlKG9DaYq9nP6AUo/eaO3KpC+daP0SQpKubG+HAXrNH7nAGaeR7YphvvVUJdTU4/vkLqJwBS0w7hmc0jna7MP680SxRCSESSsyblwyJ7t7ljWMp47S1mWPPByZGOw7uM2v+k8+18GTfi8zZ39G73jRXpbz0yjDNQ/fstWGgt7wKeQVw8RXBzz3E4+y1x4PxtbvA60FNnwmy4qYQCSU5evZdvp59DOPsfROqCqJYBM1XxjnyLmr6TGt714Z5it61DSZMQk2aGqZ96FKMfnMzHD+E+vS/BI8lho69Nrxod4v9di/+FLwe8wuZ0CVEwkmSZG/27FUM4+z1+8fNT0qmWG8UkDR14xmzXj9jZujnBz9z/6/a28xROBddZvM4ve3bWtEbfwHTzkddtijoc1SUNXvd3YXxrfswvjrwHkB4xnPfQ2/5g/lOAyTZC5GAkirZx1TGef+EWfOPclkC3bvAmLKa7H097gFJV2/7K3i9qEsXWDhpkIf+60fQ6sLx2XtDv8OIcpy98ePv+ieeaauzd5sbzfkCF16E484v9z4oN4eFSDTJkey7Yyjj9CZE/eEHUFwSfEx76MZ9nx59F8ZkwKRz7bftpT0e9J83mfVsS8stDHihOPgOuupvqBtvRZ0Tob3NcfbGm3+FvVW9LxSYN7Qj0K4mjKe+BWmjcPzzl8C3VpCFtkKIoZUcyd7Xs49luYS6GlS0i41pbS5+du70/ssQW2vc99nmV6ChDsf1t1io+59dijF+/9+QPRZ13dIITa2Ps9eGF+NPG9HPfg+mz0Rdf3PvySysw/9fP4K6Whz3f9O8/xDi3YwQYvglRbLXg1HGaaiD8RPttfElL08PnDpubzLVgOUStNbov/zWrPnPKbN+bt9hXE1wcB+q/B9Rkdaqt7EWvv7ZOvSvn4XZl+L4yrf7lkeONKHr5DHYt8OcAfyROeaDjoBhrkKIhJIUyb6vZx/DaByto6/XnzxmDimcZmEyld+AnnvNSWhuQF2x2PponoCEq//+F/Ph2aFm3Aa2tTD7VmuMDT9Hv10Jcy8ze+dj0s1NzyHsTVZdcxLju49Adg7qHz4dcF5f2+Tp2ev2Nowtr2JU/W24QxEirpJrnH2Mq16q4iiT/YG95ifnz4qice+wzXf3mDH4esER9SVs3dlhloAumA3hdscK0jakd3ai//BrmF2G43PL+l6ALNTsjV8/B9qL42v/1r80lkRlHN3Rjt78CvpPG8wtJrPHwryrhjssy/S+7Rj/8xKdS2+Hj8wd7nBEEkiOZO8bZx/rqpd2hl0Gtn13D0yeZm+phQF70OrtfzfLSDa2RNS+tnuroKUZxxf/d8xj/MG8WW08uw6KJuC450FUeubZcQfp2Wut0Zv+C/ZtRy355Nn3QCy8Kxhuem8Vxp83weFqM86PXgINZ8zZ1UlAnziM8csfw/FDAHTv2yHJXliSJMm+ExxOSIsh3IwsVGZ2dG09PajzbM4IDRyjX3sKjh1E/cPN1pK1v2Hvhx2vm8s8XPBRi+d2hJ6QdeIwxn+ug1Y3jmXf6J/oIaBnH2SZh18/i/7zJtSCJajP3Hn2wW2M5PEf091C59H9GH9/DX3sII7b7rHx7sceffAdjB88BqPHoD5+E2rOPNT0CzF++WP021vtH6+5Ab3rLaj/EPXpO22O9LJPtzRhPPEIpI1GfeYu9Ksvx/V8IrUkR7Lv7kKlp9tLlAP5JvzYEnA+y0MuB9BgbHzBTDBLBu4zG+7U5ogaXX8adm1DXfup8Ovo9GtL8GS9+y2MZ1aD04n64ldR0y8M0jZ471zvfsscNjrtfNS/LA8xc9daGUc3N6APVZtLPhzej8vXVmv0wXfikuz1qeMY678D6Rk4/s8a1ITJfd90OsHwWj+WuwX96kvo137v/7+qRddD0QR7MR09gD5xGDV/CSojM/xz39mJ8ctnoLsbx/3fQl14Ed7Xfo/2Wo9bjGzJkey7OlFjot9wBIgy2fdRk86x28L80N5mlj0WXY8aa2Mtfd/8gG1bzHV0Fn/S3rkHDtv880b0b56H3AIcD/07alxR8KZBeufGG5vRz66D/EIcyx4JM5kr/GgcvW87xh9+Db0T1Bibh/rkbeTNm48rdxzGl2+HUeH29Q3NqPobKr/wrBcwbXjRlX9Ev/SfMHoMjodW90/0YL5rtJg09fFDGE9+A7q7UZdfA3kF6D/+xnJ7AN3TjbF+FVTvMh/IyT17PwJ//Ab6ue+h3/wrTJiMuvPLqAsv6o3bYeu8YmRLnmQf48YlKjeKZB+Y1MKtYxOmrd7zFng8qLIwi56FojV691tmbzpUcg557r5k3/7nTeiXn4Up03AsX4nKGxcxbl/PXjc1oH/1E5h+IY7/9Uj4TVtU8BKQ9vSgf/lj9N/+bG788k+fQ82ca94HSUtjdGEh1NWZT/Z4rP8/fcevfR/9kyfQ4yfiXPXjvsfdLRiPPwhnPoQJk80RR8Fu0jsdlnr2+r2j5iSy7LE4HvgWavI09Pa/m1faGz5u7fXCh++jq/5m3mzvaDfLkh4P9IReJVRX/M5M9Bde1Ft2C+j0OJ0RzyuET1Ike93dhUrPiG47jp7ePwa7m4wPoLJyomv4wXvm1oPRrAJ55F1zDf6b77LXTqm+ev/ubbh/+hTMuthM9JFKQQE1e+3xYPzkCfB6cNz15ci7c/le4Azd7x6x8dM1sPMNc9jp55ahfGP5A5s6HOaLhZ0estaw43WMP/zGfCDg/oPubMd45nGoP426+S7UtUvDvCNxgjf8fQZ94rB5LZTCsfzbfYvYOXv/hEIkXV37PvqPv0FX7wJXo3mNpkxDLfwH837Bo/eb8ziCtW1uNN81TJmGY/mjqIH3rBxOKeMIy5Ii2auZc0l3KDqiadzY22OMJdlHmsQUVF9iUWUL7N9vUMrcWUs5UFdeZ7+tNtCHqjF+/F3SppZi3L3CWs3fF2dHG8Z/PAlH9qNuv9fa7GMVpAT0pw1mov/YQhx3fyV8e5s9Vf3c99FvvAbZvS/EvXvw6uOHMH76JJz5EHXjrTiu+6fwB3I4zOulddCfU+ff/oLx1KMwejSOLz3Yf7VS34zqAUlXf3ASY8Pz5hIUaWnw0UtRsy5GzfkYqqDQfI5vddEB72b0ByfRr/zKHMEFqJs+f3ai951bkr2wKKZkv3v3bp599lkMw2DJkiUsXbp0kMLqz7H4RrIKC+mor7fdVvuWNo4m2fd0mx/POc9+24CcEVUJx7ce0CWXozKz7J+8pRnjh4/BuGLyH/0ejd0WNxTpfUEwfvdLOFSNuuULOK653uJp+0pAuq4G/fsXzYXfpkxD3bk8cntnmqXkpVtbzDr8G6/BBbNxfOXbGOu+DR3tGK+8iH7lRcjIwPGvX7d27X0vgobRl7x95+rqwvXUo5BXgOP//vDsEV0Deva6ox39xw3oV18CQC38hDkKK1gZbpSvbQ/6yH6Mv/wOTh0zy04A50zHcesX4bwgN9IBnE50kPKT1tosE33wHnS0mVt6drZDiwvqasxN6k8eQ827CsddX458fQaRNrxg6OAvXiKuor7ihmHws5/9jG9+85uMGzeORx55hLKyMiZPnhy58VDqTfbR9Ox1Rztgc2eqYGztWdurrhYAR/k/2m/r6x3n5OL46v/DMTYXrL5Q+hL2jjfMmbXXfsr6eXtLQPrtrehf/AgML+qa61E3342ycuO1t2evG+vhg/fQHW3QVA/NTdDSZC4P3WR+D4Bx43HcsQyVNsrsPR88jD5x2CxZ3flllNWb8v7eucfcArOzHVrd0FSP0Tu8UV22MPjQXV/S8njQzY3m0Mi6Wii9AMcXvooKt0SH07wm+uVn+0qUl1xhzmH42MKzdyAbyOGEzg70gb3mNTt1HF3znnkD3NdRGShvHBSXQE83+sj+8McPoA2vOdigrRV9ZD9qbB5qdpn5wtLdZb6oHNhrzlloc0OrG11zkvqebrydHb3PaTd/N9PScHxrHarE5n0wEZOok/2RI0eYMGECxcVmEp0/fz5VVVUJl+wdd33F/IPND3NTMgRVdiX0dKOuWBzFiXtLGkUTYhsyGqpXF4Z2u8wQ7ljWfyNyK3yJL7cAx788YK+t7/+8b7vZK132iL3zp6WhK/+A/uv/9H989Bhzh7DMbMgrMMtiF8zpv9x0b7ZUSz6J49Yv2Yy7993Mijv63lEFGHX+LLzX3xK8bW/P3ljzzb7HPnoJjgdWRh53H/AuQi2+0Rxea6dT4nCYk6r27egNdDSMzYOZc1HnXWiWm3JyIT3DvJ+RmY3qXXLE+OUz6Mo/4v0//2q+ozG85kev1yzDGYZ5H0N7zY+Gt//yHWAe19Nz9k31MenmqKq8QtJKZ5j3+kePNmPw9KArfme+E0uPcYTdIKh3OvH63k3Geea3Y/lKlM3huYMp6mTf2NjIuHF9CXTcuHEcPnz4rOdVVFRQUVEBwOrVqyksLIzqfGlpadG1vbrc/BetpbdZelqw+Lq+/T1GTf8Ijihu7np++N+oUaNxFtlM1oDniyvoOX6IjKuWhIwtFOOqctpbm8n8xE04cm0MFQX0uHG033EvzuISxly+yPJbdV987f98N55Tx3EWTSCt9AKcBYU4CotxWChjdX/2S3iOHSLjBpsT1wDPoo/T1liHIysblZNrfswei7OgEGfJFMaMn4gnxCghnTMP9yf+CTVqNI7cfEbPLiNtxoWWY2i7azmjZsxk9IX25xZ0fvpz9OzdTtqFFzFq+kdwjp8Y9OZ3MD2fuIm2ni6Ucpgv0k4nOJzmfR2nExwOc4XX3q+Vw4EjJxecaTjyC+h6cwsqIxOVlYMjZyxq9BicJVMYPXNuv02G0tLS+l07bRi4x4zBaGqw/f+NB4dSGIFJPpaOWQQ544txjrOew6LOeSEobXWXigG2bdvG7t27uffeewHYunUrhw8f5gtf+ELYdjU1NdGcjsLCQuqjqNkPlUSOL5FjA4kvFokcG0h8sQiMraQkyuXZA0Q9v7ugoICGhr5X54aGBgoKYpu4JIQQIj6iTvbnnXcetbW11NXV4fF4eOONNygrs7BOuxBCiCEXdc3e6XRy9913s2rVKgzD4JprrmHKFJurSgohhBgSMQ12veSSS7jkkksGKxYhhBBxkhw7VQkhhIiJJHshhBgBJNkLIcQIIMleCCFGgKgnVQkhhEgeSdOzf/jhh4c7hLASOb5Ejg0kvlgkcmwg8cVisGNLmmQvhBAiepLshRBiBEiaZF9eHsPKlUMgkeNL5NhA4otFIscGEl8sBjs2uUErhBAjQNL07IUQQkRPkr0QQowASbHr71BtbB5KfX0969evp7m5GaUU5eXlXH/99bz00ku89tprjB1r7hV62223+ReG27hxI5s3b8bhcHDXXXcxd+7cuMZ43333kZ6ejsPhwOl0snr1alpbW1m7di1nzpyhqKiIFStWkJ2djdaaZ599ll27djFmzBiWLVtGaWlpXOKqqalh7dq1/q/r6uq45ZZbaGtrG7Zr9/TTT7Nz505yc3NZs2YNQFTXasuWLWzYsAGAm266iUWLFsUtvhdeeIEdO3aQlpZGcXExy5YtIysri7q6OlasWOHf3GLGjBncc889ABw7doz169fT3d3NxRdfzF133RXbFpkhYovm7yBef9PB4lu7dq1/06T29nYyMzN54oknhvzahcojQ/a7pxOc1+vV999/v/7www91T0+PfvDBB/WpU6eGNIbGxkZ99OhRrbXW7e3tevny5frUqVP6xRdf1L/97W/Pev6pU6f0gw8+qLu7u/Xp06f1/fffr71eb1xjXLZsmXa5XP0ee+GFF/TGjRu11lpv3LhRv/DCC1prrXfs2KFXrVqlDcPQBw8e1I888khcY/Pxer36i1/8oq6rqxvWa1ddXa2PHj2qv/rVr/ofs3ut3G63vu+++7Tb7e73ebzi2717t/Z4PP5YffGdPn263/MCPfzww/rgwYPaMAy9atUqvXPnzrjEZvdnGc+/6WDxBXr++ef1yy+/rLUe+msXKo8M1e9ewpdxAjc2T0tL829sPpTy8/P9r6gZGRlMmjSJxsbGkM+vqqpi/vz5jBo1ivHjxzNhwgSOHDkyVOH2i2PhwoUALFy40H/dtm/fztVXX41SivPPP5+2tjaampriHs++ffuYMGECRWH21R2Kazdz5kyys7PPOq+da7V7927mzJlDdnY22dnZzJkzh927d8ctvosuughn7wbl559/ftjfP4CmpiY6Ojo4//zzUUpx9dVXD8rfTbDYQgn1s4zn33S4+LTWvPnmmyxYsCDsMeJ17ULlkaH63Uv4Mo7Vjc2HSl1dHcePH2f69OkcOHCAP/3pT2zdupXS0lI+//nPk52dTWNjIzNmzPC3KSgoiPjHORhWrVoFwLXXXkt5eTkul4v8fHPT8Ly8PFwuF2Be08CNjMeNG0djY6P/ufHy+uuv9/tDS6RrZ/daDfy9HKo4ATZv3sz8+fP9X9fV1fH1r3+djIwMbr31Vi688MKgfzfxjM/uz3I4/qbfffddcnNzmThxov+x4bp2gXlkqH73Ej7ZJ5LOzk7WrFnDnXfeSWZmJtdddx2f+cxnAHjxxRf5+c9/zrJly4Yltscee4yCggJcLhff+c53ztqgWCkVc80xFh6Phx07dnD77bcDJNS1G2i4r1U4GzZswOl0ctVVVwFmb/Hpp58mJyeHY8eO8cQTT/hr1UMlkX+WgQZ2Nobr2g3MI4Hi+buX8GWcRNnY3OPxsGbNGq666iouu+wywHwVdjgcOBwOlixZwtGjR4PG3NjYGPeYfcfPzc1l3rx5HDlyhNzcXH95pqmpyX8DraCgwL9rPQzNNd21axfTpk0jLy8PSKxrB9i+VsMR55YtW9ixYwfLly/3J4RRo0aRk5MDQGlpKcXFxdTW1g7p343dn+Vw/E17vV7efvvtfu+IhuPaBcsjQ/W7l/DJPhE2Ntda88wzzzBp0iRuvPFG/+OBde63337bvwdvWVkZb7zxBj09PdTV1VFbW8v06dPjFl9nZycdHR3+z/fu3cvUqVMpKyujsrISgMrKSubNm+ePb+vWrWitOXToEJmZmUNewkmUa+dj91rNnTuXPXv20NraSmtrK3v27InriKvdu3fz29/+loceeogxY8b4H29pacEwDABOnz5NbW0txcXF5Ofnk5GRwaFDh9Bas3Xr1rj93dj9WQ7H3/S+ffsoKSnpV/4Y6msXKo8M1e9eUsyg3blzJ88//7x/Y/ObbrppSM9/4MABVq5cydSpU/09qttuu43XX3+dEydOoJSiqKiIe+65x580N2zYwF//+lccDgd33nknF198cdziO336NE8++SRg9mCuvPJKbrrpJtxuN2vXrqW+vv6sIV0/+9nP2LNnD6NHj2bZsmWcd955cYuvs7OTZcuW8cMf/tD/tvUHP/jBsF27devWsX//ftxuN7m5udxyyy3MmzfP9rXavHkzGzduBMzhb9dcc03c4tu4cSMej8d/89E3THDbtm289NJLOJ1OHA4HN998sz8xHT16lKeffpru7m7mzp3L3XffHXOJIFhs1dXVtn+W8fqbDhbf4sWLWb9+PTNmzOC6667zP3eor12oPDJjxowh+d1LimQvhBAiNglfxhFCCBE7SfZCCDECSLIXQogRQJK9EEKMAJLshRBiBJBkL4QQI4AkeyGEGAH+PzyFxfwoQoftAAAAAElFTkSuQmCC\n"
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Evaluate the model on test data\n",
        "\n",
        "correct = 0\n",
        "total = 0\n",
        "with torch.no_grad():\n",
        "    for datavar in test_loader:\n",
        "        images, labels = datavar\n",
        "        images,labels = images.to(device),labels.to(device)\n",
        "        outputs = model(images)\n",
        "        predicted = torch.argmax(outputs.datavar, 1)\n",
        "        total += labels.size(0)\n",
        "        correct += (predicted == labels).sum().item()\n",
        "\n",
        "print('Accuracy of the network on the test images: %f %%'% (100 * correct / total))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x3s5ALmMtGgl",
        "outputId": "65063526-d1ce-48c9-e16e-1724f7d0baf6"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy of the network on the test images: 96.216569 %\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print model's state_dict\n",
        "print(\"Model's state_dict:\")\n",
        "for param_tensor in model.state_dict():\n",
        "    print(param_tensor, \"\\t\", model.state_dict()[param_tensor].size())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i8lnu4XTRN58",
        "outputId": "91b57551-a0bc-4745-aa9f-b3a6eaf4965f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model's state_dict:\n",
            "conv_layer.0.weight \t torch.Size([16, 3, 3, 3])\n",
            "conv_layer.0.bias \t torch.Size([16])\n",
            "conv_layer.3.weight \t torch.Size([32, 16, 3, 3])\n",
            "conv_layer.3.bias \t torch.Size([32])\n",
            "conv_layer.6.weight \t torch.Size([64, 32, 3, 3])\n",
            "conv_layer.6.bias \t torch.Size([64])\n",
            "conv_layer.9.weight \t torch.Size([128, 64, 3, 3])\n",
            "conv_layer.9.bias \t torch.Size([128])\n",
            "fc_layer.0.weight \t torch.Size([512, 51200])\n",
            "fc_layer.0.bias \t torch.Size([512])\n",
            "fc_layer.2.weight \t torch.Size([256, 512])\n",
            "fc_layer.2.bias \t torch.Size([256])\n",
            "fc_layer.4.weight \t torch.Size([3, 256])\n",
            "fc_layer.4.bias \t torch.Size([3])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print optimizer's state_dict\n",
        "print(\"Optimizer's state_dict:\\n\")\n",
        "for var_name in optimizer.state_dict():\n",
        "    print(var_name, \"\\n\", optimizer.state_dict()[var_name],\"\\n\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6JyqUvgORWdH",
        "outputId": "78569890-3f7a-4e41-a706-f929dcf32d3f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Optimizer's state_dict:\n",
            "\n",
            "state \n",
            "\n",
            " {0: {'step': tensor(1920.), 'exp_avg': tensor([[[[-7.3575e-06, -9.4751e-07,  4.2864e-06],\n",
            "          [-1.0329e-05, -2.5403e-06,  3.7023e-06],\n",
            "          [-1.4133e-05, -6.5502e-06,  6.0567e-07]],\n",
            "\n",
            "         [[-7.3575e-06, -9.4751e-07,  4.2864e-06],\n",
            "          [-1.0329e-05, -2.5403e-06,  3.7023e-06],\n",
            "          [-1.4133e-05, -6.5502e-06,  6.0567e-07]],\n",
            "\n",
            "         [[-7.3575e-06, -9.4751e-07,  4.2864e-06],\n",
            "          [-1.0329e-05, -2.5403e-06,  3.7023e-06],\n",
            "          [-1.4133e-05, -6.5502e-06,  6.0567e-07]]],\n",
            "\n",
            "\n",
            "        [[[-1.7765e-05, -6.4728e-06,  1.0907e-05],\n",
            "          [-4.4381e-05, -3.2131e-05, -9.9503e-06],\n",
            "          [-6.4358e-05, -5.5500e-05, -3.5019e-05]],\n",
            "\n",
            "         [[-1.7765e-05, -6.4728e-06,  1.0907e-05],\n",
            "          [-4.4381e-05, -3.2131e-05, -9.9503e-06],\n",
            "          [-6.4358e-05, -5.5500e-05, -3.5019e-05]],\n",
            "\n",
            "         [[-1.7765e-05, -6.4728e-06,  1.0907e-05],\n",
            "          [-4.4381e-05, -3.2131e-05, -9.9503e-06],\n",
            "          [-6.4358e-05, -5.5500e-05, -3.5019e-05]]],\n",
            "\n",
            "\n",
            "        [[[-9.6786e-06, -3.1666e-05, -4.2590e-05],\n",
            "          [-1.5561e-05, -3.7506e-05, -4.8657e-05],\n",
            "          [-1.7046e-05, -3.8464e-05, -5.0350e-05]],\n",
            "\n",
            "         [[-9.6786e-06, -3.1666e-05, -4.2590e-05],\n",
            "          [-1.5561e-05, -3.7506e-05, -4.8657e-05],\n",
            "          [-1.7046e-05, -3.8464e-05, -5.0350e-05]],\n",
            "\n",
            "         [[-9.6786e-06, -3.1666e-05, -4.2590e-05],\n",
            "          [-1.5561e-05, -3.7506e-05, -4.8657e-05],\n",
            "          [-1.7046e-05, -3.8464e-05, -5.0350e-05]]],\n",
            "\n",
            "\n",
            "        [[[-7.4091e-06, -7.2083e-06, -6.6748e-06],\n",
            "          [-7.2394e-06, -7.0180e-06, -6.6034e-06],\n",
            "          [-7.3697e-06, -6.8860e-06, -6.4471e-06]],\n",
            "\n",
            "         [[-7.4091e-06, -7.2083e-06, -6.6748e-06],\n",
            "          [-7.2394e-06, -7.0180e-06, -6.6034e-06],\n",
            "          [-7.3697e-06, -6.8860e-06, -6.4471e-06]],\n",
            "\n",
            "         [[-7.4091e-06, -7.2083e-06, -6.6748e-06],\n",
            "          [-7.2394e-06, -7.0180e-06, -6.6034e-06],\n",
            "          [-7.3697e-06, -6.8860e-06, -6.4471e-06]]],\n",
            "\n",
            "\n",
            "        [[[-1.3307e-05, -5.9616e-07,  7.4740e-06],\n",
            "          [-1.8834e-05, -4.8647e-06,  5.4331e-06],\n",
            "          [-2.3694e-05, -1.1098e-05,  3.3759e-07]],\n",
            "\n",
            "         [[-1.3307e-05, -5.9616e-07,  7.4740e-06],\n",
            "          [-1.8834e-05, -4.8647e-06,  5.4331e-06],\n",
            "          [-2.3694e-05, -1.1098e-05,  3.3759e-07]],\n",
            "\n",
            "         [[-1.3307e-05, -5.9616e-07,  7.4740e-06],\n",
            "          [-1.8834e-05, -4.8647e-06,  5.4331e-06],\n",
            "          [-2.3694e-05, -1.1098e-05,  3.3760e-07]]],\n",
            "\n",
            "\n",
            "        [[[-5.1930e-06, -1.3049e-05, -1.7228e-05],\n",
            "          [-4.2219e-06, -1.2655e-05, -1.7815e-05],\n",
            "          [-3.2043e-06, -1.1100e-05, -1.6851e-05]],\n",
            "\n",
            "         [[-5.1930e-06, -1.3049e-05, -1.7228e-05],\n",
            "          [-4.2219e-06, -1.2655e-05, -1.7815e-05],\n",
            "          [-3.2043e-06, -1.1100e-05, -1.6851e-05]],\n",
            "\n",
            "         [[-5.1930e-06, -1.3049e-05, -1.7228e-05],\n",
            "          [-4.2219e-06, -1.2655e-05, -1.7815e-05],\n",
            "          [-3.2043e-06, -1.1100e-05, -1.6851e-05]]],\n",
            "\n",
            "\n",
            "        [[[-2.2707e-05, -3.5192e-05, -4.2930e-05],\n",
            "          [-3.6344e-05, -4.7961e-05, -5.4880e-05],\n",
            "          [-4.5758e-05, -5.7619e-05, -6.3934e-05]],\n",
            "\n",
            "         [[-2.2707e-05, -3.5192e-05, -4.2930e-05],\n",
            "          [-3.6344e-05, -4.7961e-05, -5.4880e-05],\n",
            "          [-4.5758e-05, -5.7619e-05, -6.3934e-05]],\n",
            "\n",
            "         [[-2.2707e-05, -3.5192e-05, -4.2930e-05],\n",
            "          [-3.6344e-05, -4.7961e-05, -5.4880e-05],\n",
            "          [-4.5758e-05, -5.7619e-05, -6.3934e-05]]],\n",
            "\n",
            "\n",
            "        [[[-2.2550e-06,  8.3767e-07,  2.2526e-06],\n",
            "          [-3.8513e-06,  3.9980e-07,  2.4580e-06],\n",
            "          [-5.8235e-06, -7.0331e-07,  1.9745e-06]],\n",
            "\n",
            "         [[-2.2550e-06,  8.3767e-07,  2.2526e-06],\n",
            "          [-3.8513e-06,  3.9980e-07,  2.4580e-06],\n",
            "          [-5.8235e-06, -7.0331e-07,  1.9745e-06]],\n",
            "\n",
            "         [[-2.2550e-06,  8.3767e-07,  2.2526e-06],\n",
            "          [-3.8513e-06,  3.9980e-07,  2.4580e-06],\n",
            "          [-5.8235e-06, -7.0331e-07,  1.9745e-06]]],\n",
            "\n",
            "\n",
            "        [[[-1.6138e-05, -1.3283e-05, -6.4287e-06],\n",
            "          [-2.2631e-05, -2.0454e-05, -1.3634e-05],\n",
            "          [-2.3494e-05, -2.2588e-05, -1.7338e-05]],\n",
            "\n",
            "         [[-1.6138e-05, -1.3283e-05, -6.4287e-06],\n",
            "          [-2.2631e-05, -2.0454e-05, -1.3634e-05],\n",
            "          [-2.3494e-05, -2.2588e-05, -1.7338e-05]],\n",
            "\n",
            "         [[-1.6138e-05, -1.3283e-05, -6.4287e-06],\n",
            "          [-2.2631e-05, -2.0454e-05, -1.3634e-05],\n",
            "          [-2.3494e-05, -2.2588e-05, -1.7338e-05]]],\n",
            "\n",
            "\n",
            "        [[[ 2.7414e-07,  3.9062e-07,  3.2443e-07],\n",
            "          [ 6.0511e-07,  5.7305e-07,  4.5510e-07],\n",
            "          [ 1.6831e-06,  1.6789e-06,  1.7257e-06]],\n",
            "\n",
            "         [[ 2.7414e-07,  3.9062e-07,  3.2443e-07],\n",
            "          [ 6.0511e-07,  5.7305e-07,  4.5510e-07],\n",
            "          [ 1.6831e-06,  1.6789e-06,  1.7257e-06]],\n",
            "\n",
            "         [[ 2.7414e-07,  3.9062e-07,  3.2443e-07],\n",
            "          [ 6.0511e-07,  5.7305e-07,  4.5510e-07],\n",
            "          [ 1.6831e-06,  1.6789e-06,  1.7257e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 3.5252e-06,  5.6884e-06,  5.8915e-06],\n",
            "          [ 2.4851e-06,  5.5339e-06,  6.2564e-06],\n",
            "          [ 7.8699e-07,  4.4850e-06,  5.8503e-06]],\n",
            "\n",
            "         [[ 3.5252e-06,  5.6884e-06,  5.8915e-06],\n",
            "          [ 2.4851e-06,  5.5339e-06,  6.2564e-06],\n",
            "          [ 7.8699e-07,  4.4850e-06,  5.8503e-06]],\n",
            "\n",
            "         [[ 3.5252e-06,  5.6884e-06,  5.8915e-06],\n",
            "          [ 2.4851e-06,  5.5339e-06,  6.2564e-06],\n",
            "          [ 7.8699e-07,  4.4850e-06,  5.8503e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 5.6643e-07,  4.7586e-06,  5.3443e-06],\n",
            "          [-1.1165e-06,  3.5008e-06,  5.1984e-06],\n",
            "          [-1.3272e-06,  3.4924e-06,  5.6492e-06]],\n",
            "\n",
            "         [[ 5.6643e-07,  4.7586e-06,  5.3443e-06],\n",
            "          [-1.1165e-06,  3.5008e-06,  5.1984e-06],\n",
            "          [-1.3272e-06,  3.4924e-06,  5.6492e-06]],\n",
            "\n",
            "         [[ 5.6643e-07,  4.7586e-06,  5.3443e-06],\n",
            "          [-1.1165e-06,  3.5008e-06,  5.1984e-06],\n",
            "          [-1.3272e-06,  3.4924e-06,  5.6492e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 1.1487e-06, -1.0750e-06, -1.3223e-06],\n",
            "          [ 1.0746e-06, -1.7661e-06, -1.8995e-06],\n",
            "          [ 8.8249e-07, -1.7489e-06, -3.4432e-06]],\n",
            "\n",
            "         [[ 1.1487e-06, -1.0750e-06, -1.3223e-06],\n",
            "          [ 1.0746e-06, -1.7661e-06, -1.8995e-06],\n",
            "          [ 8.8249e-07, -1.7489e-06, -3.4432e-06]],\n",
            "\n",
            "         [[ 1.1487e-06, -1.0750e-06, -1.3223e-06],\n",
            "          [ 1.0746e-06, -1.7661e-06, -1.8995e-06],\n",
            "          [ 8.8249e-07, -1.7489e-06, -3.4432e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 2.4483e-06,  2.4874e-06,  3.7010e-06],\n",
            "          [ 3.4584e-06,  3.1332e-06,  3.8119e-06],\n",
            "          [ 4.7700e-06,  4.3336e-06,  4.5659e-06]],\n",
            "\n",
            "         [[ 2.4483e-06,  2.4874e-06,  3.7010e-06],\n",
            "          [ 3.4584e-06,  3.1332e-06,  3.8119e-06],\n",
            "          [ 4.7700e-06,  4.3336e-06,  4.5659e-06]],\n",
            "\n",
            "         [[ 2.4483e-06,  2.4874e-06,  3.7010e-06],\n",
            "          [ 3.4584e-06,  3.1332e-06,  3.8119e-06],\n",
            "          [ 4.7700e-06,  4.3336e-06,  4.5659e-06]]],\n",
            "\n",
            "\n",
            "        [[[-3.4200e-05, -3.6484e-05, -3.6151e-05],\n",
            "          [-3.1965e-05, -3.4976e-05, -3.4968e-05],\n",
            "          [-2.7874e-05, -3.1459e-05, -3.2560e-05]],\n",
            "\n",
            "         [[-3.4200e-05, -3.6484e-05, -3.6151e-05],\n",
            "          [-3.1965e-05, -3.4976e-05, -3.4968e-05],\n",
            "          [-2.7874e-05, -3.1459e-05, -3.2560e-05]],\n",
            "\n",
            "         [[-3.4200e-05, -3.6484e-05, -3.6151e-05],\n",
            "          [-3.1965e-05, -3.4976e-05, -3.4968e-05],\n",
            "          [-2.7874e-05, -3.1459e-05, -3.2560e-05]]],\n",
            "\n",
            "\n",
            "        [[[-3.2314e-06,  1.2041e-06,  6.4791e-06],\n",
            "          [-6.0680e-06, -1.7523e-06,  4.0849e-06],\n",
            "          [-5.9618e-06, -2.4209e-06,  2.7963e-06]],\n",
            "\n",
            "         [[-3.2314e-06,  1.2041e-06,  6.4791e-06],\n",
            "          [-6.0680e-06, -1.7523e-06,  4.0849e-06],\n",
            "          [-5.9618e-06, -2.4209e-06,  2.7963e-06]],\n",
            "\n",
            "         [[-3.2314e-06,  1.2041e-06,  6.4791e-06],\n",
            "          [-6.0680e-06, -1.7523e-06,  4.0849e-06],\n",
            "          [-5.9618e-06, -2.4209e-06,  2.7963e-06]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[6.9233e-04, 5.8245e-04, 5.6565e-04],\n",
            "          [7.5622e-04, 6.0840e-04, 5.7777e-04],\n",
            "          [7.3378e-04, 5.8325e-04, 5.5130e-04]],\n",
            "\n",
            "         [[6.9233e-04, 5.8245e-04, 5.6565e-04],\n",
            "          [7.5622e-04, 6.0840e-04, 5.7777e-04],\n",
            "          [7.3378e-04, 5.8325e-04, 5.5130e-04]],\n",
            "\n",
            "         [[6.9233e-04, 5.8245e-04, 5.6565e-04],\n",
            "          [7.5622e-04, 6.0840e-04, 5.7777e-04],\n",
            "          [7.3378e-04, 5.8325e-04, 5.5130e-04]]],\n",
            "\n",
            "\n",
            "        [[[3.6994e-03, 3.5468e-03, 3.5377e-03],\n",
            "          [4.1529e-03, 3.8985e-03, 3.7753e-03],\n",
            "          [4.4276e-03, 4.2201e-03, 3.9625e-03]],\n",
            "\n",
            "         [[3.6994e-03, 3.5468e-03, 3.5377e-03],\n",
            "          [4.1529e-03, 3.8985e-03, 3.7753e-03],\n",
            "          [4.4276e-03, 4.2201e-03, 3.9625e-03]],\n",
            "\n",
            "         [[3.6994e-03, 3.5468e-03, 3.5377e-03],\n",
            "          [4.1529e-03, 3.8985e-03, 3.7753e-03],\n",
            "          [4.4276e-03, 4.2201e-03, 3.9625e-03]]],\n",
            "\n",
            "\n",
            "        [[[4.1732e-03, 4.6608e-03, 5.3351e-03],\n",
            "          [4.4604e-03, 5.1009e-03, 5.9612e-03],\n",
            "          [4.5377e-03, 5.1495e-03, 5.9561e-03]],\n",
            "\n",
            "         [[4.1732e-03, 4.6608e-03, 5.3351e-03],\n",
            "          [4.4604e-03, 5.1009e-03, 5.9612e-03],\n",
            "          [4.5377e-03, 5.1495e-03, 5.9561e-03]],\n",
            "\n",
            "         [[4.1732e-03, 4.6608e-03, 5.3351e-03],\n",
            "          [4.4604e-03, 5.1009e-03, 5.9612e-03],\n",
            "          [4.5377e-03, 5.1495e-03, 5.9561e-03]]],\n",
            "\n",
            "\n",
            "        [[[6.7822e-05, 6.7819e-05, 6.6989e-05],\n",
            "          [6.7060e-05, 6.8859e-05, 6.9238e-05],\n",
            "          [6.3543e-05, 6.5032e-05, 6.6009e-05]],\n",
            "\n",
            "         [[6.7822e-05, 6.7819e-05, 6.6989e-05],\n",
            "          [6.7060e-05, 6.8859e-05, 6.9238e-05],\n",
            "          [6.3543e-05, 6.5032e-05, 6.6009e-05]],\n",
            "\n",
            "         [[6.7822e-05, 6.7819e-05, 6.6989e-05],\n",
            "          [6.7060e-05, 6.8859e-05, 6.9238e-05],\n",
            "          [6.3543e-05, 6.5032e-05, 6.6009e-05]]],\n",
            "\n",
            "\n",
            "        [[[1.0434e-03, 8.8144e-04, 9.1152e-04],\n",
            "          [1.1830e-03, 9.2586e-04, 9.0460e-04],\n",
            "          [1.1952e-03, 9.3165e-04, 8.8450e-04]],\n",
            "\n",
            "         [[1.0434e-03, 8.8144e-04, 9.1152e-04],\n",
            "          [1.1830e-03, 9.2586e-04, 9.0460e-04],\n",
            "          [1.1952e-03, 9.3165e-04, 8.8450e-04]],\n",
            "\n",
            "         [[1.0434e-03, 8.8144e-04, 9.1152e-04],\n",
            "          [1.1830e-03, 9.2586e-04, 9.0460e-04],\n",
            "          [1.1952e-03, 9.3165e-04, 8.8450e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.6098e-04, 3.0716e-04, 3.7253e-04],\n",
            "          [2.5475e-04, 3.0449e-04, 4.0372e-04],\n",
            "          [2.4034e-04, 2.9924e-04, 4.2411e-04]],\n",
            "\n",
            "         [[2.6098e-04, 3.0716e-04, 3.7253e-04],\n",
            "          [2.5475e-04, 3.0449e-04, 4.0372e-04],\n",
            "          [2.4034e-04, 2.9924e-04, 4.2411e-04]],\n",
            "\n",
            "         [[2.6098e-04, 3.0716e-04, 3.7253e-04],\n",
            "          [2.5475e-04, 3.0449e-04, 4.0372e-04],\n",
            "          [2.4034e-04, 2.9924e-04, 4.2411e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.3940e-03, 2.5708e-03, 2.8942e-03],\n",
            "          [2.6388e-03, 2.9595e-03, 3.2565e-03],\n",
            "          [2.7508e-03, 3.0936e-03, 3.2475e-03]],\n",
            "\n",
            "         [[2.3940e-03, 2.5708e-03, 2.8942e-03],\n",
            "          [2.6388e-03, 2.9595e-03, 3.2565e-03],\n",
            "          [2.7508e-03, 3.0936e-03, 3.2475e-03]],\n",
            "\n",
            "         [[2.3940e-03, 2.5708e-03, 2.8942e-03],\n",
            "          [2.6388e-03, 2.9595e-03, 3.2565e-03],\n",
            "          [2.7508e-03, 3.0936e-03, 3.2475e-03]]],\n",
            "\n",
            "\n",
            "        [[[1.1810e-04, 1.1746e-04, 1.2422e-04],\n",
            "          [1.2500e-04, 1.1630e-04, 1.2148e-04],\n",
            "          [1.2925e-04, 1.1410e-04, 1.1756e-04]],\n",
            "\n",
            "         [[1.1810e-04, 1.1746e-04, 1.2422e-04],\n",
            "          [1.2500e-04, 1.1630e-04, 1.2148e-04],\n",
            "          [1.2925e-04, 1.1410e-04, 1.1756e-04]],\n",
            "\n",
            "         [[1.1810e-04, 1.1746e-04, 1.2422e-04],\n",
            "          [1.2500e-04, 1.1630e-04, 1.2148e-04],\n",
            "          [1.2925e-04, 1.1410e-04, 1.1756e-04]]],\n",
            "\n",
            "\n",
            "        [[[1.6569e-03, 1.6560e-03, 1.6334e-03],\n",
            "          [1.9106e-03, 1.9243e-03, 1.8827e-03],\n",
            "          [1.9478e-03, 1.9783e-03, 1.9187e-03]],\n",
            "\n",
            "         [[1.6569e-03, 1.6560e-03, 1.6334e-03],\n",
            "          [1.9106e-03, 1.9243e-03, 1.8827e-03],\n",
            "          [1.9478e-03, 1.9783e-03, 1.9187e-03]],\n",
            "\n",
            "         [[1.6569e-03, 1.6560e-03, 1.6334e-03],\n",
            "          [1.9106e-03, 1.9243e-03, 1.8827e-03],\n",
            "          [1.9478e-03, 1.9783e-03, 1.9187e-03]]],\n",
            "\n",
            "\n",
            "        [[[6.9400e-05, 7.0356e-05, 7.0680e-05],\n",
            "          [7.8981e-05, 8.0176e-05, 8.0233e-05],\n",
            "          [8.3490e-05, 8.5053e-05, 8.4979e-05]],\n",
            "\n",
            "         [[6.9400e-05, 7.0356e-05, 7.0680e-05],\n",
            "          [7.8981e-05, 8.0176e-05, 8.0233e-05],\n",
            "          [8.3490e-05, 8.5053e-05, 8.4979e-05]],\n",
            "\n",
            "         [[6.9400e-05, 7.0356e-05, 7.0680e-05],\n",
            "          [7.8981e-05, 8.0176e-05, 8.0233e-05],\n",
            "          [8.3490e-05, 8.5053e-05, 8.4979e-05]]],\n",
            "\n",
            "\n",
            "        [[[2.8920e-04, 2.9655e-04, 2.9166e-04],\n",
            "          [2.8871e-04, 3.0215e-04, 3.0035e-04],\n",
            "          [2.6910e-04, 2.8434e-04, 2.9013e-04]],\n",
            "\n",
            "         [[2.8920e-04, 2.9655e-04, 2.9166e-04],\n",
            "          [2.8871e-04, 3.0215e-04, 3.0035e-04],\n",
            "          [2.6910e-04, 2.8434e-04, 2.9013e-04]],\n",
            "\n",
            "         [[2.8920e-04, 2.9655e-04, 2.9166e-04],\n",
            "          [2.8871e-04, 3.0215e-04, 3.0035e-04],\n",
            "          [2.6910e-04, 2.8434e-04, 2.9013e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.7688e-05, 2.6120e-05, 3.3637e-05],\n",
            "          [3.0652e-05, 2.5277e-05, 3.5234e-05],\n",
            "          [3.2862e-05, 2.6018e-05, 3.5418e-05]],\n",
            "\n",
            "         [[2.7688e-05, 2.6120e-05, 3.3637e-05],\n",
            "          [3.0652e-05, 2.5277e-05, 3.5234e-05],\n",
            "          [3.2862e-05, 2.6018e-05, 3.5418e-05]],\n",
            "\n",
            "         [[2.7688e-05, 2.6120e-05, 3.3637e-05],\n",
            "          [3.0652e-05, 2.5277e-05, 3.5234e-05],\n",
            "          [3.2862e-05, 2.6018e-05, 3.5418e-05]]],\n",
            "\n",
            "\n",
            "        [[[2.2384e-04, 1.8698e-04, 1.6375e-04],\n",
            "          [1.8729e-04, 1.5506e-04, 1.4727e-04],\n",
            "          [1.5879e-04, 1.5340e-04, 1.5883e-04]],\n",
            "\n",
            "         [[2.2384e-04, 1.8698e-04, 1.6375e-04],\n",
            "          [1.8729e-04, 1.5506e-04, 1.4727e-04],\n",
            "          [1.5879e-04, 1.5340e-04, 1.5883e-04]],\n",
            "\n",
            "         [[2.2384e-04, 1.8698e-04, 1.6375e-04],\n",
            "          [1.8729e-04, 1.5506e-04, 1.4727e-04],\n",
            "          [1.5879e-04, 1.5340e-04, 1.5883e-04]]],\n",
            "\n",
            "\n",
            "        [[[4.3867e-04, 4.4752e-04, 4.4704e-04],\n",
            "          [4.4700e-04, 4.5756e-04, 4.5253e-04],\n",
            "          [4.5103e-04, 4.5647e-04, 4.4832e-04]],\n",
            "\n",
            "         [[4.3867e-04, 4.4752e-04, 4.4704e-04],\n",
            "          [4.4700e-04, 4.5756e-04, 4.5253e-04],\n",
            "          [4.5103e-04, 4.5647e-04, 4.4832e-04]],\n",
            "\n",
            "         [[4.3867e-04, 4.4752e-04, 4.4704e-04],\n",
            "          [4.4700e-04, 4.5756e-04, 4.5253e-04],\n",
            "          [4.5103e-04, 4.5647e-04, 4.4832e-04]]],\n",
            "\n",
            "\n",
            "        [[[5.0045e-04, 5.4612e-04, 5.4447e-04],\n",
            "          [4.9700e-04, 5.4348e-04, 5.3879e-04],\n",
            "          [4.7367e-04, 5.1453e-04, 5.1433e-04]],\n",
            "\n",
            "         [[5.0045e-04, 5.4612e-04, 5.4447e-04],\n",
            "          [4.9700e-04, 5.4348e-04, 5.3879e-04],\n",
            "          [4.7367e-04, 5.1453e-04, 5.1433e-04]],\n",
            "\n",
            "         [[5.0045e-04, 5.4612e-04, 5.4447e-04],\n",
            "          [4.9700e-04, 5.4348e-04, 5.3879e-04],\n",
            "          [4.7367e-04, 5.1453e-04, 5.1433e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.8639e-04, 2.5310e-04, 2.5728e-04],\n",
            "          [3.0973e-04, 2.6474e-04, 2.5774e-04],\n",
            "          [3.2370e-04, 2.7704e-04, 2.5908e-04]],\n",
            "\n",
            "         [[2.8639e-04, 2.5310e-04, 2.5728e-04],\n",
            "          [3.0973e-04, 2.6474e-04, 2.5774e-04],\n",
            "          [3.2370e-04, 2.7704e-04, 2.5908e-04]],\n",
            "\n",
            "         [[2.8639e-04, 2.5310e-04, 2.5728e-04],\n",
            "          [3.0973e-04, 2.6474e-04, 2.5774e-04],\n",
            "          [3.2370e-04, 2.7704e-04, 2.5908e-04]]]], device='cuda:0')}, 1: {'step': tensor(1920.), 'exp_avg': tensor([-9.7154e-06, -2.0288e-04, -7.9073e-05,  2.0579e-05, -6.1251e-05,\n",
            "        -1.0900e-05, -7.2585e-05, -6.8127e-06, -3.1924e-05, -2.7055e-06,\n",
            "        -3.3172e-05,  1.1339e-05,  7.9015e-07,  2.7123e-05,  1.6642e-04,\n",
            "        -9.2228e-06], device='cuda:0'), 'exp_avg_sq': tensor([5.7016e-03, 2.9535e-02, 1.6589e-02, 2.3251e-04, 5.9860e-03, 1.7322e-03,\n",
            "        1.2739e-02, 7.6791e-04, 6.9435e-03, 1.6062e-04, 4.6477e-03, 9.2923e-05,\n",
            "        6.1107e-04, 3.2373e-03, 5.5482e-03, 2.0157e-03], device='cuda:0')}, 2: {'step': tensor(1920.), 'exp_avg': tensor([[[[-4.3435e-06, -4.0039e-06, -1.9901e-06],\n",
            "          [-3.1395e-06, -3.8935e-06, -2.0991e-06],\n",
            "          [-1.8131e-06, -2.8209e-06, -1.3387e-06]],\n",
            "\n",
            "         [[-3.4029e-06, -2.7497e-06, -1.3980e-06],\n",
            "          [-4.8392e-06, -4.7800e-06, -3.5608e-06],\n",
            "          [-4.1998e-06, -5.3759e-06, -5.1059e-06]],\n",
            "\n",
            "         [[-5.5797e-07,  1.1877e-07,  5.0290e-07],\n",
            "          [-8.2814e-07,  1.1665e-07, -8.3020e-08],\n",
            "          [-1.5945e-06, -2.6390e-07,  1.1582e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-4.1854e-07, -1.7174e-07, -2.0854e-07],\n",
            "          [-2.1166e-06, -8.8267e-07, -4.2267e-07],\n",
            "          [-5.8705e-06, -4.3641e-06, -3.2761e-06]],\n",
            "\n",
            "         [[-3.4221e-06, -5.7409e-06, -5.3924e-06],\n",
            "          [-1.1999e-06, -4.0971e-06, -3.8425e-06],\n",
            "          [ 1.1243e-07, -1.0883e-06, -1.6002e-06]],\n",
            "\n",
            "         [[-3.2838e-06, -2.8302e-06, -1.3931e-06],\n",
            "          [-3.5091e-06, -3.3278e-06, -1.4386e-06],\n",
            "          [-3.0121e-06, -3.4070e-06, -2.0103e-06]]],\n",
            "\n",
            "\n",
            "        [[[-8.4926e-08, -2.3520e-07, -2.6415e-07],\n",
            "          [-2.5811e-07, -3.8437e-07, -4.5825e-07],\n",
            "          [-1.4653e-07, -2.9087e-07, -2.9993e-07]],\n",
            "\n",
            "         [[-5.2334e-08, -6.4037e-08, -2.5886e-09],\n",
            "          [-2.4205e-07, -1.8654e-07, -1.7894e-07],\n",
            "          [-3.3824e-07, -4.4560e-07, -4.3719e-07]],\n",
            "\n",
            "         [[ 1.0322e-08,  7.3865e-09, -1.5297e-08],\n",
            "          [-5.2055e-09,  1.1233e-08,  9.3980e-09],\n",
            "          [ 3.0144e-09,  1.2052e-08,  1.5149e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.8393e-06, -1.9252e-06, -2.0536e-06],\n",
            "          [-1.9648e-06, -1.9995e-06, -1.9772e-06],\n",
            "          [-2.1253e-06, -2.0437e-06, -2.0800e-06]],\n",
            "\n",
            "         [[-6.2839e-07, -6.5196e-07, -7.5317e-07],\n",
            "          [-6.4679e-07, -6.9613e-07, -8.2352e-07],\n",
            "          [-3.6406e-07, -4.3342e-07, -5.7060e-07]],\n",
            "\n",
            "         [[-5.6393e-07, -7.8381e-07, -8.1618e-07],\n",
            "          [-6.1319e-07, -8.1886e-07, -8.2973e-07],\n",
            "          [-6.5285e-07, -8.0454e-07, -7.8428e-07]]],\n",
            "\n",
            "\n",
            "        [[[-3.3833e-07, -5.4543e-07, -3.4149e-07],\n",
            "          [-1.9021e-07, -5.7855e-07, -3.8786e-07],\n",
            "          [ 1.8509e-07, -2.3147e-07, -3.5918e-08]],\n",
            "\n",
            "         [[-4.9805e-07, -4.9689e-07, -2.6734e-07],\n",
            "          [ 1.0092e-08,  2.8648e-08,  1.4603e-07],\n",
            "          [ 3.0478e-07,  1.6285e-07,  1.8456e-07]],\n",
            "\n",
            "         [[-1.2853e-06, -1.1739e-06, -9.1186e-07],\n",
            "          [-1.2292e-06, -1.0579e-06, -6.7037e-07],\n",
            "          [-1.1982e-06, -8.7880e-07, -6.3996e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-2.6475e-06, -3.0231e-06, -2.6339e-06],\n",
            "          [-2.3839e-06, -2.5956e-06, -2.0190e-06],\n",
            "          [-1.9728e-06, -2.2067e-06, -1.5277e-06]],\n",
            "\n",
            "         [[ 1.7683e-07,  1.4138e-07,  8.9545e-08],\n",
            "          [ 3.0693e-07,  2.4007e-07,  1.1634e-07],\n",
            "          [ 5.7590e-07,  6.8124e-07,  4.8484e-07]],\n",
            "\n",
            "         [[-4.3380e-07, -7.5987e-07, -7.6618e-07],\n",
            "          [-4.3821e-07, -8.3840e-07, -7.4160e-07],\n",
            "          [-3.5936e-07, -7.3961e-07, -5.9115e-07]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[ 1.0481e-07, -1.0513e-06, -2.8816e-06],\n",
            "          [ 5.8942e-07, -1.1755e-06, -2.7485e-06],\n",
            "          [-9.9040e-08, -1.4182e-06, -3.4497e-06]],\n",
            "\n",
            "         [[-2.8382e-07, -1.2709e-07,  1.2721e-07],\n",
            "          [-7.3051e-07, -7.2296e-07, -5.2357e-07],\n",
            "          [-1.2158e-07, -6.3448e-07, -9.8232e-07]],\n",
            "\n",
            "         [[-1.6734e-07, -1.5104e-06, -2.4915e-07],\n",
            "          [-6.2312e-07, -1.1420e-06, -2.1440e-07],\n",
            "          [-9.3473e-07, -1.1216e-06, -1.0077e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.2239e-06, -1.6143e-06, -1.1894e-06],\n",
            "          [-2.4977e-06, -2.6029e-06, -1.3724e-06],\n",
            "          [-2.5648e-06, -2.6680e-06, -1.3775e-06]],\n",
            "\n",
            "         [[ 2.6474e-06,  2.0968e-06,  8.8308e-08],\n",
            "          [ 2.7900e-06,  2.1766e-06, -1.0840e-07],\n",
            "          [ 1.9674e-06,  1.3813e-06, -1.0300e-06]],\n",
            "\n",
            "         [[ 2.2454e-07, -7.6989e-07, -1.8412e-06],\n",
            "          [-2.8005e-07, -1.1206e-06, -2.4323e-06],\n",
            "          [-2.0279e-07, -1.3359e-06, -2.5883e-06]]],\n",
            "\n",
            "\n",
            "        [[[-3.8835e-07, -4.8238e-07, -5.0633e-07],\n",
            "          [-5.4608e-07, -6.5752e-07, -4.5918e-07],\n",
            "          [-1.6925e-07, -3.9736e-07, -2.5859e-07]],\n",
            "\n",
            "         [[-1.3299e-06, -1.5035e-06, -2.0737e-06],\n",
            "          [-1.1050e-06, -1.1473e-06, -1.5054e-06],\n",
            "          [ 9.3300e-09,  6.3746e-08, -1.0616e-08]],\n",
            "\n",
            "         [[-4.2777e-07, -3.6246e-06, -2.3481e-06],\n",
            "          [-1.2486e-06, -4.9658e-06, -3.7074e-06],\n",
            "          [-1.2227e-06, -4.7366e-06, -3.3306e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-5.3059e-06, -7.7284e-06, -8.8004e-06],\n",
            "          [-6.8782e-06, -9.7401e-06, -1.1075e-05],\n",
            "          [-9.0978e-06, -1.2356e-05, -1.3728e-05]],\n",
            "\n",
            "         [[-6.1530e-07, -2.2803e-07, -1.6233e-07],\n",
            "          [-4.7878e-07, -1.0911e-07, -2.9810e-08],\n",
            "          [-3.4518e-07, -3.2235e-08, -1.4502e-08]],\n",
            "\n",
            "         [[-1.3107e-06, -1.6369e-06, -1.9103e-06],\n",
            "          [-8.7137e-07, -1.3883e-06, -1.6281e-06],\n",
            "          [-1.5764e-06, -2.0251e-06, -2.3442e-06]]],\n",
            "\n",
            "\n",
            "        [[[-2.0387e-07, -1.4503e-07, -2.3428e-07],\n",
            "          [-1.2098e-07,  2.3235e-08,  1.5016e-07],\n",
            "          [ 7.8501e-08,  1.6835e-07,  4.7619e-07]],\n",
            "\n",
            "         [[ 3.8793e-07,  4.2616e-07,  4.7380e-07],\n",
            "          [ 6.7335e-08,  2.3715e-07,  4.2699e-07],\n",
            "          [-5.1355e-08, -6.8583e-08,  1.9241e-07]],\n",
            "\n",
            "         [[-8.6834e-08,  1.5704e-07, -1.1895e-07],\n",
            "          [-3.9044e-08,  1.6301e-07,  8.5924e-08],\n",
            "          [-9.1763e-08,  2.0396e-07,  1.0518e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-9.1751e-08, -8.1914e-08, -1.0036e-07],\n",
            "          [-1.3705e-09, -3.1394e-08, -1.6460e-08],\n",
            "          [ 5.7716e-09, -8.1750e-08,  9.7859e-10]],\n",
            "\n",
            "         [[ 1.3408e-06,  1.4776e-06,  1.1085e-06],\n",
            "          [ 8.6693e-07,  8.3343e-07,  4.6222e-07],\n",
            "          [ 1.2823e-06,  9.4316e-07,  4.7014e-07]],\n",
            "\n",
            "         [[-1.6738e-07, -2.3623e-08, -1.5730e-07],\n",
            "          [-1.1287e-07,  2.4444e-09,  2.3175e-08],\n",
            "          [-1.0246e-07,  3.8064e-08,  1.7391e-07]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[1.0138e-05, 9.6506e-06, 5.0039e-06],\n",
            "          [8.2859e-06, 9.9527e-06, 5.8054e-06],\n",
            "          [5.6481e-06, 8.1714e-06, 5.8634e-06]],\n",
            "\n",
            "         [[3.5795e-06, 2.5180e-06, 1.8674e-06],\n",
            "          [7.1403e-06, 7.5306e-06, 6.3593e-06],\n",
            "          [4.8481e-06, 6.2643e-06, 6.1142e-06]],\n",
            "\n",
            "         [[4.0954e-07, 2.0774e-07, 2.0646e-06],\n",
            "          [7.7152e-07, 4.6130e-07, 2.1987e-06],\n",
            "          [1.3477e-06, 5.9053e-07, 1.7341e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[4.1998e-06, 2.1624e-06, 2.4349e-06],\n",
            "          [6.8380e-06, 4.6303e-06, 4.3965e-06],\n",
            "          [1.3513e-05, 1.0773e-05, 9.0189e-06]],\n",
            "\n",
            "         [[1.0981e-04, 1.2335e-04, 1.2478e-04],\n",
            "          [8.8024e-05, 1.0315e-04, 1.0832e-04],\n",
            "          [7.1975e-05, 8.3215e-05, 8.9918e-05]],\n",
            "\n",
            "         [[1.4426e-05, 1.2381e-05, 8.4951e-06],\n",
            "          [1.3568e-05, 1.2705e-05, 8.4219e-06],\n",
            "          [1.2017e-05, 1.3272e-05, 1.0497e-05]]],\n",
            "\n",
            "\n",
            "        [[[1.1617e-06, 1.2420e-06, 1.3282e-06],\n",
            "          [1.8101e-06, 1.6211e-06, 1.6455e-06],\n",
            "          [1.4124e-06, 1.1532e-06, 1.3458e-06]],\n",
            "\n",
            "         [[4.5476e-07, 5.1739e-07, 5.5399e-07],\n",
            "          [5.1504e-07, 5.1795e-07, 5.7107e-07],\n",
            "          [7.8314e-07, 8.1396e-07, 8.3315e-07]],\n",
            "\n",
            "         [[5.4752e-08, 5.1368e-08, 9.6474e-08],\n",
            "          [4.2240e-08, 5.3443e-08, 1.1430e-07],\n",
            "          [5.8875e-08, 7.4543e-08, 1.2304e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[3.9679e-05, 3.9987e-05, 3.9340e-05],\n",
            "          [3.8708e-05, 3.9367e-05, 3.9960e-05],\n",
            "          [3.8698e-05, 3.9022e-05, 3.9881e-05]],\n",
            "\n",
            "         [[2.4210e-05, 2.4568e-05, 2.4886e-05],\n",
            "          [2.6093e-05, 2.6351e-05, 2.6452e-05],\n",
            "          [2.5065e-05, 2.5396e-05, 2.5318e-05]],\n",
            "\n",
            "         [[5.8268e-06, 6.5398e-06, 6.4287e-06],\n",
            "          [5.9338e-06, 6.2966e-06, 6.2366e-06],\n",
            "          [5.9348e-06, 6.0778e-06, 6.1268e-06]]],\n",
            "\n",
            "\n",
            "        [[[1.1763e-06, 1.7764e-06, 1.2860e-06],\n",
            "          [7.6001e-07, 1.5752e-06, 1.2790e-06],\n",
            "          [1.1180e-06, 2.1833e-06, 1.6981e-06]],\n",
            "\n",
            "         [[4.0349e-06, 4.2567e-06, 4.0698e-06],\n",
            "          [1.2467e-06, 1.3211e-06, 1.2904e-06],\n",
            "          [8.1860e-07, 8.4620e-07, 8.0041e-07]],\n",
            "\n",
            "         [[4.0782e-06, 5.8231e-06, 4.0260e-06],\n",
            "          [4.0583e-06, 5.3769e-06, 3.0790e-06],\n",
            "          [4.1572e-06, 4.6564e-06, 2.8068e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.6124e-04, 1.6485e-04, 1.6440e-04],\n",
            "          [1.6795e-04, 1.7192e-04, 1.6899e-04],\n",
            "          [1.6856e-04, 1.7058e-04, 1.6650e-04]],\n",
            "\n",
            "         [[2.8032e-06, 3.3178e-06, 1.3168e-06],\n",
            "          [1.5704e-06, 2.1853e-06, 5.2076e-07],\n",
            "          [1.7816e-06, 2.5684e-06, 7.6473e-07]],\n",
            "\n",
            "         [[1.0733e-05, 1.1433e-05, 1.1515e-05],\n",
            "          [1.0936e-05, 1.2150e-05, 1.2396e-05],\n",
            "          [1.1291e-05, 1.2693e-05, 1.2654e-05]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[5.1421e-06, 1.1047e-05, 1.9255e-05],\n",
            "          [7.5156e-06, 1.1262e-05, 1.5425e-05],\n",
            "          [7.0631e-06, 1.4995e-05, 1.7925e-05]],\n",
            "\n",
            "         [[4.1054e-06, 4.9498e-06, 4.1206e-06],\n",
            "          [3.5994e-06, 3.3598e-06, 2.7467e-06],\n",
            "          [2.0495e-06, 2.3873e-06, 2.4927e-06]],\n",
            "\n",
            "         [[5.8439e-06, 3.8166e-06, 1.4400e-06],\n",
            "          [7.7339e-06, 3.0310e-06, 1.1911e-06],\n",
            "          [7.2825e-06, 2.1789e-06, 1.3135e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[5.8455e-05, 5.2687e-05, 4.3942e-05],\n",
            "          [6.3790e-05, 5.5155e-05, 4.2339e-05],\n",
            "          [6.6948e-05, 5.5030e-05, 4.0906e-05]],\n",
            "\n",
            "         [[6.9197e-05, 7.1869e-05, 9.3801e-05],\n",
            "          [7.4140e-05, 7.9880e-05, 9.1033e-05],\n",
            "          [8.7207e-05, 9.4985e-05, 1.0480e-04]],\n",
            "\n",
            "         [[1.0240e-05, 1.4442e-05, 1.9470e-05],\n",
            "          [1.0555e-05, 1.6541e-05, 2.0611e-05],\n",
            "          [1.0928e-05, 1.7777e-05, 1.8117e-05]]],\n",
            "\n",
            "\n",
            "        [[[4.7209e-07, 2.6094e-07, 3.1059e-07],\n",
            "          [3.5455e-07, 1.6157e-07, 1.7976e-07],\n",
            "          [3.6248e-07, 2.0620e-07, 4.0222e-07]],\n",
            "\n",
            "         [[1.1034e-06, 1.2988e-06, 1.7132e-06],\n",
            "          [1.2982e-06, 1.5119e-06, 1.8013e-06],\n",
            "          [2.5248e-07, 2.2032e-07, 1.6289e-07]],\n",
            "\n",
            "         [[1.3513e-06, 5.4573e-06, 7.2280e-06],\n",
            "          [1.6950e-06, 7.5237e-06, 1.0145e-05],\n",
            "          [1.9642e-06, 7.3255e-06, 9.3949e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[2.1684e-05, 2.8909e-05, 4.1949e-05],\n",
            "          [2.6058e-05, 3.5191e-05, 5.3961e-05],\n",
            "          [2.8936e-05, 4.0513e-05, 6.1979e-05]],\n",
            "\n",
            "         [[1.7516e-06, 9.9606e-07, 5.1160e-07],\n",
            "          [1.1775e-06, 4.1215e-07, 4.1482e-08],\n",
            "          [1.0556e-06, 2.8770e-07, 3.9262e-08]],\n",
            "\n",
            "         [[2.0779e-06, 1.8608e-06, 2.0155e-06],\n",
            "          [2.6419e-06, 2.3195e-06, 2.5091e-06],\n",
            "          [2.4620e-06, 2.1932e-06, 2.4930e-06]]],\n",
            "\n",
            "\n",
            "        [[[1.4374e-06, 1.3822e-06, 9.9189e-07],\n",
            "          [9.5783e-07, 9.6794e-07, 7.5770e-07],\n",
            "          [7.5952e-07, 8.9836e-07, 7.6015e-07]],\n",
            "\n",
            "         [[6.0055e-07, 5.4021e-07, 4.6347e-07],\n",
            "          [1.4791e-06, 1.4472e-06, 1.2375e-06],\n",
            "          [5.5538e-07, 5.2945e-07, 4.8851e-07]],\n",
            "\n",
            "         [[8.0763e-08, 3.6256e-07, 2.9388e-07],\n",
            "          [1.2674e-07, 4.5034e-07, 2.1487e-07],\n",
            "          [1.9332e-07, 4.2445e-07, 1.4743e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[9.1578e-08, 1.0331e-07, 3.4859e-07],\n",
            "          [1.8441e-07, 1.9336e-07, 4.5645e-07],\n",
            "          [4.6759e-07, 4.8492e-07, 7.0060e-07]],\n",
            "\n",
            "         [[2.6129e-05, 2.5941e-05, 2.3308e-05],\n",
            "          [2.1182e-05, 2.1264e-05, 1.9137e-05],\n",
            "          [1.5790e-05, 1.5712e-05, 1.4426e-05]],\n",
            "\n",
            "         [[1.2552e-06, 1.1837e-06, 9.5939e-07],\n",
            "          [1.2510e-06, 1.1801e-06, 1.0793e-06],\n",
            "          [1.3037e-06, 1.3585e-06, 1.2955e-06]]]], device='cuda:0')}, 3: {'step': tensor(1920.), 'exp_avg': tensor([-5.1136e-05, -2.4200e-05, -3.1779e-06, -2.2542e-06, -3.9742e-07,\n",
            "        -1.0388e-04, -1.6043e-06,  8.9924e-06, -6.4465e-05, -1.3418e-05,\n",
            "        -3.5301e-05, -6.1819e-06,  3.5276e-05, -9.4045e-05,  5.6052e-45,\n",
            "        -4.6637e-05, -8.5094e-05, -1.0310e-05, -1.1297e-05, -1.2486e-05,\n",
            "        -3.0521e-06, -4.9658e-05,  1.6254e-05, -1.1409e-05,  7.7661e-05,\n",
            "         7.0988e-05,  4.2465e-05,  7.0732e-06, -1.0954e-04,  6.0816e-06,\n",
            "        -2.7444e-05,  1.2922e-06], device='cuda:0'), 'exp_avg_sq': tensor([2.4064e-03, 8.9549e-04, 1.2565e-03, 3.8580e-03, 1.3887e-03, 3.2604e-03,\n",
            "        1.5234e-03, 1.2455e-03, 2.1971e-03, 8.7378e-04, 1.4263e-03, 2.4618e-03,\n",
            "        1.7102e-03, 5.8110e-03, 1.8795e-12, 9.0029e-04, 1.4850e-03, 7.3742e-04,\n",
            "        7.2691e-04, 1.4205e-03, 7.7476e-04, 8.5718e-04, 3.6230e-03, 1.3762e-03,\n",
            "        3.8263e-03, 2.8874e-03, 6.1353e-03, 1.5042e-03, 1.4887e-02, 9.4641e-04,\n",
            "        7.6206e-04, 4.2047e-04], device='cuda:0')}, 4: {'step': tensor(1920.), 'exp_avg': tensor([[[[ 3.0059e-07,  4.7616e-07,  6.0556e-07],\n",
            "          [-6.7258e-08, -3.4168e-08,  1.7690e-07],\n",
            "          [ 9.4438e-08,  1.9426e-08,  8.9327e-08]],\n",
            "\n",
            "         [[ 1.0385e-07,  1.3760e-07,  1.8709e-07],\n",
            "          [ 5.6259e-08,  7.7400e-08,  5.5574e-08],\n",
            "          [ 3.3463e-08,  3.2491e-08,  4.8734e-09]],\n",
            "\n",
            "         [[-5.3414e-08, -5.9144e-07, -3.1166e-07],\n",
            "          [-2.2983e-07, -7.3011e-07, -3.9629e-07],\n",
            "          [-2.7291e-07, -6.7441e-07, -2.3776e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 1.3720e-08, -1.7877e-07,  1.4882e-07],\n",
            "          [-4.9543e-08, -3.7442e-07,  1.2156e-08],\n",
            "          [-1.2523e-07, -4.3908e-07, -4.2232e-08]],\n",
            "\n",
            "         [[ 1.2121e-08, -1.3748e-07,  2.9744e-07],\n",
            "          [-3.4135e-07, -3.6088e-07,  7.9111e-08],\n",
            "          [-3.2196e-07, -3.4221e-07,  4.6904e-08]],\n",
            "\n",
            "         [[ 4.0355e-08,  2.7258e-08,  8.4939e-08],\n",
            "          [-8.4836e-08, -1.1418e-08,  8.0068e-10],\n",
            "          [ 2.8853e-08,  9.1820e-08,  4.5756e-08]]],\n",
            "\n",
            "\n",
            "        [[[-1.8092e-06, -1.9930e-06, -8.7557e-07],\n",
            "          [-2.4374e-06, -2.7148e-06, -1.4424e-06],\n",
            "          [-9.2081e-07, -1.3339e-06, -1.6481e-06]],\n",
            "\n",
            "         [[-8.4976e-08, -1.4224e-07, -1.5988e-07],\n",
            "          [ 4.0233e-08, -8.3550e-08, -2.5349e-07],\n",
            "          [ 4.5435e-09, -1.4118e-07, -1.9185e-07]],\n",
            "\n",
            "         [[-1.3479e-06, -1.1273e-06, -6.1016e-07],\n",
            "          [-2.1986e-06, -1.6633e-06, -1.0740e-06],\n",
            "          [-2.5827e-06, -1.5100e-06, -5.8932e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-9.6270e-07, -8.6572e-07, -5.2132e-07],\n",
            "          [-1.8401e-06, -2.1220e-06, -9.9582e-07],\n",
            "          [-2.0691e-06, -1.9122e-06, -6.0085e-07]],\n",
            "\n",
            "         [[-9.7654e-07, -5.1967e-07, -3.5136e-07],\n",
            "          [-1.9177e-06, -9.4124e-07, -5.7106e-07],\n",
            "          [-1.7740e-06, -5.6429e-07, -3.3532e-07]],\n",
            "\n",
            "         [[-4.2836e-07, -6.6483e-07, -8.3868e-08],\n",
            "          [-1.0051e-06, -7.5719e-07, -1.0986e-07],\n",
            "          [-4.3866e-07, -1.7846e-07, -4.7641e-08]]],\n",
            "\n",
            "\n",
            "        [[[ 7.7976e-08,  1.1412e-07,  3.0421e-07],\n",
            "          [-1.2039e-07, -1.6760e-07, -1.0744e-07],\n",
            "          [ 1.0017e-08,  2.4038e-08, -3.1274e-08]],\n",
            "\n",
            "         [[ 5.0399e-08,  2.4806e-08,  8.0995e-09],\n",
            "          [-3.1644e-08, -7.8961e-08, -8.1404e-08],\n",
            "          [-6.8682e-08, -1.2425e-07, -1.2674e-07]],\n",
            "\n",
            "         [[-1.1692e-06, -6.2484e-07, -2.8058e-07],\n",
            "          [-1.0892e-06, -5.1788e-07, -1.2564e-07],\n",
            "          [-1.0479e-06, -5.5497e-07, -2.2234e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-6.7852e-07, -4.1353e-07,  3.7647e-08],\n",
            "          [-4.9274e-07, -4.0064e-07,  3.0730e-09],\n",
            "          [-3.3078e-07, -2.3322e-07, -4.8996e-08]],\n",
            "\n",
            "         [[-6.1457e-07,  1.6240e-07,  2.7088e-07],\n",
            "          [-6.7416e-07,  1.4019e-07,  1.6285e-07],\n",
            "          [-9.2445e-07, -1.7903e-07, -1.6439e-07]],\n",
            "\n",
            "         [[-8.7036e-08, -2.0941e-08,  1.9585e-08],\n",
            "          [ 3.4440e-09, -2.3430e-08, -5.5674e-08],\n",
            "          [ 1.2074e-08, -2.0153e-09, -1.6753e-08]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[-3.6056e-06, -3.3371e-06, -2.7013e-06],\n",
            "          [-3.6809e-06, -4.0269e-06, -3.4050e-06],\n",
            "          [-1.0735e-06, -1.6603e-06, -1.8903e-06]],\n",
            "\n",
            "         [[-4.9401e-07, -5.8745e-07, -6.4158e-07],\n",
            "          [-4.1510e-07, -5.6842e-07, -6.6189e-07],\n",
            "          [-2.7122e-07, -4.2964e-07, -5.6902e-07]],\n",
            "\n",
            "         [[-5.4347e-08,  2.7431e-08, -4.0623e-08],\n",
            "          [-9.3876e-08,  6.3839e-08,  4.7578e-08],\n",
            "          [-4.7424e-07, -3.6068e-07, -3.7075e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.2296e-06, -6.2533e-07, -4.4859e-07],\n",
            "          [-9.4293e-07, -8.1387e-07, -8.1697e-07],\n",
            "          [-6.3351e-07, -6.8098e-07, -6.2740e-07]],\n",
            "\n",
            "         [[-2.5964e-07, -1.5619e-07, -4.2652e-07],\n",
            "          [-6.5156e-07, -5.2130e-07, -7.2202e-07],\n",
            "          [-8.4307e-07, -7.2163e-07, -9.7266e-07]],\n",
            "\n",
            "         [[-8.7645e-07, -1.2087e-06, -1.3460e-06],\n",
            "          [-1.0165e-06, -1.4123e-06, -1.4688e-06],\n",
            "          [-4.3977e-07, -5.9864e-07, -8.2557e-07]]],\n",
            "\n",
            "\n",
            "        [[[-1.0285e-06, -1.4328e-06, -1.6928e-06],\n",
            "          [-3.0150e-07, -1.0041e-06, -1.5412e-06],\n",
            "          [ 2.2714e-07, -3.7972e-07, -1.3796e-08]],\n",
            "\n",
            "         [[ 8.3556e-08,  9.7392e-08,  1.0550e-07],\n",
            "          [ 1.3640e-07,  1.1363e-07,  1.0838e-07],\n",
            "          [ 1.0423e-07,  1.6866e-07,  1.4314e-07]],\n",
            "\n",
            "         [[-4.3796e-07, -1.5621e-06, -1.0896e-06],\n",
            "          [-9.0123e-07, -2.1369e-06, -1.4202e-06],\n",
            "          [-4.8204e-07, -1.6613e-06, -1.0877e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-4.2869e-07, -7.8049e-07, -1.7604e-06],\n",
            "          [-4.6595e-07, -9.5089e-07, -1.8376e-06],\n",
            "          [-5.6434e-08, -7.2680e-07, -1.4024e-06]],\n",
            "\n",
            "         [[-5.7920e-07, -1.3648e-06, -1.7290e-07],\n",
            "          [-1.3405e-06, -2.0354e-06, -5.1354e-07],\n",
            "          [-4.5816e-07, -1.2807e-06, -3.0637e-07]],\n",
            "\n",
            "         [[-3.6498e-07, -6.5751e-07, -6.9189e-07],\n",
            "          [-1.4228e-07, -2.2080e-07, -2.4081e-07],\n",
            "          [ 1.1686e-09, -1.7642e-07,  2.1551e-08]]],\n",
            "\n",
            "\n",
            "        [[[ 4.3383e-07, -2.0390e-07,  6.1283e-07],\n",
            "          [ 5.1478e-07,  6.0942e-08,  1.4488e-07],\n",
            "          [ 4.8648e-07,  2.7399e-07,  5.2170e-07]],\n",
            "\n",
            "         [[ 1.0258e-07,  6.0315e-08,  4.9209e-08],\n",
            "          [ 1.0096e-07,  6.9129e-08,  9.3155e-09],\n",
            "          [ 7.9968e-08,  8.5747e-08,  4.4098e-08]],\n",
            "\n",
            "         [[-3.8217e-07, -3.3596e-07,  2.5401e-07],\n",
            "          [-2.2527e-07, -2.0031e-07,  3.1686e-07],\n",
            "          [-3.0356e-08, -1.2407e-07,  1.4117e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-3.5128e-07, -3.9424e-07,  1.4358e-08],\n",
            "          [-3.1681e-07, -4.2122e-07, -2.9662e-08],\n",
            "          [ 1.9228e-07, -3.0733e-08, -3.4364e-08]],\n",
            "\n",
            "         [[-2.1283e-07,  1.7567e-07,  1.4757e-07],\n",
            "          [-1.2858e-07,  7.9798e-08,  2.8749e-07],\n",
            "          [-7.1309e-09, -1.0530e-07,  1.2048e-07]],\n",
            "\n",
            "         [[ 1.2962e-07,  1.4245e-07,  4.9233e-07],\n",
            "          [ 2.5310e-07,  8.7884e-08,  2.7362e-07],\n",
            "          [ 1.5017e-07,  1.8763e-08,  2.7561e-07]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[3.0986e-07, 5.0486e-07, 4.3215e-07],\n",
            "          [5.0103e-07, 6.3634e-07, 6.1051e-07],\n",
            "          [3.7969e-07, 4.5772e-07, 4.5321e-07]],\n",
            "\n",
            "         [[1.0178e-06, 1.0324e-06, 1.0589e-06],\n",
            "          [1.0802e-06, 1.1217e-06, 1.1301e-06],\n",
            "          [1.0396e-06, 1.0767e-06, 1.0520e-06]],\n",
            "\n",
            "         [[6.1690e-06, 6.2448e-06, 6.1362e-06],\n",
            "          [6.9972e-06, 7.2321e-06, 6.9911e-06],\n",
            "          [7.0142e-06, 7.1314e-06, 6.8859e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.0819e-07, 2.9283e-07, 2.7160e-07],\n",
            "          [2.8226e-07, 7.0786e-07, 5.7503e-07],\n",
            "          [2.6868e-07, 6.2549e-07, 4.4282e-07]],\n",
            "\n",
            "         [[1.6506e-06, 1.6729e-06, 1.5655e-06],\n",
            "          [1.9853e-06, 1.8794e-06, 1.6465e-06],\n",
            "          [1.8015e-06, 1.6718e-06, 1.5526e-06]],\n",
            "\n",
            "         [[1.3321e-07, 1.5187e-07, 1.0967e-07],\n",
            "          [2.0816e-07, 2.3125e-07, 1.7401e-07],\n",
            "          [1.8635e-07, 1.9862e-07, 1.4252e-07]]],\n",
            "\n",
            "\n",
            "        [[[1.0756e-06, 1.9083e-06, 1.2683e-06],\n",
            "          [3.1716e-06, 4.1632e-06, 3.3950e-06],\n",
            "          [1.7462e-06, 2.3284e-06, 2.2983e-06]],\n",
            "\n",
            "         [[5.2871e-07, 6.8506e-07, 9.4858e-07],\n",
            "          [4.0534e-07, 5.2489e-07, 9.0116e-07],\n",
            "          [3.0260e-07, 4.0541e-07, 7.3144e-07]],\n",
            "\n",
            "         [[2.1143e-06, 2.1738e-06, 1.6893e-06],\n",
            "          [4.9137e-06, 4.9207e-06, 4.0438e-06],\n",
            "          [3.8378e-06, 3.8233e-06, 3.0215e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[7.8938e-07, 1.7921e-06, 7.1009e-07],\n",
            "          [1.9573e-06, 3.6864e-06, 1.7413e-06],\n",
            "          [2.7657e-06, 3.9429e-06, 1.5092e-06]],\n",
            "\n",
            "         [[6.1035e-07, 2.0292e-07, 1.2288e-07],\n",
            "          [2.1489e-06, 9.6252e-07, 6.5814e-07],\n",
            "          [1.4078e-06, 3.2400e-07, 2.1061e-07]],\n",
            "\n",
            "         [[1.9444e-07, 2.4507e-07, 1.6389e-07],\n",
            "          [7.4828e-07, 8.1175e-07, 5.1219e-07],\n",
            "          [7.1043e-07, 5.3790e-07, 3.1591e-07]]],\n",
            "\n",
            "\n",
            "        [[[2.2980e-07, 2.6904e-07, 2.2405e-07],\n",
            "          [1.4393e-07, 1.5195e-07, 1.2099e-07],\n",
            "          [9.0121e-08, 1.3389e-07, 1.3966e-07]],\n",
            "\n",
            "         [[2.5247e-07, 2.5147e-07, 3.2711e-07],\n",
            "          [2.0703e-07, 2.1175e-07, 2.8700e-07],\n",
            "          [1.8960e-07, 1.9438e-07, 2.6713e-07]],\n",
            "\n",
            "         [[2.4119e-06, 2.5794e-06, 2.3677e-06],\n",
            "          [2.6932e-06, 2.8439e-06, 2.6014e-06],\n",
            "          [2.8887e-06, 3.0486e-06, 2.8140e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[2.8320e-07, 4.3752e-07, 3.6082e-07],\n",
            "          [2.8279e-07, 4.4190e-07, 3.2086e-07],\n",
            "          [2.9908e-07, 4.0579e-07, 3.3097e-07]],\n",
            "\n",
            "         [[4.9660e-07, 1.6464e-07, 9.6420e-08],\n",
            "          [4.6451e-07, 9.0318e-08, 6.7278e-08],\n",
            "          [3.9022e-07, 8.0797e-08, 6.5659e-08]],\n",
            "\n",
            "         [[7.8489e-08, 4.1697e-08, 3.3699e-08],\n",
            "          [5.5767e-08, 1.5282e-08, 1.0196e-08],\n",
            "          [4.7327e-08, 1.2179e-08, 1.0862e-08]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[2.3084e-06, 2.9247e-06, 2.1862e-06],\n",
            "          [1.8648e-06, 2.9698e-06, 2.3249e-06],\n",
            "          [8.2831e-07, 1.5234e-06, 1.4582e-06]],\n",
            "\n",
            "         [[2.0476e-06, 2.3108e-06, 2.2741e-06],\n",
            "          [1.6280e-06, 1.8519e-06, 1.8818e-06],\n",
            "          [1.3229e-06, 1.4606e-06, 1.5529e-06]],\n",
            "\n",
            "         [[5.9832e-08, 5.0528e-08, 6.5236e-08],\n",
            "          [1.6998e-07, 1.7043e-07, 2.0453e-07],\n",
            "          [1.6592e-07, 1.6756e-07, 1.7974e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[6.9077e-07, 3.6556e-07, 3.2328e-07],\n",
            "          [4.5761e-07, 3.3651e-07, 3.3824e-07],\n",
            "          [2.9983e-07, 2.9496e-07, 3.0526e-07]],\n",
            "\n",
            "         [[1.0172e-07, 1.3537e-07, 2.2414e-07],\n",
            "          [2.1198e-07, 2.6737e-07, 3.5675e-07],\n",
            "          [2.6026e-07, 3.0682e-07, 3.1430e-07]],\n",
            "\n",
            "         [[7.5996e-07, 9.8080e-07, 1.0724e-06],\n",
            "          [7.4225e-07, 9.8199e-07, 1.0167e-06],\n",
            "          [3.0622e-07, 4.0273e-07, 4.2823e-07]]],\n",
            "\n",
            "\n",
            "        [[[2.8246e-06, 2.5238e-06, 3.9448e-06],\n",
            "          [3.7247e-06, 3.2848e-06, 4.1341e-06],\n",
            "          [2.6409e-06, 1.5174e-06, 1.9994e-06]],\n",
            "\n",
            "         [[4.2484e-06, 2.7158e-06, 2.4847e-06],\n",
            "          [3.9989e-06, 2.2013e-06, 2.0364e-06],\n",
            "          [3.7706e-06, 1.9313e-06, 1.7130e-06]],\n",
            "\n",
            "         [[1.7942e-06, 3.0794e-06, 3.8144e-06],\n",
            "          [4.3002e-06, 6.4998e-06, 7.7424e-06],\n",
            "          [2.8627e-06, 4.5337e-06, 5.5250e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[3.8692e-07, 9.5095e-07, 5.7628e-06],\n",
            "          [6.7215e-07, 2.0049e-06, 7.2305e-06],\n",
            "          [5.0038e-07, 2.0641e-06, 6.1134e-06]],\n",
            "\n",
            "         [[1.6375e-06, 3.6833e-06, 1.4122e-06],\n",
            "          [3.4152e-06, 6.3498e-06, 2.5667e-06],\n",
            "          [2.2949e-06, 4.5443e-06, 1.6056e-06]],\n",
            "\n",
            "         [[1.1258e-06, 1.3492e-06, 1.1334e-06],\n",
            "          [1.7630e-06, 2.1747e-06, 1.4524e-06],\n",
            "          [1.5230e-06, 1.3681e-06, 6.2619e-07]]],\n",
            "\n",
            "\n",
            "        [[[1.8527e-06, 2.3828e-06, 2.4356e-06],\n",
            "          [1.5804e-06, 2.0176e-06, 2.0242e-06],\n",
            "          [1.2950e-06, 1.9714e-06, 2.1253e-06]],\n",
            "\n",
            "         [[1.3299e-06, 1.5096e-06, 1.8384e-06],\n",
            "          [1.4856e-06, 1.6976e-06, 2.0180e-06],\n",
            "          [1.5270e-06, 1.8189e-06, 2.1312e-06]],\n",
            "\n",
            "         [[1.3515e-06, 1.3901e-06, 1.1768e-06],\n",
            "          [9.7207e-07, 9.2083e-07, 7.6455e-07],\n",
            "          [5.6625e-07, 5.2454e-07, 4.5077e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.1708e-06, 2.3532e-06, 1.3373e-06],\n",
            "          [1.1982e-06, 2.2509e-06, 1.0022e-06],\n",
            "          [8.3523e-07, 1.4529e-06, 5.6391e-07]],\n",
            "\n",
            "         [[5.0479e-07, 3.3014e-07, 2.3018e-07],\n",
            "          [2.3365e-07, 1.1127e-07, 1.0889e-07],\n",
            "          [1.0840e-07, 5.8786e-08, 7.4479e-08]],\n",
            "\n",
            "         [[5.8606e-07, 7.0011e-07, 6.6834e-07],\n",
            "          [6.0011e-07, 5.5318e-07, 4.9396e-07],\n",
            "          [4.9188e-07, 5.1123e-07, 5.6457e-07]]]], device='cuda:0')}, 5: {'step': tensor(1920.), 'exp_avg': tensor([ 2.0983e-05, -3.2798e-05,  6.5683e-06, -6.7589e-05, -9.9508e-06,\n",
            "         7.8585e-06, -3.8804e-05, -1.9757e-05, -2.2632e-05, -7.5030e-06,\n",
            "        -2.1895e-05,  1.2668e-05,  3.4285e-05,  2.1819e-34, -1.5227e-05,\n",
            "        -3.9344e-06, -2.7476e-05,  2.1728e-05, -7.8376e-06, -8.9848e-06,\n",
            "         2.7580e-05,  2.2019e-05,  2.2518e-05,  7.5749e-06, -5.9262e-06,\n",
            "        -1.2595e-08,  2.0754e-06, -3.4399e-06, -1.5217e-05,  1.5407e-05,\n",
            "        -2.9236e-06,  1.1009e-05,  1.7371e-05, -1.0557e-05,  2.8318e-05,\n",
            "         3.5323e-06,  1.5618e-05, -3.6110e-07, -8.6270e-06,  7.9632e-06,\n",
            "         1.6127e-05,  5.6052e-45,  7.1318e-06, -1.9162e-06,  8.6521e-06,\n",
            "         3.6459e-06, -1.2222e-05, -2.9192e-05, -7.9990e-06, -7.6261e-06,\n",
            "        -5.5924e-06,  5.6830e-06,  5.6052e-45,  1.4698e-05, -1.5483e-05,\n",
            "        -7.1776e-06, -7.2419e-06,  3.2914e-05,  2.4507e-06, -5.2265e-05,\n",
            "        -3.5098e-09, -3.9663e-05, -1.5736e-06,  6.7930e-06], device='cuda:0'), 'exp_avg_sq': tensor([1.1794e-03, 4.8262e-04, 6.6513e-04, 3.8137e-03, 2.9459e-05, 9.5889e-04,\n",
            "        1.1062e-03, 1.1778e-03, 1.8178e-04, 8.6370e-04, 5.5693e-04, 5.3871e-04,\n",
            "        1.0662e-03, 1.3509e-08, 5.4360e-04, 1.1467e-03, 3.7691e-04, 6.0956e-04,\n",
            "        2.1478e-03, 3.0017e-04, 5.9228e-04, 4.2019e-04, 1.1980e-03, 7.2360e-04,\n",
            "        9.3373e-04, 4.2667e-05, 1.9273e-04, 1.5298e-04, 3.0365e-04, 6.2724e-04,\n",
            "        1.3470e-03, 3.8241e-05, 3.7018e-04, 1.6139e-04, 5.4610e-04, 6.4091e-05,\n",
            "        4.1172e-04, 3.5114e-03, 2.5144e-04, 1.4497e-03, 4.0199e-04, 3.9180e-14,\n",
            "        1.8480e-03, 4.9104e-04, 4.9557e-04, 1.0075e-03, 5.5726e-04, 4.0668e-04,\n",
            "        2.1804e-04, 1.4240e-03, 4.2266e-04, 6.7197e-04, 4.0351e-10, 7.7840e-04,\n",
            "        1.3055e-03, 6.7740e-04, 7.7970e-05, 2.1432e-03, 2.6664e-04, 1.7025e-03,\n",
            "        1.4073e-06, 5.4636e-04, 1.5152e-03, 2.2769e-04], device='cuda:0')}, 6: {'step': tensor(1920.), 'exp_avg': tensor([[[[-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45]],\n",
            "\n",
            "         [[ 5.6052e-45,  5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45]],\n",
            "\n",
            "         [[-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [ 5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [ 5.6052e-45, -5.6052e-45, -5.6052e-45]],\n",
            "\n",
            "         [[ 5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45]],\n",
            "\n",
            "         [[ 5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [ 5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45]]],\n",
            "\n",
            "\n",
            "        [[[ 9.2136e-08,  4.9005e-09, -1.4286e-08],\n",
            "          [ 7.3352e-08,  1.1220e-10, -2.3532e-08],\n",
            "          [ 3.2180e-08, -4.5505e-08, -1.2694e-07]],\n",
            "\n",
            "         [[ 1.0536e-07, -3.2324e-08, -2.8300e-08],\n",
            "          [-5.2195e-08, -6.5654e-08, -2.1735e-07],\n",
            "          [-5.2578e-08, -1.9249e-07, -3.0899e-07]],\n",
            "\n",
            "         [[ 2.7587e-07,  5.3450e-08,  6.4270e-09],\n",
            "          [ 2.2729e-07,  6.5263e-08,  3.3260e-09],\n",
            "          [ 1.4962e-07, -1.2119e-08, -1.1699e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 2.5067e-07,  1.4892e-07,  1.6348e-07],\n",
            "          [ 1.3544e-07,  1.0947e-07, -2.4641e-07],\n",
            "          [ 6.7831e-08,  1.8886e-07, -1.2827e-08]],\n",
            "\n",
            "         [[-9.2936e-08, -4.9206e-09, -1.1609e-08],\n",
            "          [-8.4117e-08, -9.5346e-08, -1.6818e-07],\n",
            "          [-1.5198e-07, -1.6973e-07, -2.3024e-07]],\n",
            "\n",
            "         [[-8.1424e-08, -1.0984e-07, -1.2274e-08],\n",
            "          [-1.3845e-07,  1.4390e-08, -9.4605e-08],\n",
            "          [-1.9605e-07, -1.1670e-07, -6.6048e-08]]],\n",
            "\n",
            "\n",
            "        [[[-1.1333e-07, -1.1823e-07, -2.0388e-09],\n",
            "          [-1.1187e-07, -1.6921e-07, -1.0234e-08],\n",
            "          [-9.2609e-08, -5.2410e-08,  8.0651e-09]],\n",
            "\n",
            "         [[-1.9446e-07, -2.8977e-07, -7.4939e-08],\n",
            "          [-7.5458e-08, -3.6791e-07, -3.5900e-08],\n",
            "          [-4.0229e-08, -1.1936e-07, -1.0399e-08]],\n",
            "\n",
            "         [[-2.6812e-07, -3.6485e-07, -5.7667e-08],\n",
            "          [-2.4232e-07, -3.3320e-07, -2.1198e-08],\n",
            "          [-2.1309e-07, -2.7231e-07,  2.1888e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-5.7873e-09, -2.2913e-07, -3.2275e-07],\n",
            "          [ 4.3397e-09, -2.4209e-07, -5.9901e-07],\n",
            "          [ 2.2783e-08, -1.2853e-07, -4.7748e-07]],\n",
            "\n",
            "         [[-5.3392e-07, -1.4711e-07,  3.1707e-10],\n",
            "          [-5.7432e-07, -1.0823e-07, -3.0410e-08],\n",
            "          [-4.5670e-07, -7.5690e-08, -1.0827e-07]],\n",
            "\n",
            "         [[-1.6508e-07, -7.8601e-07, -5.8405e-07],\n",
            "          [-1.6706e-07, -8.3801e-07, -5.1533e-07],\n",
            "          [-2.0678e-07, -6.5474e-07, -4.6938e-07]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[-4.7889e-10, -1.3688e-09,  2.0237e-10],\n",
            "          [ 1.9597e-10, -2.7334e-09, -1.8780e-09],\n",
            "          [ 1.1409e-08,  2.1130e-08,  2.0304e-08]],\n",
            "\n",
            "         [[-4.8125e-12, -1.5526e-09,  1.2919e-09],\n",
            "          [-6.3917e-10, -9.4602e-09, -1.1182e-08],\n",
            "          [ 2.2400e-08,  4.8062e-08,  6.0262e-08]],\n",
            "\n",
            "         [[ 9.7002e-11, -3.7821e-09,  1.1248e-09],\n",
            "          [ 4.8117e-11, -1.7468e-09, -4.9483e-09],\n",
            "          [ 4.3301e-10,  7.0587e-09,  1.5749e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.4954e-09, -3.2439e-09, -5.6342e-11],\n",
            "          [ 9.8633e-09,  1.6433e-08,  1.1083e-08],\n",
            "          [ 1.2556e-08,  1.9735e-08,  1.7079e-08]],\n",
            "\n",
            "         [[-7.7268e-10,  1.0097e-10,  7.5618e-10],\n",
            "          [-2.7000e-09, -9.3635e-09, -9.1553e-09],\n",
            "          [ 2.9995e-08,  3.0147e-08,  1.9098e-08]],\n",
            "\n",
            "         [[ 4.4682e-09, -8.3979e-09,  5.1345e-10],\n",
            "          [ 5.0934e-09, -1.6989e-08, -1.7709e-08],\n",
            "          [ 7.4914e-09,  2.3626e-12,  8.2032e-09]]],\n",
            "\n",
            "\n",
            "        [[[-5.4461e-07, -6.6592e-07, -4.8451e-07],\n",
            "          [-4.7478e-07, -4.7850e-07, -4.5005e-07],\n",
            "          [-1.3387e-07, -4.8090e-08, -6.6253e-08]],\n",
            "\n",
            "         [[-3.4860e-07, -7.7531e-07, -4.8479e-07],\n",
            "          [-3.9957e-07, -5.5046e-07, -4.7437e-07],\n",
            "          [-1.0324e-07, -1.7860e-07, -1.3482e-07]],\n",
            "\n",
            "         [[-3.5626e-07, -7.6731e-07, -4.8835e-07],\n",
            "          [-2.4641e-07, -4.9117e-07, -3.7531e-07],\n",
            "          [-1.0796e-07, -1.6871e-07, -1.3348e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-4.3979e-07, -4.6934e-07, -2.0787e-07],\n",
            "          [-6.5225e-08, -7.3386e-08, -3.5065e-08],\n",
            "          [-4.1476e-08, -4.6756e-08,  3.2974e-08]],\n",
            "\n",
            "         [[-6.5020e-07, -5.2303e-07, -1.6003e-07],\n",
            "          [-1.9725e-07, -1.9664e-07, -1.0443e-07],\n",
            "          [-2.7314e-08, -6.5722e-08, -7.2950e-08]],\n",
            "\n",
            "         [[-2.6032e-07, -4.8246e-07, -1.1110e-07],\n",
            "          [-1.6267e-07, -1.9504e-07, -9.8405e-08],\n",
            "          [-9.3157e-09,  3.3535e-09,  7.2539e-09]]],\n",
            "\n",
            "\n",
            "        [[[-1.3894e-09, -6.1859e-10,  3.9112e-10],\n",
            "          [ 3.2214e-09, -1.8399e-08,  8.5492e-12],\n",
            "          [ 3.9735e-09,  3.4735e-09,  1.5439e-09]],\n",
            "\n",
            "         [[-1.5486e-08, -2.5509e-09,  5.6423e-10],\n",
            "          [ 9.8815e-09,  2.3548e-09, -1.8390e-09],\n",
            "          [-2.9704e-09, -1.0806e-09, -7.2918e-10]],\n",
            "\n",
            "         [[-1.9437e-09, -4.3093e-10, -1.6421e-09],\n",
            "          [-3.2569e-10, -4.2787e-15, -1.1704e-09],\n",
            "          [ 2.4795e-10,  4.4694e-10, -3.6261e-10]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 1.5145e-07,  6.1059e-08,  8.8054e-08],\n",
            "          [ 5.6936e-09, -6.1526e-08, -2.3307e-08],\n",
            "          [ 2.7961e-07,  1.9581e-07,  2.3849e-07]],\n",
            "\n",
            "         [[ 2.9143e-09, -3.6420e-08, -1.3159e-08],\n",
            "          [-2.4936e-08, -8.8255e-08, -8.2967e-09],\n",
            "          [ 3.4761e-08, -4.7745e-08,  3.4459e-08]],\n",
            "\n",
            "         [[-1.3636e-07, -6.4964e-08, -2.4185e-08],\n",
            "          [-1.9346e-07, -9.5570e-08, -3.4726e-08],\n",
            "          [-5.5813e-08, -4.8931e-08, -1.8465e-08]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[5.9860e-16, 1.8735e-15, 1.6150e-17],\n",
            "          [1.3010e-15, 9.3373e-16, 3.1585e-17],\n",
            "          [1.5083e-16, 1.4147e-16, 5.2339e-17]],\n",
            "\n",
            "         [[4.0832e-16, 5.9786e-19, 7.9552e-33],\n",
            "          [4.1533e-16, 3.7826e-18, 2.5241e-21],\n",
            "          [3.9514e-16, 4.8049e-20, 4.1994e-18]],\n",
            "\n",
            "         [[1.7602e-15, 6.7050e-16, 4.0796e-17],\n",
            "          [3.3528e-15, 3.2493e-15, 3.9524e-17],\n",
            "          [1.8685e-15, 3.9281e-16, 5.0720e-17]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.9122e-15, 1.5950e-15, 5.1166e-19],\n",
            "          [3.7152e-15, 7.3042e-15, 1.3601e-18],\n",
            "          [5.7760e-16, 6.6202e-16, 1.6486e-28]],\n",
            "\n",
            "         [[8.5795e-17, 7.3037e-17, 2.4246e-17],\n",
            "          [4.8801e-17, 3.4017e-16, 3.7477e-17],\n",
            "          [4.8280e-17, 2.2959e-17, 8.0384e-17]],\n",
            "\n",
            "         [[1.1233e-15, 1.0492e-18, 1.4275e-20],\n",
            "          [1.2010e-15, 1.0664e-17, 1.1416e-31],\n",
            "          [1.3019e-15, 3.1131e-19, 2.9437e-18]]],\n",
            "\n",
            "\n",
            "        [[[1.5086e-07, 1.3603e-07, 1.3997e-07],\n",
            "          [1.6365e-07, 1.3894e-07, 1.3928e-07],\n",
            "          [1.6670e-07, 1.4083e-07, 1.4270e-07]],\n",
            "\n",
            "         [[4.8344e-08, 2.3558e-08, 1.5725e-08],\n",
            "          [1.4736e-07, 7.3072e-08, 7.6969e-08],\n",
            "          [1.0693e-07, 4.8924e-08, 5.5885e-08]],\n",
            "\n",
            "         [[1.5463e-07, 1.2029e-07, 1.1936e-07],\n",
            "          [1.9499e-07, 1.3911e-07, 1.3442e-07],\n",
            "          [2.2869e-07, 1.6096e-07, 1.5943e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[3.2141e-07, 3.6153e-07, 2.7463e-07],\n",
            "          [4.9226e-07, 7.3107e-07, 7.4887e-07],\n",
            "          [3.1933e-07, 4.8966e-07, 6.8186e-07]],\n",
            "\n",
            "         [[3.6985e-08, 2.9307e-08, 8.0743e-08],\n",
            "          [9.0078e-08, 8.7640e-08, 1.3811e-07],\n",
            "          [7.8107e-08, 6.4462e-08, 1.0407e-07]],\n",
            "\n",
            "         [[2.1319e-07, 3.2151e-07, 2.4022e-07],\n",
            "          [2.2050e-07, 3.1268e-07, 2.2309e-07],\n",
            "          [2.3403e-07, 3.2926e-07, 2.0748e-07]]],\n",
            "\n",
            "\n",
            "        [[[2.8045e-07, 3.6239e-07, 1.0406e-07],\n",
            "          [2.4963e-07, 3.5358e-07, 8.9025e-08],\n",
            "          [1.8530e-07, 2.5305e-07, 7.2708e-08]],\n",
            "\n",
            "         [[1.7925e-07, 6.7849e-07, 1.1560e-07],\n",
            "          [1.2056e-07, 7.8397e-07, 8.0455e-08],\n",
            "          [8.1151e-08, 6.6577e-07, 1.1919e-07]],\n",
            "\n",
            "         [[2.7272e-07, 6.5601e-07, 1.3852e-07],\n",
            "          [2.7775e-07, 6.8312e-07, 1.1529e-07],\n",
            "          [2.7112e-07, 5.5859e-07, 1.1311e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[6.3198e-08, 1.2378e-07, 8.9696e-08],\n",
            "          [4.2143e-08, 1.5449e-07, 1.4076e-07],\n",
            "          [3.6162e-08, 1.6000e-07, 1.9476e-07]],\n",
            "\n",
            "         [[6.5243e-07, 6.0663e-07, 4.0083e-08],\n",
            "          [6.5165e-07, 5.9538e-07, 2.9479e-08],\n",
            "          [5.5846e-07, 5.4456e-07, 2.9781e-08]],\n",
            "\n",
            "         [[1.2797e-07, 1.6139e-06, 7.4917e-07],\n",
            "          [1.6347e-07, 1.6742e-06, 6.9567e-07],\n",
            "          [2.1881e-07, 1.6765e-06, 7.1421e-07]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[1.3630e-08, 1.8300e-08, 1.9622e-08],\n",
            "          [5.8047e-08, 9.5604e-08, 9.7507e-08],\n",
            "          [6.1707e-08, 8.5235e-08, 7.4948e-08]],\n",
            "\n",
            "         [[4.1728e-09, 1.1620e-08, 2.1630e-08],\n",
            "          [1.1695e-07, 1.9523e-07, 2.8359e-07],\n",
            "          [7.7561e-08, 1.2108e-07, 1.5460e-07]],\n",
            "\n",
            "         [[1.0317e-08, 1.5204e-08, 1.6579e-08],\n",
            "          [1.8110e-08, 2.9849e-08, 5.2996e-08],\n",
            "          [3.0371e-08, 4.9643e-08, 7.8487e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[2.5048e-08, 2.1136e-08, 2.2617e-08],\n",
            "          [9.0909e-08, 1.0700e-07, 1.0934e-07],\n",
            "          [8.9629e-08, 1.2692e-07, 1.5823e-07]],\n",
            "\n",
            "         [[6.1682e-09, 1.6523e-08, 1.0720e-08],\n",
            "          [3.7392e-08, 1.2995e-07, 5.3401e-08],\n",
            "          [5.0416e-08, 1.3520e-07, 4.2383e-08]],\n",
            "\n",
            "         [[9.6730e-08, 1.4848e-07, 1.8791e-07],\n",
            "          [2.8987e-07, 4.6802e-07, 6.2227e-07],\n",
            "          [3.4712e-07, 6.0836e-07, 8.1144e-07]]],\n",
            "\n",
            "\n",
            "        [[[7.0991e-07, 9.1545e-07, 7.9270e-07],\n",
            "          [7.1611e-07, 8.5759e-07, 8.0351e-07],\n",
            "          [9.2220e-07, 1.0653e-06, 1.0727e-06]],\n",
            "\n",
            "         [[7.0944e-08, 2.4252e-07, 1.3896e-07],\n",
            "          [3.6869e-08, 8.9221e-08, 4.9681e-08],\n",
            "          [2.3615e-08, 3.5906e-08, 3.0586e-08]],\n",
            "\n",
            "         [[7.1816e-07, 1.0884e-06, 9.8572e-07],\n",
            "          [7.4225e-07, 9.9708e-07, 9.3805e-07],\n",
            "          [8.0721e-07, 9.7382e-07, 1.0269e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[7.2880e-08, 7.7855e-08, 7.5584e-08],\n",
            "          [1.9834e-08, 1.6277e-08, 2.4748e-08],\n",
            "          [1.1066e-08, 1.0474e-08, 1.3252e-08]],\n",
            "\n",
            "         [[3.4273e-07, 3.7462e-07, 9.3878e-08],\n",
            "          [1.1106e-07, 1.3529e-07, 3.1757e-08],\n",
            "          [5.7180e-08, 7.5242e-08, 3.9954e-08]],\n",
            "\n",
            "         [[3.0110e-07, 5.7246e-07, 3.1811e-07],\n",
            "          [9.3908e-08, 2.0123e-07, 1.1076e-07],\n",
            "          [8.4572e-09, 1.7304e-08, 9.8436e-09]]],\n",
            "\n",
            "\n",
            "        [[[2.5278e-07, 2.1247e-07, 1.9869e-07],\n",
            "          [2.3492e-07, 1.9880e-07, 1.8669e-07],\n",
            "          [1.9118e-07, 1.6378e-07, 1.5848e-07]],\n",
            "\n",
            "         [[8.2057e-08, 3.3389e-08, 2.0788e-08],\n",
            "          [7.7195e-08, 2.9837e-08, 1.8751e-08],\n",
            "          [5.5661e-08, 2.0544e-08, 1.8268e-08]],\n",
            "\n",
            "         [[4.2082e-07, 3.8312e-07, 3.5275e-07],\n",
            "          [4.2635e-07, 3.8617e-07, 3.5470e-07],\n",
            "          [2.8763e-07, 2.5042e-07, 2.2980e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.7494e-07, 1.7038e-07, 1.3201e-07],\n",
            "          [2.2482e-07, 2.5907e-07, 2.0146e-07],\n",
            "          [2.8533e-07, 3.3244e-07, 3.0709e-07]],\n",
            "\n",
            "         [[6.0903e-08, 3.5425e-08, 6.5333e-08],\n",
            "          [4.9032e-08, 3.4633e-08, 6.9175e-08],\n",
            "          [6.2794e-08, 5.4130e-08, 8.9772e-08]],\n",
            "\n",
            "         [[7.7502e-07, 5.8260e-07, 2.9527e-07],\n",
            "          [9.1601e-07, 7.1091e-07, 4.0783e-07],\n",
            "          [7.2122e-07, 5.3753e-07, 2.8227e-07]]]], device='cuda:0')}, 7: {'step': tensor(1920.), 'exp_avg': tensor([-5.6052e-45,  8.8870e-06, -2.1876e-06, -1.4117e-05, -5.4881e-06,\n",
            "         2.6558e-06,  5.6052e-45, -3.3807e-05,  1.3301e-08, -6.6602e-07,\n",
            "         5.6052e-45, -4.8954e-13, -1.2110e-05, -2.2201e-05, -2.4244e-05,\n",
            "         7.2223e-06, -4.6394e-07, -1.1403e-05,  5.6052e-45, -1.3877e-05,\n",
            "        -3.8452e-06,  5.6052e-45, -4.0417e-06, -1.2874e-15,  1.1011e-07,\n",
            "        -3.4786e-06, -8.3651e-06, -1.1794e-05, -3.5751e-06, -2.4884e-06,\n",
            "         5.0838e-06,  3.3271e-06, -5.8541e-06, -1.2587e-10, -1.3990e-06,\n",
            "        -7.6781e-06, -2.7685e-05, -1.8685e-09,  3.1101e-08,  2.3770e-07,\n",
            "         2.0977e-31,  1.2237e-05,  6.2244e-06, -5.4248e-06, -2.2026e-06,\n",
            "        -1.0984e-05, -1.7434e-05, -3.2445e-06,  7.4926e-06, -2.2724e-08,\n",
            "        -6.9169e-06, -9.0819e-06,  1.2041e-06, -7.3995e-06, -9.5825e-07,\n",
            "        -1.1623e-05, -2.2787e-05, -1.1211e-05, -3.4276e-20, -4.7408e-26,\n",
            "         6.1888e-07,  2.0888e-06, -4.4916e-06, -7.4829e-06, -1.0699e-05,\n",
            "         5.7207e-06, -8.0564e-06, -2.5436e-05, -8.9859e-08,  1.4452e-05,\n",
            "         1.3983e-09, -1.0837e-06, -3.9824e-06,  4.4645e-06, -1.3065e-11,\n",
            "         5.6052e-45, -3.3434e-06, -3.7534e-06,  1.1045e-05,  6.1194e-06,\n",
            "        -7.2387e-06, -4.0982e-06,  4.2513e-07, -4.2582e-06, -1.0815e-05,\n",
            "        -5.2303e-06,  3.7236e-07, -5.2501e-10, -1.5556e-05, -2.1080e-05,\n",
            "         3.0508e-07, -1.4794e-05, -1.3174e-05, -1.7390e-05,  5.6052e-45,\n",
            "         7.8723e-06, -1.5940e-06, -1.3368e-06, -1.5934e-05, -4.4487e-06,\n",
            "        -1.1401e-06,  1.1367e-08,  5.6052e-45, -1.0414e-05,  1.0618e-05,\n",
            "         5.6052e-45,  3.2715e-06, -1.2412e-05, -5.0524e-06, -1.3184e-05,\n",
            "        -5.7470e-06,  8.8274e-06, -3.1882e-06, -7.9111e-06, -1.5292e-05,\n",
            "         1.1359e-33, -1.7243e-12, -5.6052e-45, -5.2980e-06, -2.5388e-07,\n",
            "         5.8017e-06, -5.8510e-07, -7.6242e-38,  3.7967e-31,  2.2307e-05,\n",
            "         6.6128e-08, -6.5315e-06,  3.7510e-06], device='cuda:0'), 'exp_avg_sq': tensor([5.1631e-13, 2.1746e-04, 5.6281e-05, 2.1517e-03, 2.9253e-04, 3.1785e-04,\n",
            "        2.5445e-08, 5.9172e-04, 6.4631e-06, 1.5734e-04, 4.3226e-13, 1.4513e-05,\n",
            "        5.9479e-04, 1.0651e-03, 9.9570e-04, 8.1083e-04, 1.9891e-05, 7.2594e-04,\n",
            "        5.8221e-07, 3.7270e-04, 1.0081e-04, 8.7206e-07, 8.4821e-05, 9.7949e-05,\n",
            "        1.7400e-04, 1.6241e-04, 1.1922e-04, 6.3980e-04, 1.7515e-04, 4.5291e-05,\n",
            "        3.4263e-04, 2.6211e-04, 3.2651e-04, 1.5376e-05, 5.0572e-05, 2.5399e-04,\n",
            "        5.0241e-04, 4.5698e-05, 2.5244e-06, 1.3930e-04, 1.5724e-07, 5.1849e-04,\n",
            "        1.4752e-04, 6.2816e-05, 2.8721e-05, 6.7027e-05, 7.0230e-05, 2.8002e-04,\n",
            "        3.9358e-04, 7.8467e-06, 1.2077e-04, 2.9407e-04, 1.0311e-04, 5.7000e-04,\n",
            "        2.6183e-04, 1.2045e-04, 1.2458e-03, 6.9745e-04, 2.8346e-07, 3.1369e-07,\n",
            "        1.6065e-04, 4.3904e-04, 2.3570e-04, 1.5289e-04, 4.7216e-04, 1.7600e-03,\n",
            "        6.5281e-04, 1.8267e-03, 6.2636e-05, 2.8696e-04, 1.4921e-06, 1.8300e-04,\n",
            "        1.8129e-04, 1.2594e-04, 1.4493e-08, 4.1941e-05, 1.8718e-04, 4.8401e-04,\n",
            "        2.6417e-04, 9.4321e-05, 1.0782e-03, 2.2164e-04, 5.1378e-04, 8.2616e-05,\n",
            "        3.8294e-04, 9.1665e-05, 1.0390e-04, 2.7038e-07, 2.0514e-03, 1.0549e-03,\n",
            "        3.8168e-05, 2.2824e-04, 3.1583e-04, 5.6601e-04, 1.1953e-12, 1.6108e-04,\n",
            "        6.3969e-04, 8.6334e-06, 6.8710e-03, 2.4462e-04, 5.5193e-05, 1.3665e-06,\n",
            "        8.6563e-11, 5.2075e-04, 2.8554e-04, 2.0063e-16, 5.8663e-05, 9.2561e-04,\n",
            "        2.3647e-04, 4.6786e-04, 2.5829e-04, 1.5419e-04, 8.7421e-05, 3.5077e-04,\n",
            "        1.6264e-04, 4.0930e-05, 1.6755e-05, 5.2979e-11, 1.6292e-04, 1.7760e-04,\n",
            "        1.2864e-04, 1.1824e-04, 1.3551e-06, 6.2619e-06, 8.1790e-04, 1.5693e-05,\n",
            "        1.6016e-04, 1.9500e-04], device='cuda:0')}, 8: {'step': tensor(1920.), 'exp_avg': tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 3.2151e-10, 4.7531e-09,\n",
            "         3.3656e-09],\n",
            "        ...,\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45]], device='cuda:0'), 'exp_avg_sq': tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 1.6201e-13, 2.4327e-13,\n",
            "         3.8381e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.5525e-13, 3.6317e-13,\n",
            "         3.4274e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.3927e-09, 4.5701e-09,\n",
            "         5.3099e-09],\n",
            "        ...,\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 4.7059e-13, 4.0209e-13,\n",
            "         3.0205e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 9.0464e-14, 1.7059e-13,\n",
            "         3.5410e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 3.6768e-11, 1.0212e-10,\n",
            "         1.3188e-10]], device='cuda:0')}, 9: {'step': tensor(1920.), 'exp_avg': tensor([ 5.6052e-45,  5.6052e-45, -2.8088e-07,  5.6052e-45,  3.5345e-07,\n",
            "         5.6052e-45, -8.8008e-17, -5.2413e-09, -2.9867e-08, -5.5406e-08,\n",
            "         5.6052e-45,  5.6052e-45, -1.3142e-07,  5.6052e-45, -1.8576e-08,\n",
            "         5.6052e-45, -3.7381e-08, -2.5968e-07, -1.0773e-08,  5.6052e-45,\n",
            "         5.6052e-45,  4.1422e-08,  5.6052e-45, -1.2963e-07,  4.4890e-07,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -7.5288e-08,  4.0809e-08,\n",
            "         5.6052e-45,  5.6052e-45, -3.7727e-08,  5.6052e-45,  5.6052e-45,\n",
            "         7.7274e-08,  5.6052e-45, -1.3067e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -2.1280e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -7.7154e-08, -4.1719e-07,  5.6052e-45,  5.6052e-45,\n",
            "         2.6260e-07,  5.6052e-45, -1.4027e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -9.4945e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.0635e-07, -1.9612e-07,  1.1029e-07, -8.6121e-09,  5.6052e-45,\n",
            "         4.3321e-36,  5.6052e-45,  5.6052e-45,  5.6052e-45, -6.8288e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -8.9211e-08,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  3.1726e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -7.2555e-08, -1.8095e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -2.0694e-07,  5.6052e-45,  5.6052e-45,  2.9903e-40,\n",
            "        -1.1179e-16,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  1.3550e-07,  1.9022e-29, -2.1126e-08, -5.0369e-08,\n",
            "         5.6052e-45, -3.1718e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         6.8802e-38,  4.7559e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -8.0513e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  7.7301e-08,  1.8170e-08,\n",
            "         1.0970e-08,  5.6052e-45, -3.6103e-07, -4.8644e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -3.1252e-09,  2.7887e-09,  9.4459e-08,\n",
            "         5.6052e-45,  5.6052e-45, -1.2406e-07,  5.6052e-45,  5.6052e-45,\n",
            "        -2.3496e-07,  5.6052e-45, -3.7453e-07, -1.4366e-07,  5.6052e-45,\n",
            "         5.6052e-45, -2.5462e-07,  5.6052e-45,  5.6052e-45, -1.5947e-07,\n",
            "        -2.0019e-07,  5.6052e-45, -5.3826e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -4.8504e-08, -1.5304e-34,  4.1630e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  1.1712e-07,\n",
            "         5.6052e-45,  5.6052e-45, -2.5880e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -6.8192e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -5.4777e-08,  1.9605e-36,  8.2112e-08,  5.6052e-45, -1.0364e-08,\n",
            "         5.6052e-45, -3.0110e-07,  5.6052e-45,  1.5900e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -2.3397e-07,\n",
            "         5.6052e-45, -7.1517e-09, -1.1536e-07,  5.6052e-45,  5.6052e-45,\n",
            "        -2.9520e-08,  5.6052e-45,  5.6052e-45, -2.2537e-07, -1.2070e-07,\n",
            "         5.6052e-45,  5.6052e-45, -9.6025e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  2.8289e-12,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -8.8401e-08, -8.7024e-08, -2.5848e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -1.9390e-08, -1.0072e-07, -1.8746e-09,\n",
            "        -5.5343e-08,  5.6052e-45, -2.3316e-07,  5.6052e-45, -2.8512e-07,\n",
            "         5.6052e-45,  5.6052e-45, -1.1579e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.3652e-08,  5.6052e-45, -6.5299e-08, -7.0888e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -6.7796e-07,  5.6052e-45,\n",
            "         5.6052e-45, -9.4855e-09, -1.3231e-08, -1.1078e-07, -4.4066e-16,\n",
            "         5.6052e-45,  7.4767e-08, -9.5429e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  8.8176e-08, -1.6111e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.3712e-07,  5.6052e-45,  5.6052e-45,  3.6645e-08,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -3.0244e-08,  5.6052e-45,  5.6052e-45, -4.0009e-08,\n",
            "         5.6052e-45, -2.4618e-09, -2.8133e-08, -3.4168e-09,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -1.5991e-08,  5.6052e-45,\n",
            "        -7.5107e-08,  5.6052e-45,  5.6052e-45, -4.8448e-08,  1.2800e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -9.8662e-08,\n",
            "        -3.0874e-08, -1.8804e-07,  5.6052e-45,  5.6052e-45, -2.0318e-07,\n",
            "         5.6052e-45, -7.5433e-07, -1.9811e-15,  5.6052e-45, -4.9909e-07,\n",
            "        -3.7516e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         7.0205e-09,  5.6052e-45,  4.4506e-12,  5.6052e-45, -4.3186e-07,\n",
            "        -1.0606e-07, -4.1156e-07, -1.0645e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -5.4061e-08, -8.8578e-08, -1.1805e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.3649e-42,  7.9227e-09,  5.6052e-45,  5.6052e-45, -1.5674e-07,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -3.0310e-08,  1.3830e-07,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.3725e-08,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -1.6927e-08,\n",
            "        -9.1075e-08,  4.1109e-08,  5.6052e-45, -1.6740e-07, -4.9385e-08,\n",
            "        -1.3888e-07, -9.7392e-08,  5.6052e-45,  5.6052e-45, -1.8026e-07,\n",
            "         5.6052e-45, -4.6457e-09,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  1.5486e-07,  5.6052e-45, -7.9703e-09,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -2.4018e-07,\n",
            "         5.6052e-45, -1.2072e-08, -5.3552e-08, -6.6488e-09,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -8.8307e-08,  5.6052e-45, -1.0845e-07,  5.6052e-45,  5.6052e-45,\n",
            "        -5.8368e-11,  7.3434e-08,  1.0983e-07, -8.0880e-08, -3.5850e-18,\n",
            "         5.6052e-45,  5.6052e-45,  9.8721e-08,  5.6052e-45, -8.5457e-07,\n",
            "        -2.3591e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.3703e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45, -2.6119e-07,\n",
            "        -7.4409e-42, -9.7621e-08, -1.2425e-17, -1.8263e-07, -5.5764e-08,\n",
            "         5.6052e-45, -2.6744e-07,  5.6052e-45, -8.1387e-08,  5.6052e-45,\n",
            "        -1.6563e-09, -5.1245e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -8.3644e-08,  1.6989e-20,  9.0780e-08,  5.6052e-45,  5.6052e-45,\n",
            "        -1.7714e-10, -1.6859e-08,  6.0406e-08, -8.6140e-08,  5.6052e-45,\n",
            "        -1.7005e-07, -2.9557e-20,  5.6052e-45, -3.8010e-07,  2.7280e-08,\n",
            "        -4.6774e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.8815e-08, -9.4587e-08, -1.6428e-07,  5.6052e-45, -5.4576e-08,\n",
            "         5.6052e-45,  5.6052e-45, -1.3043e-06,  5.6052e-45, -4.9776e-08,\n",
            "        -1.1922e-07, -2.7546e-07, -2.3192e-07,  5.6052e-45, -4.3974e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -1.2224e-07, -1.1323e-07,\n",
            "         5.6052e-45,  2.1149e-07, -1.3909e-07,  5.6052e-45, -1.1117e-07,\n",
            "        -3.0972e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -1.1805e-07,  5.6052e-45,  1.5881e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45], device='cuda:0'), 'exp_avg_sq': tensor([1.8175e-10, 1.1196e-10, 7.1843e-07, 7.1927e-12, 1.0045e-06, 3.2982e-10,\n",
            "        1.1819e-08, 3.7170e-08, 1.7375e-07, 3.2047e-08, 1.6811e-09, 6.1353e-11,\n",
            "        3.3786e-07, 3.2943e-10, 5.6780e-08, 2.4521e-09, 8.1126e-08, 2.9364e-07,\n",
            "        1.0133e-07, 1.5154e-12, 2.9167e-13, 3.6427e-07, 4.6549e-11, 1.3816e-07,\n",
            "        6.0473e-07, 2.3099e-13, 4.6199e-09, 9.0676e-12, 1.9594e-09, 2.5258e-12,\n",
            "        9.3337e-12, 4.4748e-11, 8.4461e-11, 6.6691e-07, 9.7378e-07, 1.9565e-12,\n",
            "        8.1170e-10, 1.6270e-08, 1.1967e-10, 1.9722e-10, 1.4629e-07, 8.3807e-10,\n",
            "        2.0802e-07, 2.5020e-10, 7.4600e-10, 2.6600e-09, 2.0196e-12, 1.2354e-12,\n",
            "        1.0673e-10, 1.1090e-09, 1.5411e-14, 1.2915e-07, 5.4990e-12, 1.2012e-11,\n",
            "        2.6593e-10, 7.3222e-12, 3.3601e-07, 3.0776e-07, 6.6609e-14, 4.9000e-09,\n",
            "        2.9222e-07, 8.8280e-10, 1.4777e-07, 3.9350e-12, 3.0584e-12, 1.7689e-10,\n",
            "        4.3176e-07, 8.7164e-11, 1.0408e-12, 4.2906e-11, 8.6088e-08, 3.0501e-07,\n",
            "        1.3506e-07, 4.6614e-07, 2.1738e-09, 2.7586e-09, 1.9142e-09, 4.4801e-10,\n",
            "        2.8018e-12, 4.5561e-07, 2.4359e-14, 7.0407e-11, 5.5491e-12, 4.8432e-07,\n",
            "        3.0638e-10, 1.6738e-09, 1.8710e-09, 8.3112e-09, 1.3235e-10, 9.2069e-11,\n",
            "        2.4966e-13, 6.3768e-11, 1.3381e-06, 5.2963e-11, 3.0314e-13, 1.0094e-09,\n",
            "        3.3883e-07, 2.3848e-07, 2.7675e-09, 1.1404e-09, 4.7400e-13, 7.1340e-08,\n",
            "        4.4979e-11, 1.5204e-09, 1.2619e-08, 2.0172e-08, 7.2272e-12, 9.6916e-12,\n",
            "        3.1574e-12, 3.7109e-09, 4.9893e-15, 1.5001e-07, 5.5829e-09, 1.1421e-07,\n",
            "        4.1055e-07, 6.3459e-12, 2.4453e-07, 1.0719e-10, 5.8161e-09, 9.7510e-13,\n",
            "        8.1119e-10, 1.3237e-07, 9.7787e-10, 5.7638e-12, 3.1876e-10, 2.7311e-10,\n",
            "        9.6378e-10, 4.6527e-08, 8.7848e-11, 4.7338e-11, 6.4376e-11, 3.4736e-12,\n",
            "        1.0660e-09, 5.5185e-07, 8.6038e-08, 1.0393e-07, 3.7276e-10, 1.1982e-07,\n",
            "        1.2465e-07, 9.2043e-11, 4.1756e-09, 1.1948e-11, 9.5559e-08, 5.2748e-08,\n",
            "        1.0651e-07, 7.4982e-11, 3.3579e-10, 2.0648e-07, 1.5845e-09, 2.8503e-14,\n",
            "        3.9808e-07, 1.7101e-09, 6.5325e-07, 1.2656e-07, 3.8740e-12, 3.6673e-13,\n",
            "        2.6642e-07, 2.0770e-09, 1.2671e-09, 6.1093e-07, 1.6483e-07, 1.1471e-09,\n",
            "        5.4675e-08, 7.9149e-10, 1.2436e-09, 2.1868e-13, 8.9554e-13, 1.1598e-07,\n",
            "        5.4408e-09, 1.8283e-07, 5.6142e-11, 1.3629e-10, 2.0430e-12, 4.0562e-10,\n",
            "        1.2403e-07, 5.2884e-10, 1.9159e-13, 4.7881e-08, 1.5216e-12, 6.7029e-11,\n",
            "        2.5687e-12, 1.6576e-10, 2.9868e-11, 2.9231e-07, 1.8286e-10, 5.2981e-10,\n",
            "        4.1780e-11, 4.9847e-11, 7.4507e-10, 8.4047e-10, 1.3654e-07, 3.3831e-10,\n",
            "        4.5475e-08, 5.4344e-10, 1.3483e-07, 1.9088e-11, 8.3741e-07, 1.3044e-10,\n",
            "        3.6507e-07, 7.4308e-14, 8.0144e-09, 2.1320e-11, 8.7917e-12, 8.5668e-12,\n",
            "        6.6994e-08, 6.7024e-12, 3.2250e-08, 2.4319e-07, 1.3866e-11, 2.4112e-10,\n",
            "        1.7753e-07, 3.9954e-10, 1.7584e-10, 2.2913e-07, 2.0428e-07, 1.0321e-11,\n",
            "        1.3044e-09, 3.6242e-07, 7.0789e-10, 3.6249e-12, 6.6525e-11, 3.9823e-12,\n",
            "        2.3799e-08, 2.9694e-11, 2.1889e-12, 2.1690e-10, 8.6927e-08, 3.5083e-07,\n",
            "        1.4259e-07, 1.3001e-09, 4.4068e-11, 3.9787e-12, 1.4067e-08, 1.8961e-07,\n",
            "        2.8926e-08, 1.1151e-07, 1.1961e-10, 3.3740e-07, 3.6861e-11, 1.0437e-07,\n",
            "        6.3893e-11, 1.0292e-10, 1.6145e-07, 2.0704e-09, 7.8648e-10, 5.5267e-10,\n",
            "        1.4825e-07, 3.8940e-10, 1.1123e-07, 5.0301e-08, 3.3289e-10, 3.3161e-09,\n",
            "        1.1694e-09, 3.7892e-07, 2.1457e-10, 6.6675e-10, 4.1203e-08, 1.6028e-07,\n",
            "        1.8425e-07, 8.1798e-09, 1.8576e-10, 1.1372e-07, 6.5299e-08, 1.4079e-09,\n",
            "        3.5512e-10, 8.3230e-10, 1.9688e-10, 2.9267e-07, 2.2528e-07, 3.2408e-12,\n",
            "        8.5253e-09, 3.9427e-09, 3.1691e-10, 2.9713e-13, 1.9134e-12, 1.5744e-09,\n",
            "        1.1899e-10, 8.4111e-13, 6.0599e-10, 2.2400e-09, 5.9309e-08, 1.4135e-10,\n",
            "        2.8683e-09, 8.0040e-07, 2.8083e-13, 1.2178e-09, 4.7913e-10, 1.1080e-09,\n",
            "        8.6273e-10, 4.4805e-09, 8.4133e-10, 1.1912e-07, 1.2181e-11, 5.4007e-13,\n",
            "        1.0766e-07, 1.2858e-12, 1.9983e-08, 6.0205e-07, 7.4677e-09, 1.3852e-09,\n",
            "        1.0227e-09, 3.0579e-11, 1.0451e-09, 1.0831e-07, 1.8995e-12, 1.4634e-07,\n",
            "        1.9380e-10, 8.0997e-10, 2.9543e-08, 5.7872e-07, 2.2177e-10, 2.3276e-10,\n",
            "        9.8717e-11, 8.5489e-10, 1.1030e-07, 1.4184e-07, 3.2851e-07, 2.7047e-11,\n",
            "        5.6914e-11, 6.7272e-07, 8.3687e-11, 5.8490e-07, 3.5221e-09, 7.1403e-11,\n",
            "        3.7204e-07, 1.7962e-07, 3.2907e-11, 8.5262e-11, 4.0045e-11, 1.4579e-12,\n",
            "        6.1293e-08, 1.6667e-12, 4.9582e-09, 1.5275e-13, 5.2577e-07, 3.7649e-07,\n",
            "        3.5198e-09, 3.2420e-07, 1.4220e-08, 1.9954e-11, 3.0781e-10, 7.4702e-08,\n",
            "        2.7275e-07, 1.4945e-07, 6.8537e-11, 1.3062e-09, 6.4105e-12, 3.6779e-10,\n",
            "        4.0695e-11, 2.7636e-10, 1.0896e-12, 1.7365e-07, 5.4300e-11, 1.7637e-13,\n",
            "        1.0240e-07, 3.8291e-10, 3.2219e-10, 6.4651e-11, 4.5746e-07, 1.5749e-07,\n",
            "        9.3676e-12, 6.1561e-10, 6.7153e-10, 4.3075e-08, 8.3952e-10, 3.3720e-11,\n",
            "        6.1937e-10, 8.0353e-12, 1.1318e-09, 2.4816e-07, 2.1253e-07, 1.8406e-07,\n",
            "        3.1753e-11, 5.0488e-07, 9.3754e-08, 3.0746e-07, 8.0960e-07, 4.7360e-11,\n",
            "        2.3098e-10, 2.6794e-07, 6.0479e-11, 7.7424e-08, 3.3792e-10, 5.7836e-11,\n",
            "        7.0772e-14, 2.4015e-11, 2.6388e-10, 3.7243e-12, 2.3119e-10, 9.5944e-12,\n",
            "        4.0236e-11, 2.0744e-12, 3.0749e-07, 4.7402e-10, 1.6073e-07, 6.4971e-12,\n",
            "        2.2928e-13, 1.0345e-11, 2.6869e-10, 2.7131e-07, 1.0980e-10, 4.2275e-08,\n",
            "        3.0284e-08, 2.5457e-08, 3.4984e-10, 1.0081e-10, 8.6345e-10, 1.8833e-10,\n",
            "        2.9048e-12, 8.2459e-10, 3.7348e-07, 8.8852e-11, 4.1992e-07, 2.0794e-11,\n",
            "        3.9484e-10, 1.4892e-08, 4.5895e-07, 4.6250e-07, 3.3415e-07, 5.3606e-09,\n",
            "        9.1213e-10, 4.1314e-10, 1.0849e-07, 1.0515e-12, 5.2706e-07, 2.4340e-07,\n",
            "        6.2064e-14, 3.1942e-10, 1.3517e-09, 3.0711e-11, 3.4649e-07, 4.7045e-09,\n",
            "        5.6463e-10, 2.7361e-09, 4.2932e-07, 6.8742e-09, 1.2498e-07, 3.0723e-09,\n",
            "        2.4816e-07, 2.2397e-07, 5.3841e-11, 2.4450e-07, 6.1444e-12, 1.7662e-07,\n",
            "        9.7853e-10, 3.0358e-08, 3.4887e-07, 7.7247e-12, 7.3748e-10, 3.4767e-12,\n",
            "        3.2877e-07, 9.9482e-09, 7.7772e-08, 3.2904e-09, 1.0398e-13, 1.8493e-08,\n",
            "        9.0804e-08, 1.0259e-07, 2.6572e-07, 5.1397e-10, 3.2234e-07, 2.2748e-09,\n",
            "        3.1418e-13, 3.1522e-06, 1.0374e-07, 8.1409e-08, 5.4961e-10, 7.0916e-11,\n",
            "        1.7342e-13, 1.8807e-09, 2.6229e-07, 4.0787e-08, 2.0345e-07, 2.2053e-10,\n",
            "        1.0024e-07, 1.3358e-11, 7.3021e-10, 1.1006e-06, 1.0286e-12, 1.5862e-07,\n",
            "        4.6932e-07, 3.2849e-07, 2.5973e-07, 3.0591e-14, 7.1764e-08, 1.3796e-10,\n",
            "        2.4791e-09, 1.3703e-11, 6.9228e-11, 1.0095e-11, 2.4073e-11, 3.6116e-10,\n",
            "        7.8101e-12, 2.1516e-08, 1.5498e-07, 3.4176e-11, 4.0481e-07, 1.9188e-07,\n",
            "        1.5351e-10, 1.3218e-07, 2.6437e-08, 5.8659e-10, 3.5837e-11, 2.9644e-11,\n",
            "        1.4594e-11, 6.4354e-11, 2.8798e-07, 4.9237e-11, 2.0112e-07, 3.1845e-11,\n",
            "        3.1369e-10, 3.8239e-09], device='cuda:0')}, 10: {'step': tensor(1920.), 'exp_avg': tensor([[ 0.0000e+00,  0.0000e+00, -5.0238e-30,  ...,  0.0000e+00,\n",
            "          0.0000e+00,  5.6052e-45],\n",
            "        [-5.6052e-45, -5.6052e-45, -8.2683e-07,  ..., -5.6052e-45,\n",
            "         -5.6052e-45,  5.6052e-45],\n",
            "        [ 0.0000e+00,  0.0000e+00, -9.4504e-08,  ...,  0.0000e+00,\n",
            "          5.6052e-45,  5.6052e-45],\n",
            "        ...,\n",
            "        [ 5.6052e-45, -5.6052e-45, -2.1020e-06,  ..., -5.6052e-45,\n",
            "          5.6052e-45,  5.6052e-45],\n",
            "        [ 5.6052e-45,  5.6052e-45,  2.1046e-07,  ...,  5.6052e-45,\n",
            "         -5.6052e-45,  5.6052e-45],\n",
            "        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n",
            "          5.6052e-45,  5.6052e-45]], device='cuda:0'), 'exp_avg_sq': tensor([[0.0000e+00, 0.0000e+00, 9.6366e-13,  ..., 0.0000e+00, 0.0000e+00,\n",
            "         2.2277e-09],\n",
            "        [2.8315e-15, 1.2421e-14, 1.1266e-06,  ..., 5.1316e-13, 2.0743e-13,\n",
            "         4.4270e-08],\n",
            "        [0.0000e+00, 0.0000e+00, 8.0259e-07,  ..., 0.0000e+00, 7.0816e-16,\n",
            "         5.8378e-09],\n",
            "        ...,\n",
            "        [1.5060e-14, 4.3494e-14, 5.3897e-06,  ..., 8.4541e-13, 3.2791e-13,\n",
            "         3.5727e-07],\n",
            "        [2.4110e-13, 6.3793e-15, 2.1063e-06,  ..., 2.0340e-12, 1.0879e-12,\n",
            "         7.3120e-08],\n",
            "        [2.6509e-13, 3.4684e-14, 1.0709e-11,  ..., 4.0154e-12, 1.7295e-12,\n",
            "         1.7083e-13]], device='cuda:0')}, 11: {'step': tensor(1920.), 'exp_avg': tensor([-2.5140e-11, -1.5801e-07, -5.4711e-08, -1.1062e-08, -3.9508e-10,\n",
            "        -3.8151e-07,  5.6052e-45, -5.6052e-45, -2.3397e-07, -3.5800e-07,\n",
            "         1.5782e-07,  2.3309e-36,  1.5668e-36, -1.9553e-07,  3.2582e-09,\n",
            "        -9.6083e-08, -3.5740e-10, -4.5451e-08, -2.8519e-07,  2.0646e-40,\n",
            "        -4.4676e-07, -1.6577e-07, -6.9549e-08, -1.2115e-08, -1.2608e-07,\n",
            "        -8.1256e-08, -6.2316e-07, -1.8267e-07,  5.5718e-08,  6.8844e-08,\n",
            "        -6.5610e-08,  2.5582e-07, -1.4824e-08,  1.0860e-22, -2.8974e-07,\n",
            "         5.6052e-45, -3.0075e-10, -1.8219e-07,  1.0823e-29,  5.6052e-45,\n",
            "        -2.7834e-07, -8.9397e-09, -4.8525e-07,  5.6052e-45, -3.9529e-07,\n",
            "        -1.0970e-07, -3.8112e-07,  5.6052e-45, -2.3831e-08,  5.6052e-45,\n",
            "        -1.5114e-07, -1.2972e-08, -1.6677e-07, -2.0791e-07, -2.3424e-08,\n",
            "        -3.5181e-08, -2.7034e-07, -3.2246e-08,  1.0267e-07, -4.9816e-09,\n",
            "        -5.8542e-08, -1.5545e-07, -9.2013e-07,  1.8014e-39, -1.5150e-12,\n",
            "         7.9472e-08, -5.2231e-07, -2.6458e-07, -2.1976e-07,  9.9819e-38,\n",
            "        -1.8105e-08, -2.5617e-20, -6.3803e-08, -6.2247e-09, -3.4210e-08,\n",
            "        -4.1557e-07, -4.0805e-07, -1.3457e-07, -8.1225e-08, -1.0693e-08,\n",
            "        -2.6101e-07,  6.9877e-08, -4.8446e-08,  1.2019e-08, -4.3635e-08,\n",
            "        -6.7550e-08, -6.7262e-07, -4.4926e-08,  6.2161e-09,  1.8347e-08,\n",
            "        -2.4085e-08,  5.6052e-45,  5.6052e-45, -2.6615e-14, -4.0488e-07,\n",
            "         6.3401e-08,  5.6052e-45, -2.4430e-08, -3.0418e-11, -1.1541e-07,\n",
            "        -1.0858e-07, -1.3711e-08,  2.2615e-39,  5.3314e-36, -3.0377e-08,\n",
            "         5.6052e-45, -1.6591e-07,  2.5585e-07, -1.7524e-07, -1.1278e-08,\n",
            "        -5.5820e-10, -1.7429e-08, -1.9367e-07, -1.1265e-07, -8.6009e-08,\n",
            "         5.6052e-45,  2.0526e-07,  1.3741e-07, -6.2071e-08, -6.0983e-08,\n",
            "         5.6052e-45, -1.7457e-07, -1.4901e-11, -7.4724e-12, -1.5544e-07,\n",
            "        -7.6967e-08,  5.4454e-16, -1.3218e-07, -1.5188e-07, -3.3889e-08,\n",
            "        -2.6306e-07, -4.5664e-07, -7.6783e-07, -1.2370e-07, -9.6043e-10,\n",
            "         5.6052e-45, -3.2016e-10,  5.6052e-45, -7.0995e-07, -3.3867e-07,\n",
            "        -1.6645e-07, -1.7496e-07, -7.4140e-07, -2.2127e-07, -6.3641e-08,\n",
            "         5.6052e-45, -3.0967e-07, -2.9952e-07, -1.5708e-07, -1.4480e-07,\n",
            "        -2.3088e-08, -6.0877e-10,  2.7769e-09,  1.5384e-30, -1.4687e-09,\n",
            "         1.8246e-07,  5.6052e-45, -2.1727e-07,  1.2164e-08, -6.1657e-44,\n",
            "        -7.3462e-11,  1.0808e-07, -3.1706e-07,  5.8836e-12, -2.4552e-12,\n",
            "        -1.8263e-07, -3.1480e-07, -2.6697e-08,  1.8904e-08, -2.0418e-07,\n",
            "         5.6052e-45,  8.1429e-10, -2.6622e-08, -1.3536e-07,  5.6052e-45,\n",
            "        -1.5417e-13, -8.2829e-08, -8.1149e-08, -2.8307e-09,  6.6344e-10,\n",
            "         5.6052e-45, -2.0120e-07,  1.6414e-07, -7.4694e-08, -1.1007e-08,\n",
            "        -4.2050e-08, -1.4888e-07,  2.8988e-32, -7.2581e-08, -6.0462e-10,\n",
            "        -5.4825e-07,  5.6052e-45, -2.8995e-08, -1.3868e-07, -4.8723e-08,\n",
            "         1.7200e-07, -3.3733e-08, -1.6681e-10, -8.4286e-08,  3.8818e-38,\n",
            "        -1.3776e-08, -3.0763e-11, -2.2994e-07, -2.0031e-07, -2.7236e-07,\n",
            "        -3.7058e-07, -9.2267e-09, -3.2288e-07, -1.7942e-07, -3.0790e-07,\n",
            "        -2.2409e-07, -9.5553e-11, -1.9941e-08, -1.6743e-07, -1.2216e-07,\n",
            "         1.2610e-07, -9.3054e-08,  2.0738e-18, -1.1640e-07,  4.6908e-08,\n",
            "        -1.6416e-10, -3.0619e-19,  8.5092e-35, -1.8265e-17, -1.0500e-07,\n",
            "        -2.2201e-07, -9.9618e-09, -7.1318e-10,  5.6052e-45, -7.2106e-08,\n",
            "         5.6052e-45, -2.2509e-07,  1.3620e-08, -2.5145e-08, -5.7494e-08,\n",
            "         4.0989e-14, -1.7695e-07, -2.4308e-09, -1.4172e-07,  8.6976e-26,\n",
            "        -1.2119e-07, -1.9608e-07, -2.3635e-08, -6.6048e-07,  1.8722e-11,\n",
            "         5.6052e-45,  5.9364e-08, -2.9305e-08, -3.8504e-08,  2.7709e-32,\n",
            "        -2.1504e-08, -1.0996e-07, -1.8373e-08, -4.7312e-07, -2.1291e-08,\n",
            "        -5.6052e-45], device='cuda:0'), 'exp_avg_sq': tensor([4.8186e-09, 1.9786e-07, 1.0181e-07, 3.3121e-08, 1.9950e-09, 1.4681e-06,\n",
            "        3.2836e-09, 1.4280e-08, 5.5806e-07, 3.6307e-07, 8.5814e-07, 1.2523e-09,\n",
            "        1.4653e-09, 3.0636e-07, 4.7887e-07, 3.5072e-07, 1.5220e-08, 2.0036e-06,\n",
            "        1.0527e-06, 4.4126e-11, 1.6446e-06, 1.4438e-07, 6.5334e-08, 6.0400e-09,\n",
            "        3.9584e-07, 1.3341e-07, 2.0436e-06, 7.2513e-07, 7.9297e-07, 6.4857e-07,\n",
            "        8.0234e-07, 2.3353e-06, 2.3097e-09, 3.0035e-09, 1.5867e-07, 3.0887e-09,\n",
            "        6.7285e-09, 3.0242e-08, 1.1566e-08, 3.7663e-09, 1.9597e-07, 3.9681e-07,\n",
            "        1.6995e-06, 2.9161e-10, 2.1327e-07, 8.1816e-07, 6.4745e-07, 4.9244e-10,\n",
            "        1.0899e-09, 1.2165e-09, 5.2704e-08, 2.8541e-08, 6.9546e-08, 4.2895e-09,\n",
            "        1.3691e-07, 2.2150e-08, 7.2632e-08, 5.9336e-08, 1.8061e-07, 5.8468e-09,\n",
            "        1.1583e-07, 4.0433e-07, 1.0976e-06, 2.1945e-10, 6.0740e-09, 5.0538e-07,\n",
            "        2.2823e-06, 2.9615e-07, 1.2272e-06, 2.3365e-08, 1.7658e-08, 1.7871e-08,\n",
            "        4.2300e-07, 2.7165e-07, 3.1740e-09, 1.1618e-06, 1.1266e-07, 4.2498e-08,\n",
            "        6.3214e-08, 3.1549e-08, 3.7186e-07, 4.8105e-07, 1.6925e-07, 3.4258e-07,\n",
            "        7.5741e-09, 2.8115e-07, 3.9666e-06, 3.0577e-07, 4.9913e-09, 6.7999e-07,\n",
            "        1.9902e-08, 1.1625e-08, 7.5782e-11, 4.9736e-09, 1.5249e-06, 7.5528e-07,\n",
            "        4.0502e-09, 2.8700e-08, 2.1063e-10, 1.2694e-07, 4.7964e-07, 1.3873e-07,\n",
            "        2.9453e-08, 2.9426e-09, 1.6311e-07, 2.8414e-08, 8.3369e-07, 2.3779e-06,\n",
            "        3.6029e-07, 1.6573e-09, 2.1718e-09, 2.7562e-07, 1.3586e-07, 1.9397e-08,\n",
            "        5.0360e-07, 2.6892e-10, 1.6692e-09, 8.1604e-07, 1.9002e-09, 6.1962e-08,\n",
            "        2.4408e-08, 6.5380e-07, 6.7344e-10, 1.4639e-08, 2.1625e-07, 4.0758e-07,\n",
            "        1.2574e-08, 4.5956e-07, 1.7927e-07, 2.7383e-08, 6.7540e-08, 4.8237e-07,\n",
            "        7.0889e-07, 7.8608e-07, 1.2987e-08, 4.5319e-10, 2.6481e-09, 1.1918e-08,\n",
            "        2.1268e-06, 6.7390e-08, 2.9367e-07, 4.9207e-07, 3.1487e-06, 6.8530e-08,\n",
            "        8.2976e-08, 1.8940e-09, 8.1913e-07, 3.4041e-07, 4.4659e-08, 2.4092e-06,\n",
            "        1.0751e-07, 2.4521e-08, 9.0339e-09, 6.1232e-12, 5.6566e-10, 1.5946e-06,\n",
            "        2.5504e-10, 3.8015e-07, 4.7192e-07, 8.6493e-09, 4.7383e-08, 1.7744e-06,\n",
            "        1.3713e-06, 8.7011e-09, 2.9893e-10, 3.7546e-07, 6.4665e-07, 2.3062e-08,\n",
            "        1.9148e-07, 1.2685e-07, 1.6429e-12, 1.8928e-08, 9.9374e-10, 2.7167e-07,\n",
            "        4.8450e-09, 4.0783e-09, 2.0168e-09, 4.5523e-08, 2.1869e-09, 9.6731e-11,\n",
            "        2.1830e-10, 8.1907e-07, 2.1682e-06, 3.0953e-10, 2.8030e-07, 7.5130e-08,\n",
            "        3.4218e-08, 3.9819e-09, 1.1366e-07, 1.1426e-08, 1.8486e-06, 1.3628e-08,\n",
            "        3.3172e-08, 3.8195e-08, 1.5549e-07, 1.3511e-06, 1.1287e-07, 5.2454e-09,\n",
            "        1.4207e-07, 2.6834e-10, 2.1750e-08, 5.6198e-10, 5.1474e-07, 8.6447e-07,\n",
            "        3.9847e-07, 3.4368e-07, 7.8729e-09, 2.1230e-07, 3.8500e-08, 5.4713e-07,\n",
            "        5.7793e-07, 1.4451e-08, 3.8232e-08, 6.7450e-07, 3.0505e-07, 2.3727e-06,\n",
            "        2.9903e-08, 1.4763e-08, 5.7740e-08, 3.9846e-07, 1.3806e-09, 1.0520e-10,\n",
            "        1.7068e-09, 3.6566e-08, 2.1062e-08, 3.7427e-07, 1.2357e-07, 5.4582e-09,\n",
            "        1.3656e-09, 4.6517e-07, 3.2824e-08, 8.9528e-07, 3.6479e-07, 2.8883e-09,\n",
            "        4.3501e-08, 1.0607e-09, 1.6861e-07, 4.3746e-09, 1.8903e-07, 1.4658e-08,\n",
            "        2.3366e-08, 9.6536e-08, 2.2619e-08, 3.0825e-06, 1.0439e-08, 7.9096e-09,\n",
            "        2.8393e-07, 1.3197e-06, 2.2237e-07, 4.1128e-09, 2.4161e-08, 1.4414e-07,\n",
            "        3.9619e-07, 1.1538e-06, 5.9279e-07, 2.6610e-08], device='cuda:0')}, 12: {'step': tensor(1920.), 'exp_avg': tensor([[ 4.0412e-10,  2.7548e-05,  2.1088e-06,  9.3537e-08, -1.8357e-08,\n",
            "          5.3770e-06,  5.6052e-45,  5.6052e-45,  2.6638e-05,  4.6122e-06,\n",
            "         -8.9158e-06,  1.2124e-37, -1.4411e-35,  3.1222e-05,  2.1600e-06,\n",
            "          1.1914e-05,  3.1921e-09, -1.2245e-05, -7.4120e-06,  2.7143e-42,\n",
            "          2.0993e-05,  5.9992e-07,  2.5727e-05,  5.1539e-09,  2.5886e-06,\n",
            "         -1.4350e-06,  2.5808e-05,  3.0342e-06, -1.7401e-05,  8.6468e-08,\n",
            "         -1.6310e-05, -5.8057e-06, -1.2132e-07,  4.6948e-22,  2.5175e-05,\n",
            "          5.6052e-45,  2.1788e-08, -1.9149e-06,  3.7463e-32,  5.6052e-44,\n",
            "          1.8732e-07,  8.7083e-06,  2.6206e-05,  5.6052e-45, -1.1704e-05,\n",
            "          2.1112e-06,  1.5944e-05,  5.6052e-45,  1.3655e-06, -5.6052e-45,\n",
            "          3.3261e-06, -6.2228e-07,  2.7552e-05,  7.8291e-06,  3.0376e-06,\n",
            "          2.2714e-06,  1.2335e-05, -3.9325e-06, -3.8371e-06,  7.0457e-10,\n",
            "         -1.2439e-07,  1.8692e-06,  2.7760e-05,  7.0203e-39,  1.2796e-11,\n",
            "         -3.9614e-07, -1.0988e-05,  2.6467e-05,  2.5446e-06, -2.6468e-37,\n",
            "         -8.7679e-07,  1.0194e-24, -4.0639e-07,  2.5699e-06,  2.3826e-08,\n",
            "         -7.5885e-06,  8.4254e-06, -2.5316e-07,  3.3544e-06,  3.8282e-06,\n",
            "          1.6176e-05, -7.9387e-06, -9.2516e-06,  1.5649e-05, -1.1435e-07,\n",
            "          1.2102e-07,  2.7412e-05,  3.5116e-06,  5.7560e-07,  2.9585e-06,\n",
            "          1.3265e-06,  5.6052e-45, -5.6052e-45,  3.9812e-12,  3.0659e-05,\n",
            "         -9.3502e-06,  5.6052e-45, -8.0102e-06,  2.1842e-09,  2.1326e-05,\n",
            "          2.6608e-05,  2.1508e-07,  1.8778e-39,  3.8804e-35,  2.3224e-06,\n",
            "         -5.6052e-45,  9.5939e-07, -7.2690e-06,  3.6339e-06, -2.3365e-06,\n",
            "         -4.8607e-09,  3.0874e-05,  3.3859e-06,  4.2306e-07,  3.2273e-06,\n",
            "         -5.6052e-45, -1.3885e-05, -5.9665e-06, -4.6883e-06,  1.3378e-05,\n",
            "          5.6052e-45,  4.2359e-07,  6.2958e-11,  8.1592e-10,  3.6891e-05,\n",
            "          4.0393e-06,  1.0226e-14, -6.8200e-06,  2.6752e-05,  2.6599e-08,\n",
            "          1.2743e-05,  1.1867e-05,  6.3114e-06, -5.2396e-06,  1.3832e-08,\n",
            "          5.6052e-45,  7.3546e-09,  5.6052e-45,  2.0569e-05,  2.6355e-05,\n",
            "         -2.5196e-06,  4.0829e-06,  2.7302e-05, -1.4553e-07,  3.5001e-06,\n",
            "          5.6052e-45,  2.4477e-05,  1.8377e-05,  1.2032e-05, -9.0317e-06,\n",
            "          6.5268e-07,  1.3725e-08,  6.2404e-09,  1.5936e-30,  3.5189e-08,\n",
            "         -1.1223e-05,  5.6052e-45,  2.2862e-06, -9.6213e-06,  2.3262e-43,\n",
            "          1.2067e-09, -1.2514e-05,  1.7176e-05,  9.2740e-13, -3.5621e-11,\n",
            "          1.6083e-06,  1.4752e-06,  2.3657e-06,  9.3892e-06,  2.0411e-05,\n",
            "          5.6052e-45, -5.8596e-08,  1.1915e-07,  5.9076e-07, -5.6052e-45,\n",
            "          8.4874e-13, -3.4689e-07, -1.1067e-08,  8.9440e-07,  5.3151e-10,\n",
            "         -5.6052e-45,  2.8310e-05, -4.3298e-06,  2.3106e-07,  1.3730e-06,\n",
            "          1.3983e-06,  1.3989e-05,  5.8810e-34,  2.0097e-05, -3.6359e-08,\n",
            "          2.4225e-05,  5.6052e-45,  1.1697e-05,  1.2269e-06,  2.2105e-05,\n",
            "         -4.9890e-06,  2.9432e-06,  1.3139e-07,  2.3510e-06, -1.3498e-38,\n",
            "         -2.3468e-09,  1.2872e-12,  2.9158e-05,  2.7108e-05, -3.2889e-06,\n",
            "          1.6928e-05,  3.9532e-09,  1.7167e-05, -8.4102e-06,  1.9764e-05,\n",
            "          2.7959e-05,  2.0804e-09,  2.1380e-06, -1.1228e-05,  2.6878e-06,\n",
            "         -1.2001e-05, -1.8008e-07,  2.3176e-17, -1.3832e-05, -7.0420e-06,\n",
            "         -2.2172e-10,  7.8528e-20,  1.2501e-37,  2.3608e-22,  8.9564e-07,\n",
            "          3.5463e-06, -1.1799e-07, -3.6387e-08, -5.6052e-45,  5.3417e-06,\n",
            "         -5.6052e-45,  3.0781e-05,  3.2382e-06,  2.2256e-09,  5.4047e-07,\n",
            "          1.3036e-14,  1.1764e-05,  4.0640e-09,  1.2460e-05,  1.5129e-25,\n",
            "          1.4062e-05,  1.7742e-05, -3.5554e-08,  2.5902e-05,  4.8513e-13,\n",
            "          5.6052e-45, -2.6356e-06,  5.2381e-06,  3.8892e-06,  1.9984e-37,\n",
            "          5.2024e-08,  3.0815e-06,  8.3539e-06,  1.7745e-05,  3.0788e-06,\n",
            "          2.7746e-43],\n",
            "        [-5.6056e-10, -2.7583e-05, -1.3476e-05, -2.8675e-07,  1.1015e-08,\n",
            "         -6.2527e-06, -5.6052e-45,  5.6052e-45, -3.8822e-05, -1.1377e-05,\n",
            "          8.4447e-06, -2.6780e-36,  1.7370e-35, -3.1404e-05, -1.0134e-05,\n",
            "         -2.1406e-05, -1.2246e-07,  1.5661e-05,  1.3801e-05, -7.3441e-39,\n",
            "         -2.1738e-05,  1.8670e-06, -2.5873e-05, -5.6281e-07, -1.5275e-05,\n",
            "          2.5067e-06, -3.8795e-05,  5.3205e-06,  1.8897e-05, -9.8675e-07,\n",
            "          1.7173e-05,  4.9625e-06,  7.3631e-07,  2.2339e-23, -2.5256e-05,\n",
            "         -5.6052e-45, -2.1823e-08,  2.1309e-06,  2.4182e-28, -5.3249e-44,\n",
            "          8.6421e-06, -9.7994e-06, -2.6277e-05, -5.6052e-45,  1.4739e-05,\n",
            "          3.5605e-06, -2.1655e-05, -5.6052e-45,  2.1416e-06,  5.6052e-45,\n",
            "          6.6815e-06,  6.2003e-07, -2.7677e-05, -1.2733e-05, -1.4585e-05,\n",
            "         -7.6273e-06, -1.2435e-05,  3.5807e-06,  3.7789e-06, -4.6803e-08,\n",
            "          3.3388e-07, -1.5230e-05, -2.8000e-05, -6.6286e-39,  9.1205e-18,\n",
            "         -7.0360e-07,  1.5393e-05, -2.7182e-05,  4.4971e-06,  2.6310e-37,\n",
            "          7.8866e-07,  4.1983e-22,  4.9611e-06, -9.7324e-06, -6.1855e-07,\n",
            "          1.1513e-05, -8.6458e-06,  8.2590e-06, -1.2376e-05, -4.1769e-06,\n",
            "         -2.0826e-05,  8.0907e-06,  7.5914e-06, -1.7205e-05,  1.3552e-06,\n",
            "          3.5268e-06, -3.3491e-05, -1.5562e-05, -5.7545e-07, -9.6618e-07,\n",
            "         -1.1680e-06, -5.6052e-45,  5.6052e-45, -5.1888e-12, -4.3745e-05,\n",
            "          8.6680e-06, -5.6052e-45,  7.4376e-06, -2.2584e-09, -2.1450e-05,\n",
            "         -2.9891e-05,  2.1708e-06,  1.6818e-38, -3.8804e-35,  3.4911e-06,\n",
            "          5.6052e-45, -1.2270e-05,  7.7599e-06, -2.3931e-05,  2.4347e-06,\n",
            "          4.0730e-09, -3.2177e-05,  5.8757e-06, -7.8045e-06, -1.7777e-05,\n",
            "          5.6052e-45,  1.1880e-05,  5.0946e-06,  4.4623e-06, -1.4018e-05,\n",
            "         -5.6052e-45, -1.2426e-05, -6.3036e-11, -8.3740e-10, -4.0565e-05,\n",
            "         -1.8329e-05, -9.5460e-26,  8.4565e-06, -2.7135e-05, -1.0362e-06,\n",
            "         -1.3104e-05, -1.4058e-05, -6.0252e-06,  8.4716e-06,  7.7250e-10,\n",
            "         -5.6052e-45, -7.2761e-09, -5.6052e-45, -2.7562e-05, -2.6367e-05,\n",
            "          2.3641e-06, -2.1228e-05, -3.9041e-05,  6.0762e-06, -1.2854e-05,\n",
            "          5.6052e-45, -3.4399e-05, -2.4592e-05, -1.1884e-05,  1.5474e-05,\n",
            "          1.1180e-06, -7.5767e-08, -8.9341e-09, -1.6154e-30,  4.5140e-07,\n",
            "          1.2125e-05, -5.6052e-45,  3.7517e-06,  1.0201e-05, -2.3262e-43,\n",
            "         -1.3670e-09,  1.2475e-05, -2.1374e-05,  3.1161e-11,  3.4609e-11,\n",
            "          3.0439e-06, -1.5215e-05,  3.5246e-06, -1.0258e-05, -2.0718e-05,\n",
            "         -5.6052e-45,  1.9869e-08,  6.5848e-07, -1.6241e-05,  5.6052e-45,\n",
            "         -9.9748e-13,  3.9958e-06,  1.3046e-06, -1.0988e-06,  1.9315e-08,\n",
            "          5.6052e-45, -4.4754e-05,  1.6598e-06,  5.0494e-06, -6.4287e-06,\n",
            "          2.9116e-06, -1.3996e-05,  1.6242e-32, -2.0718e-05,  2.9304e-08,\n",
            "         -3.1956e-05,  5.6052e-45, -1.2150e-05,  8.7743e-06, -2.3424e-05,\n",
            "          3.8249e-06,  3.5756e-06, -1.3371e-07,  3.8277e-06, -6.9202e-38,\n",
            "         -6.0722e-07,  1.9165e-08, -3.8077e-05, -2.7453e-05,  7.5933e-06,\n",
            "         -1.6985e-05, -6.5791e-07, -1.9327e-05,  7.5515e-06, -3.0876e-05,\n",
            "         -3.7640e-05, -2.1240e-09, -4.1861e-06,  1.1509e-05,  4.7283e-06,\n",
            "          1.2219e-05,  3.9100e-06, -5.9135e-43,  1.3615e-05,  6.0593e-06,\n",
            "          1.3274e-10, -5.6052e-45,  1.4548e-33,  7.9292e-18,  2.8348e-06,\n",
            "         -1.4185e-05,  9.1269e-08,  2.1807e-08,  5.6052e-45, -2.0129e-05,\n",
            "          5.6052e-45, -4.7010e-05,  1.6346e-06,  3.3443e-07,  1.6859e-06,\n",
            "         -2.3152e-25, -1.0327e-05, -7.2505e-08, -1.1827e-05, -1.6146e-25,\n",
            "         -1.4092e-05, -1.8787e-05,  4.6469e-07, -3.2735e-05,  7.9866e-10,\n",
            "          5.6052e-45,  1.9889e-06, -1.1091e-05, -6.6829e-06, -2.9123e-32,\n",
            "          1.3580e-06,  5.9506e-06, -1.3968e-05, -1.7898e-05, -4.6474e-06,\n",
            "         -3.0408e-43],\n",
            "        [ 1.7436e-10,  4.3354e-08,  1.1372e-05,  1.9335e-07,  7.3332e-09,\n",
            "          8.7911e-07,  5.6052e-45, -5.6052e-45,  1.2201e-05,  6.7697e-06,\n",
            "          4.8315e-07,  2.5568e-36, -2.9582e-36,  1.9333e-07,  7.9787e-06,\n",
            "          9.5080e-06,  1.1934e-07, -3.3989e-06, -6.3750e-06,  7.3414e-39,\n",
            "          7.5092e-07, -2.4657e-06,  1.5665e-07,  5.5806e-07,  1.2691e-05,\n",
            "         -1.0702e-06,  1.3000e-05, -8.3526e-06, -1.4779e-06,  9.1169e-07,\n",
            "         -8.4406e-07,  8.6087e-07, -6.1437e-07, -4.9181e-22,  8.6373e-08,\n",
            "          5.6052e-45,  2.1572e-10, -2.1581e-07, -2.4170e-28, -5.6052e-45,\n",
            "         -8.8226e-06,  1.1091e-06,  7.8348e-08,  5.6052e-45, -3.0203e-06,\n",
            "         -5.6698e-06,  5.7187e-06,  5.6052e-45, -3.5066e-06,  5.6052e-45,\n",
            "         -1.0004e-05,  2.3383e-09,  1.3315e-07,  4.9097e-06,  1.1561e-05,\n",
            "          5.3588e-06,  1.0369e-07,  3.5502e-07,  5.9409e-08,  4.6046e-08,\n",
            "         -2.0935e-07,  1.3364e-05,  2.4701e-07, -3.9165e-40,  1.4404e-13,\n",
            "          1.1148e-06, -4.3869e-06,  7.2418e-07, -7.0396e-06,  1.5761e-39,\n",
            "          8.9462e-08, -4.2534e-22, -4.5459e-06,  7.1684e-06,  5.9538e-07,\n",
            "         -3.9069e-06,  2.2311e-07, -7.9997e-06,  9.0259e-06,  3.5076e-07,\n",
            "          4.6563e-06, -1.4042e-07,  1.6765e-06,  1.5760e-06, -1.2399e-06,\n",
            "         -3.6427e-06,  6.0888e-06,  1.2056e-05,  5.1977e-10, -1.9896e-06,\n",
            "         -1.5690e-07,  5.6052e-45,  5.6052e-45,  5.3401e-13,  1.3103e-05,\n",
            "          6.9260e-07,  5.6052e-45,  5.8091e-07,  3.9424e-11,  1.3073e-07,\n",
            "          3.3127e-06, -2.3819e-06, -1.8696e-38,  6.1657e-44, -5.8118e-06,\n",
            "          5.6052e-45,  1.1312e-05, -4.7498e-07,  2.0303e-05, -9.6489e-08,\n",
            "          8.0256e-10,  1.3266e-06, -9.2588e-06,  7.3846e-06,  1.4555e-05,\n",
            "          5.6052e-45,  2.0285e-06,  8.8534e-07,  2.2696e-07,  6.4429e-07,\n",
            "          5.6052e-45,  1.2005e-05,  4.6097e-14,  2.5454e-11,  3.7024e-06,\n",
            "          1.4297e-05,  2.6005e-16, -1.6227e-06,  3.9562e-07,  1.0109e-06,\n",
            "          3.6546e-07,  2.1947e-06, -2.8317e-07, -3.2168e-06, -1.4606e-08,\n",
            "          5.6052e-45,  4.4721e-11,  5.6052e-45,  7.0006e-06,  1.8151e-08,\n",
            "          1.6006e-07,  1.7152e-05,  1.1751e-05, -5.9277e-06,  9.3585e-06,\n",
            "         -5.6052e-45,  9.9356e-06,  6.2240e-06, -1.4445e-07, -6.4291e-06,\n",
            "         -1.7692e-06,  6.2215e-08,  2.6564e-09,  3.7876e-32, -4.8657e-07,\n",
            "         -8.8370e-07,  5.6052e-45, -6.0361e-06, -5.6815e-07,  5.6052e-45,\n",
            "          1.8454e-10,  5.6029e-08,  4.2065e-06, -2.9837e-11,  4.1392e-13,\n",
            "         -4.6507e-06,  1.3742e-05, -5.8888e-06,  8.8065e-07,  3.1299e-07,\n",
            "          5.6052e-45,  3.8984e-08, -7.7703e-07,  1.5655e-05,  5.6052e-45,\n",
            "          1.0956e-13, -3.6471e-06, -1.2933e-06,  2.0485e-07, -1.9838e-08,\n",
            "          5.6052e-45,  1.6474e-05,  2.6852e-06, -5.2788e-06,  5.0577e-06,\n",
            "         -4.3083e-06,  1.1445e-08, -1.6830e-32,  6.3395e-07,  7.0556e-09,\n",
            "          7.7421e-06, -5.6052e-45,  4.5993e-07, -9.9965e-06,  1.3359e-06,\n",
            "          1.1771e-06, -6.5175e-06,  2.5424e-09, -6.1770e-06,  8.2700e-38,\n",
            "          6.1037e-07, -1.9055e-08,  8.9344e-06,  3.5401e-07, -4.2973e-06,\n",
            "          6.1253e-08,  6.5438e-07,  2.1687e-06,  8.6212e-07,  1.1121e-05,\n",
            "          9.7069e-06,  6.4005e-11,  2.0500e-06, -2.7122e-07, -7.4146e-06,\n",
            "         -2.0715e-07, -3.7276e-06,  1.5499e-23,  2.3194e-07,  9.9404e-07,\n",
            "          8.8894e-11,  8.8467e-19, -5.6052e-45, -2.3994e-19, -3.7290e-06,\n",
            "          1.0642e-05,  2.6813e-08,  1.4567e-08,  5.6052e-45,  1.4794e-05,\n",
            "          5.6052e-45,  1.6257e-05, -4.8710e-06, -3.3664e-07, -2.2257e-06,\n",
            "          4.2998e-14, -1.4325e-06,  6.8476e-08, -6.2828e-07,  1.0561e-26,\n",
            "          3.3767e-08,  1.0502e-06, -4.2892e-07,  6.8421e-06, -7.9871e-10,\n",
            "         -5.6052e-45,  6.5639e-07,  5.8695e-06,  2.7975e-06,  2.9124e-32,\n",
            "         -1.4097e-06, -9.0293e-06,  5.6334e-06,  1.5759e-07,  1.5705e-06,\n",
            "          5.6052e-45]], device='cuda:0'), 'exp_avg_sq': tensor([[2.0564e-08, 9.5689e-04, 1.9027e-05, 2.0620e-06, 3.9586e-08, 8.2411e-05,\n",
            "         2.6194e-09, 8.8574e-12, 1.6380e-03, 1.5191e-05, 5.1492e-04, 2.2084e-09,\n",
            "         2.5153e-06, 1.0350e-03, 4.5325e-05, 1.5228e-03, 1.9714e-08, 9.0862e-04,\n",
            "         4.3064e-04, 2.2715e-09, 3.0101e-04, 1.3427e-05, 1.1597e-03, 1.6216e-06,\n",
            "         4.3198e-05, 1.6626e-05, 8.2528e-04, 7.6085e-05, 5.3503e-04, 5.1190e-04,\n",
            "         1.1974e-03, 2.4969e-03, 2.0153e-05, 1.0580e-08, 1.7018e-04, 3.8593e-09,\n",
            "         1.5029e-05, 5.4735e-06, 8.4564e-10, 8.4965e-07, 2.8044e-04, 2.3429e-03,\n",
            "         7.4290e-04, 1.1636e-11, 3.2848e-04, 5.0211e-05, 1.5923e-04, 2.4523e-10,\n",
            "         1.7634e-06, 6.5616e-09, 3.8476e-05, 1.7198e-05, 4.6631e-04, 2.3174e-06,\n",
            "         1.2033e-03, 9.4070e-05, 1.4483e-04, 1.5844e-04, 5.3914e-05, 4.5596e-09,\n",
            "         1.9126e-06, 1.8856e-05, 1.8981e-04, 1.5215e-11, 3.5013e-05, 1.1583e-03,\n",
            "         5.4197e-04, 5.0745e-04, 8.7369e-05, 1.1780e-09, 3.0812e-05, 4.3173e-09,\n",
            "         3.8774e-04, 5.1447e-05, 4.8602e-08, 3.4474e-04, 4.2162e-05, 1.9210e-04,\n",
            "         4.8134e-05, 5.3172e-04, 2.3860e-04, 1.0387e-03, 3.4068e-04, 2.2950e-03,\n",
            "         1.4999e-06, 2.5330e-04, 5.9963e-04, 6.3672e-05, 2.5239e-05, 4.2861e-05,\n",
            "         1.7489e-05, 1.1119e-09, 1.8107e-10, 5.8322e-09, 1.5421e-03, 3.9769e-04,\n",
            "         3.7104e-10, 7.2391e-05, 5.2798e-08, 7.3906e-04, 3.8805e-03, 1.1472e-04,\n",
            "         7.5388e-09, 2.4570e-08, 4.4964e-05, 1.1698e-08, 2.4733e-05, 1.3709e-03,\n",
            "         2.7665e-05, 4.3139e-05, 9.4055e-10, 2.9562e-03, 5.2010e-05, 1.3105e-06,\n",
            "         6.4121e-05, 3.7502e-10, 5.3632e-06, 2.0638e-03, 9.7132e-07, 4.2057e-04,\n",
            "         1.4350e-09, 3.7811e-06, 1.9289e-09, 4.7837e-07, 4.3292e-03, 3.2596e-05,\n",
            "         2.2223e-07, 3.1298e-04, 1.5545e-03, 1.5367e-07, 2.2038e-05, 6.2850e-05,\n",
            "         7.2239e-05, 4.2688e-04, 2.0474e-10, 3.7933e-10, 4.0233e-06, 1.8452e-07,\n",
            "         2.7888e-04, 1.3928e-04, 5.3671e-05, 3.8001e-05, 6.2250e-04, 6.6687e-05,\n",
            "         2.3292e-05, 4.2988e-09, 1.3181e-03, 1.6464e-04, 2.1195e-05, 6.6068e-04,\n",
            "         4.6526e-05, 1.1437e-05, 3.1819e-09, 3.8372e-10, 2.3733e-06, 7.4544e-04,\n",
            "         6.6759e-13, 3.5340e-05, 5.6964e-04, 9.2722e-10, 2.4913e-08, 1.0954e-03,\n",
            "         2.4360e-04, 1.4798e-08, 6.5503e-10, 8.3302e-05, 9.1194e-06, 3.1814e-05,\n",
            "         1.1588e-03, 2.7801e-04, 7.5050e-13, 5.9678e-05, 7.5701e-06, 7.3361e-06,\n",
            "         6.3698e-10, 2.9855e-08, 8.5197e-07, 9.8900e-09, 2.1263e-05, 1.0815e-06,\n",
            "         4.8543e-09, 4.1901e-03, 1.9573e-03, 1.5241e-06, 1.3056e-05, 4.6050e-05,\n",
            "         6.1732e-05, 1.0279e-09, 1.4733e-03, 5.7847e-06, 8.2072e-04, 5.0610e-09,\n",
            "         1.0250e-03, 8.6415e-05, 2.2532e-03, 7.6523e-04, 4.5290e-05, 9.7882e-05,\n",
            "         6.5860e-05, 4.5352e-11, 1.3463e-06, 3.8976e-07, 1.6650e-03, 9.8983e-04,\n",
            "         3.2868e-04, 2.8032e-04, 3.5847e-07, 1.3465e-04, 1.9452e-05, 6.7373e-04,\n",
            "         2.9926e-03, 6.6029e-08, 2.1682e-05, 2.8422e-04, 4.9095e-05, 6.3672e-04,\n",
            "         8.0547e-10, 9.2969e-09, 2.6402e-04, 5.3494e-04, 9.7129e-07, 4.6188e-09,\n",
            "         2.9516e-10, 6.0926e-10, 9.8431e-06, 1.4513e-05, 3.0254e-06, 1.1305e-07,\n",
            "         8.2080e-11, 7.5525e-05, 1.0898e-09, 3.6903e-03, 9.4210e-05, 1.3767e-09,\n",
            "         1.3447e-07, 1.8258e-08, 6.9562e-05, 1.8380e-06, 2.2480e-04, 6.2654e-10,\n",
            "         6.2448e-05, 2.6624e-04, 1.4965e-06, 3.5803e-04, 8.3690e-09, 5.9005e-09,\n",
            "         1.2503e-04, 2.3371e-03, 1.1629e-04, 8.4688e-07, 1.9390e-07, 7.3117e-05,\n",
            "         1.9354e-03, 5.5616e-04, 5.3068e-05, 1.2262e-09],\n",
            "        [2.1083e-08, 9.5348e-04, 4.9460e-04, 1.9047e-05, 1.1126e-07, 1.6400e-04,\n",
            "         4.3983e-08, 8.4956e-11, 1.7825e-03, 1.2670e-04, 4.0965e-04, 6.4127e-09,\n",
            "         1.2897e-06, 1.0327e-03, 5.0334e-04, 1.4845e-03, 6.2038e-07, 6.4861e-04,\n",
            "         2.7313e-04, 2.8956e-08, 2.9919e-04, 8.1262e-06, 1.1573e-03, 1.2000e-05,\n",
            "         1.0046e-03, 8.0813e-06, 9.7427e-04, 2.0711e-04, 3.4541e-04, 4.7281e-04,\n",
            "         9.5619e-04, 2.1532e-03, 2.0888e-05, 4.9037e-08, 1.7229e-04, 2.7104e-08,\n",
            "         1.4966e-05, 7.8263e-06, 1.5597e-08, 7.9788e-07, 1.0090e-04, 2.2463e-03,\n",
            "         7.4316e-04, 3.4713e-13, 1.8054e-04, 5.9135e-05, 4.2344e-04, 2.7787e-09,\n",
            "         1.3135e-05, 6.6640e-09, 2.3853e-05, 1.7045e-05, 4.6552e-04, 8.6246e-06,\n",
            "         1.1239e-03, 1.3140e-04, 1.3790e-04, 1.4731e-04, 4.8975e-05, 4.0708e-07,\n",
            "         1.1705e-06, 6.2029e-04, 1.9273e-04, 3.1227e-12, 3.5749e-05, 1.0864e-03,\n",
            "         3.6385e-04, 5.2891e-04, 1.8237e-04, 1.2592e-09, 1.7726e-05, 4.8849e-09,\n",
            "         4.7338e-04, 5.7357e-04, 1.2726e-07, 1.3674e-04, 4.2493e-05, 1.0005e-04,\n",
            "         1.2189e-04, 5.2038e-04, 2.7338e-04, 8.7043e-04, 3.2142e-04, 2.2236e-03,\n",
            "         2.2046e-06, 2.7709e-04, 6.8764e-04, 1.0406e-03, 2.4040e-05, 2.6888e-04,\n",
            "         1.7683e-05, 1.6197e-09, 2.0506e-10, 1.1404e-09, 1.6622e-03, 3.7691e-04,\n",
            "         8.8593e-10, 6.7711e-05, 6.3411e-08, 7.3937e-04, 3.7258e-03, 8.1424e-05,\n",
            "         3.7609e-08, 2.7369e-08, 1.3260e-04, 1.1796e-08, 1.5735e-04, 1.1461e-03,\n",
            "         1.4894e-03, 3.0836e-05, 2.4474e-09, 2.9070e-03, 3.6717e-05, 1.3028e-04,\n",
            "         1.3843e-03, 2.6523e-10, 5.6156e-06, 1.8759e-03, 7.2868e-07, 4.2558e-04,\n",
            "         3.2027e-09, 1.5906e-04, 2.2412e-09, 3.9652e-07, 4.1273e-03, 8.1668e-04,\n",
            "         2.2552e-07, 2.2743e-04, 1.5431e-03, 4.2827e-06, 2.5620e-05, 2.1261e-04,\n",
            "         8.3190e-05, 3.9167e-04, 7.1636e-09, 2.0709e-09, 4.1637e-06, 2.3841e-07,\n",
            "         3.6004e-04, 1.3923e-04, 3.2660e-05, 1.1533e-03, 8.6175e-04, 4.5445e-05,\n",
            "         6.0285e-04, 4.5106e-09, 1.4037e-03, 4.9646e-04, 2.1733e-05, 4.4316e-04,\n",
            "         1.4865e-04, 1.6436e-05, 7.3752e-09, 8.1324e-11, 6.3640e-06, 5.3872e-04,\n",
            "         2.6645e-10, 5.7346e-05, 4.1532e-04, 1.2442e-09, 7.2634e-08, 8.1991e-04,\n",
            "         5.8334e-04, 3.2449e-06, 3.0689e-08, 8.1878e-05, 1.7820e-04, 3.3083e-05,\n",
            "         1.1054e-03, 2.7354e-04, 4.0864e-12, 5.4164e-05, 7.2561e-06, 3.7929e-04,\n",
            "         6.5771e-10, 4.0192e-08, 1.7279e-06, 3.4192e-07, 2.3933e-05, 4.9690e-07,\n",
            "         2.3613e-07, 4.1630e-03, 1.7574e-03, 1.7953e-06, 2.8606e-04, 6.6884e-05,\n",
            "         6.2977e-05, 7.7660e-08, 1.4371e-03, 6.4651e-06, 8.8207e-04, 1.8177e-08,\n",
            "         9.7341e-04, 3.2449e-05, 2.1811e-03, 6.3823e-04, 1.1374e-04, 8.9559e-05,\n",
            "         1.8941e-04, 1.1020e-09, 1.8645e-05, 1.8044e-06, 1.7328e-03, 9.8925e-04,\n",
            "         2.2656e-04, 2.8115e-04, 7.9182e-06, 2.9115e-04, 1.1972e-05, 8.0165e-04,\n",
            "         2.9223e-03, 8.2194e-08, 3.0346e-04, 1.9289e-04, 1.0086e-04, 5.3468e-04,\n",
            "         1.2146e-09, 1.0506e-08, 1.9112e-04, 4.8135e-04, 8.7191e-07, 4.3137e-09,\n",
            "         1.9167e-10, 5.1013e-08, 1.2401e-05, 1.5984e-04, 2.2073e-06, 1.1318e-07,\n",
            "         1.7688e-10, 1.5591e-03, 1.0520e-09, 3.8859e-03, 4.4371e-04, 3.0876e-09,\n",
            "         1.6879e-07, 3.0206e-08, 1.7017e-04, 1.8427e-06, 3.5089e-04, 8.6284e-10,\n",
            "         6.2627e-05, 3.8905e-04, 1.7959e-06, 5.4893e-04, 1.2339e-06, 5.9871e-08,\n",
            "         1.2019e-04, 2.2464e-03, 7.8987e-04, 1.0063e-06, 6.8999e-06, 1.9611e-04,\n",
            "         1.8104e-03, 5.6927e-04, 5.5413e-04, 1.1638e-09],\n",
            "        [6.0407e-11, 1.8660e-06, 5.0309e-04, 1.7219e-05, 1.0249e-07, 8.6351e-05,\n",
            "         4.4723e-08, 1.1316e-10, 1.5538e-04, 9.8826e-05, 6.3290e-05, 1.4619e-08,\n",
            "         5.4991e-07, 3.3297e-06, 5.5803e-04, 5.5841e-05, 5.9207e-07, 2.4534e-04,\n",
            "         2.7138e-04, 2.6609e-08, 8.4196e-06, 2.1396e-05, 4.2459e-06, 1.0844e-05,\n",
            "         1.0251e-03, 1.7066e-06, 1.3890e-04, 2.8683e-04, 1.4624e-04, 1.5714e-05,\n",
            "         1.7809e-04, 2.0127e-04, 1.1398e-06, 1.0147e-07, 1.2973e-06, 4.9536e-08,\n",
            "         2.0352e-09, 3.4348e-06, 1.8193e-08, 2.2515e-08, 2.6465e-04, 2.9158e-05,\n",
            "         3.9263e-06, 1.5823e-11, 1.3630e-04, 1.0173e-04, 2.6450e-04, 3.5917e-09,\n",
            "         1.4283e-05, 4.4022e-11, 6.1352e-05, 1.8254e-08, 1.1844e-06, 6.2706e-06,\n",
            "         9.7493e-05, 4.6568e-05, 7.9912e-06, 4.5869e-06, 5.3334e-06, 4.1472e-07,\n",
            "         1.1842e-06, 6.0542e-04, 1.8137e-06, 2.4931e-11, 3.4007e-07, 2.5305e-05,\n",
            "         1.9121e-04, 1.9757e-05, 2.6056e-04, 1.8752e-09, 9.3609e-06, 6.7431e-09,\n",
            "         6.0373e-04, 6.1885e-04, 8.1306e-08, 2.2512e-04, 5.5510e-07, 1.2135e-04,\n",
            "         7.0784e-05, 6.5129e-06, 4.6494e-05, 8.0232e-05, 1.7204e-05, 3.2157e-05,\n",
            "         8.3596e-07, 3.2367e-04, 7.0532e-05, 1.0992e-03, 6.4697e-07, 3.2079e-04,\n",
            "         1.7738e-07, 5.4157e-09, 2.9772e-11, 9.8696e-09, 1.3442e-04, 1.0408e-05,\n",
            "         2.3412e-09, 1.9749e-06, 1.1659e-08, 5.0806e-06, 6.7238e-05, 9.6834e-05,\n",
            "         2.2048e-08, 3.9000e-10, 1.6468e-04, 4.5303e-10, 1.3160e-04, 1.3658e-04,\n",
            "         1.4878e-03, 6.0265e-06, 6.4094e-09, 2.3743e-05, 9.2173e-05, 1.3126e-04,\n",
            "         1.4553e-03, 3.1201e-11, 6.5691e-08, 7.5753e-05, 1.1180e-07, 1.0673e-05,\n",
            "         8.3149e-09, 1.5335e-04, 7.7373e-09, 2.0589e-07, 6.9592e-05, 8.4716e-04,\n",
            "         1.1809e-09, 1.2073e-04, 6.3338e-06, 3.6619e-06, 3.4068e-06, 1.4835e-04,\n",
            "         9.2103e-06, 4.6631e-04, 7.2831e-09, 8.2617e-10, 1.3502e-07, 9.8678e-09,\n",
            "         7.4146e-05, 3.4402e-07, 1.0862e-05, 1.1538e-03, 2.1437e-04, 7.8278e-05,\n",
            "         6.1929e-04, 4.1381e-09, 9.2017e-05, 3.2168e-04, 1.8847e-07, 3.9889e-04,\n",
            "         1.1034e-04, 4.2630e-06, 1.1006e-08, 8.1782e-10, 5.7825e-06, 1.6232e-04,\n",
            "         2.6554e-10, 9.6720e-05, 1.6911e-04, 4.3545e-10, 4.2809e-08, 2.4102e-04,\n",
            "         3.4313e-04, 3.2561e-06, 3.0446e-08, 1.1330e-04, 1.6388e-04, 5.6356e-05,\n",
            "         2.4743e-05, 6.2148e-06, 1.3836e-12, 5.3020e-06, 5.4954e-06, 3.6479e-04,\n",
            "         2.5870e-09, 1.6858e-08, 2.9222e-06, 3.3724e-07, 5.5283e-06, 1.8642e-07,\n",
            "         2.4695e-07, 2.5243e-04, 1.1653e-04, 2.8468e-07, 2.8917e-04, 9.9886e-05,\n",
            "         1.1150e-06, 8.1423e-08, 1.3436e-05, 2.4828e-07, 6.8355e-05, 3.9898e-08,\n",
            "         1.8202e-05, 9.5609e-05, 2.2791e-05, 7.7612e-05, 1.6297e-04, 5.2551e-06,\n",
            "         2.5207e-04, 1.2576e-09, 1.8803e-05, 2.1870e-06, 9.6929e-05, 2.7980e-06,\n",
            "         1.7544e-04, 3.8863e-06, 7.6472e-06, 1.5451e-04, 6.0329e-06, 1.0640e-04,\n",
            "         1.0321e-04, 2.1553e-08, 3.2699e-04, 6.7489e-05, 1.5385e-04, 5.9080e-05,\n",
            "         3.7505e-09, 5.7037e-10, 5.5295e-05, 2.1253e-05, 1.0369e-07, 2.9688e-10,\n",
            "         7.6497e-10, 5.2850e-08, 2.2440e-05, 1.4061e-04, 1.8581e-06, 4.2336e-09,\n",
            "         4.9994e-10, 1.6125e-03, 4.2832e-09, 2.3468e-04, 5.4762e-04, 5.0418e-09,\n",
            "         1.9781e-08, 1.2127e-08, 1.0731e-04, 6.6998e-09, 1.6838e-04, 2.8148e-09,\n",
            "         1.1579e-07, 1.5959e-04, 5.1810e-07, 1.6169e-04, 1.2468e-06, 5.9095e-08,\n",
            "         2.1610e-05, 1.5289e-04, 9.0570e-04, 1.5933e-07, 6.5836e-06, 2.5824e-04,\n",
            "         7.9146e-05, 2.3815e-05, 6.1656e-04, 7.0927e-11]], device='cuda:0')}, 13: {'step': tensor(1920.), 'exp_avg': tensor([ 2.8580e-06, -3.6963e-06,  8.4277e-07], device='cuda:0'), 'exp_avg_sq': tensor([2.8075e-04, 2.8888e-04, 9.1226e-05], device='cuda:0')}}\n",
            "param_groups \n",
            "\n",
            " [{'lr': 0.001, 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0.01, 'amsgrad': False, 'foreach': None, 'maximize': False, 'capturable': False, 'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# file = \"AdamW-lr001-Batch64-15epochs.pth\"\n",
        "# torch.save(model.state_dict(),file)"
      ],
      "metadata": {
        "id": "n1xklJ8ni21-"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "file=\"AdamW-lr001-Batch64-15epochs.pth\"\n",
        "torch.save({\n",
        "            'epoch': epoch,\n",
        "            'model_state_dict': model.state_dict(),\n",
        "            'optimizer_state_dict': optimizer.state_dict(),\n",
        "            'loss': loss\n",
        "            }, file)"
      ],
      "metadata": {
        "id": "eZyb8IyVRXh7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "checkpoint = torch.load(file)"
      ],
      "metadata": {
        "id": "GajcRj2rpYKe"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model2 = CNNModel().to(device)\n",
        "optimizer2 = optim.Adam(model.parameters(),lr=0.001)"
      ],
      "metadata": {
        "id": "qhhx4HQ4p7z7"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# checkpoint = torch.load(file)\n",
        "# model2.load_state_dict(checkpoint['model_state_dict'])\n",
        "# optimizer2.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "# epoch = checkpoint[\"epoch\"]\n",
        "# loss = checkpoint['loss']"
      ],
      "metadata": {
        "id": "Fha0DoYOsihW"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Print model's state_dict\n",
        "# print(\"Model's state_dict:\")\n",
        "# for param_tensor in model2.state_dict():\n",
        "#     print(param_tensor, \"\\t\", model2.state_dict()[param_tensor].size())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2b7RjAKbszk9",
        "outputId": "588aa757-61e9-4db9-8e5d-43932a2c1187"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model's state_dict:\n",
            "conv_layer.0.weight \t torch.Size([16, 3, 3, 3])\n",
            "conv_layer.0.bias \t torch.Size([16])\n",
            "conv_layer.3.weight \t torch.Size([32, 16, 3, 3])\n",
            "conv_layer.3.bias \t torch.Size([32])\n",
            "conv_layer.6.weight \t torch.Size([64, 32, 3, 3])\n",
            "conv_layer.6.bias \t torch.Size([64])\n",
            "conv_layer.9.weight \t torch.Size([128, 64, 3, 3])\n",
            "conv_layer.9.bias \t torch.Size([128])\n",
            "fc_layer.0.weight \t torch.Size([512, 51200])\n",
            "fc_layer.0.bias \t torch.Size([512])\n",
            "fc_layer.2.weight \t torch.Size([256, 512])\n",
            "fc_layer.2.bias \t torch.Size([256])\n",
            "fc_layer.4.weight \t torch.Size([3, 256])\n",
            "fc_layer.4.bias \t torch.Size([3])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# Print optimizer's state_dict\n",
        "# print(\"Optimizer's state_dict:\\n\")\n",
        "# for var_name in optimizer2.state_dict():\n",
        "#     print(var_name, \"\\n\\n\", optimizer2.state_dict()[var_name])"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FFzp6du6s2Gy",
        "outputId": "117ee4b1-68da-43a8-e9eb-a67e9c9dcc91"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Optimizer's state_dict:\n",
            "\n",
            "state \n",
            "\n",
            " {0: {'step': tensor(1920.), 'exp_avg': tensor([[[[-7.3575e-06, -9.4751e-07,  4.2864e-06],\n",
            "          [-1.0329e-05, -2.5403e-06,  3.7023e-06],\n",
            "          [-1.4133e-05, -6.5502e-06,  6.0567e-07]],\n",
            "\n",
            "         [[-7.3575e-06, -9.4751e-07,  4.2864e-06],\n",
            "          [-1.0329e-05, -2.5403e-06,  3.7023e-06],\n",
            "          [-1.4133e-05, -6.5502e-06,  6.0567e-07]],\n",
            "\n",
            "         [[-7.3575e-06, -9.4751e-07,  4.2864e-06],\n",
            "          [-1.0329e-05, -2.5403e-06,  3.7023e-06],\n",
            "          [-1.4133e-05, -6.5502e-06,  6.0567e-07]]],\n",
            "\n",
            "\n",
            "        [[[-1.7765e-05, -6.4728e-06,  1.0907e-05],\n",
            "          [-4.4381e-05, -3.2131e-05, -9.9503e-06],\n",
            "          [-6.4358e-05, -5.5500e-05, -3.5019e-05]],\n",
            "\n",
            "         [[-1.7765e-05, -6.4728e-06,  1.0907e-05],\n",
            "          [-4.4381e-05, -3.2131e-05, -9.9503e-06],\n",
            "          [-6.4358e-05, -5.5500e-05, -3.5019e-05]],\n",
            "\n",
            "         [[-1.7765e-05, -6.4728e-06,  1.0907e-05],\n",
            "          [-4.4381e-05, -3.2131e-05, -9.9503e-06],\n",
            "          [-6.4358e-05, -5.5500e-05, -3.5019e-05]]],\n",
            "\n",
            "\n",
            "        [[[-9.6786e-06, -3.1666e-05, -4.2590e-05],\n",
            "          [-1.5561e-05, -3.7506e-05, -4.8657e-05],\n",
            "          [-1.7046e-05, -3.8464e-05, -5.0350e-05]],\n",
            "\n",
            "         [[-9.6786e-06, -3.1666e-05, -4.2590e-05],\n",
            "          [-1.5561e-05, -3.7506e-05, -4.8657e-05],\n",
            "          [-1.7046e-05, -3.8464e-05, -5.0350e-05]],\n",
            "\n",
            "         [[-9.6786e-06, -3.1666e-05, -4.2590e-05],\n",
            "          [-1.5561e-05, -3.7506e-05, -4.8657e-05],\n",
            "          [-1.7046e-05, -3.8464e-05, -5.0350e-05]]],\n",
            "\n",
            "\n",
            "        [[[-7.4091e-06, -7.2083e-06, -6.6748e-06],\n",
            "          [-7.2394e-06, -7.0180e-06, -6.6034e-06],\n",
            "          [-7.3697e-06, -6.8860e-06, -6.4471e-06]],\n",
            "\n",
            "         [[-7.4091e-06, -7.2083e-06, -6.6748e-06],\n",
            "          [-7.2394e-06, -7.0180e-06, -6.6034e-06],\n",
            "          [-7.3697e-06, -6.8860e-06, -6.4471e-06]],\n",
            "\n",
            "         [[-7.4091e-06, -7.2083e-06, -6.6748e-06],\n",
            "          [-7.2394e-06, -7.0180e-06, -6.6034e-06],\n",
            "          [-7.3697e-06, -6.8860e-06, -6.4471e-06]]],\n",
            "\n",
            "\n",
            "        [[[-1.3307e-05, -5.9616e-07,  7.4740e-06],\n",
            "          [-1.8834e-05, -4.8647e-06,  5.4331e-06],\n",
            "          [-2.3694e-05, -1.1098e-05,  3.3759e-07]],\n",
            "\n",
            "         [[-1.3307e-05, -5.9616e-07,  7.4740e-06],\n",
            "          [-1.8834e-05, -4.8647e-06,  5.4331e-06],\n",
            "          [-2.3694e-05, -1.1098e-05,  3.3759e-07]],\n",
            "\n",
            "         [[-1.3307e-05, -5.9616e-07,  7.4740e-06],\n",
            "          [-1.8834e-05, -4.8647e-06,  5.4331e-06],\n",
            "          [-2.3694e-05, -1.1098e-05,  3.3760e-07]]],\n",
            "\n",
            "\n",
            "        [[[-5.1930e-06, -1.3049e-05, -1.7228e-05],\n",
            "          [-4.2219e-06, -1.2655e-05, -1.7815e-05],\n",
            "          [-3.2043e-06, -1.1100e-05, -1.6851e-05]],\n",
            "\n",
            "         [[-5.1930e-06, -1.3049e-05, -1.7228e-05],\n",
            "          [-4.2219e-06, -1.2655e-05, -1.7815e-05],\n",
            "          [-3.2043e-06, -1.1100e-05, -1.6851e-05]],\n",
            "\n",
            "         [[-5.1930e-06, -1.3049e-05, -1.7228e-05],\n",
            "          [-4.2219e-06, -1.2655e-05, -1.7815e-05],\n",
            "          [-3.2043e-06, -1.1100e-05, -1.6851e-05]]],\n",
            "\n",
            "\n",
            "        [[[-2.2707e-05, -3.5192e-05, -4.2930e-05],\n",
            "          [-3.6344e-05, -4.7961e-05, -5.4880e-05],\n",
            "          [-4.5758e-05, -5.7619e-05, -6.3934e-05]],\n",
            "\n",
            "         [[-2.2707e-05, -3.5192e-05, -4.2930e-05],\n",
            "          [-3.6344e-05, -4.7961e-05, -5.4880e-05],\n",
            "          [-4.5758e-05, -5.7619e-05, -6.3934e-05]],\n",
            "\n",
            "         [[-2.2707e-05, -3.5192e-05, -4.2930e-05],\n",
            "          [-3.6344e-05, -4.7961e-05, -5.4880e-05],\n",
            "          [-4.5758e-05, -5.7619e-05, -6.3934e-05]]],\n",
            "\n",
            "\n",
            "        [[[-2.2550e-06,  8.3767e-07,  2.2526e-06],\n",
            "          [-3.8513e-06,  3.9980e-07,  2.4580e-06],\n",
            "          [-5.8235e-06, -7.0331e-07,  1.9745e-06]],\n",
            "\n",
            "         [[-2.2550e-06,  8.3767e-07,  2.2526e-06],\n",
            "          [-3.8513e-06,  3.9980e-07,  2.4580e-06],\n",
            "          [-5.8235e-06, -7.0331e-07,  1.9745e-06]],\n",
            "\n",
            "         [[-2.2550e-06,  8.3767e-07,  2.2526e-06],\n",
            "          [-3.8513e-06,  3.9980e-07,  2.4580e-06],\n",
            "          [-5.8235e-06, -7.0331e-07,  1.9745e-06]]],\n",
            "\n",
            "\n",
            "        [[[-1.6138e-05, -1.3283e-05, -6.4287e-06],\n",
            "          [-2.2631e-05, -2.0454e-05, -1.3634e-05],\n",
            "          [-2.3494e-05, -2.2588e-05, -1.7338e-05]],\n",
            "\n",
            "         [[-1.6138e-05, -1.3283e-05, -6.4287e-06],\n",
            "          [-2.2631e-05, -2.0454e-05, -1.3634e-05],\n",
            "          [-2.3494e-05, -2.2588e-05, -1.7338e-05]],\n",
            "\n",
            "         [[-1.6138e-05, -1.3283e-05, -6.4287e-06],\n",
            "          [-2.2631e-05, -2.0454e-05, -1.3634e-05],\n",
            "          [-2.3494e-05, -2.2588e-05, -1.7338e-05]]],\n",
            "\n",
            "\n",
            "        [[[ 2.7414e-07,  3.9062e-07,  3.2443e-07],\n",
            "          [ 6.0511e-07,  5.7305e-07,  4.5510e-07],\n",
            "          [ 1.6831e-06,  1.6789e-06,  1.7257e-06]],\n",
            "\n",
            "         [[ 2.7414e-07,  3.9062e-07,  3.2443e-07],\n",
            "          [ 6.0511e-07,  5.7305e-07,  4.5510e-07],\n",
            "          [ 1.6831e-06,  1.6789e-06,  1.7257e-06]],\n",
            "\n",
            "         [[ 2.7414e-07,  3.9062e-07,  3.2443e-07],\n",
            "          [ 6.0511e-07,  5.7305e-07,  4.5510e-07],\n",
            "          [ 1.6831e-06,  1.6789e-06,  1.7257e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 3.5252e-06,  5.6884e-06,  5.8915e-06],\n",
            "          [ 2.4851e-06,  5.5339e-06,  6.2564e-06],\n",
            "          [ 7.8699e-07,  4.4850e-06,  5.8503e-06]],\n",
            "\n",
            "         [[ 3.5252e-06,  5.6884e-06,  5.8915e-06],\n",
            "          [ 2.4851e-06,  5.5339e-06,  6.2564e-06],\n",
            "          [ 7.8699e-07,  4.4850e-06,  5.8503e-06]],\n",
            "\n",
            "         [[ 3.5252e-06,  5.6884e-06,  5.8915e-06],\n",
            "          [ 2.4851e-06,  5.5339e-06,  6.2564e-06],\n",
            "          [ 7.8699e-07,  4.4850e-06,  5.8503e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 5.6643e-07,  4.7586e-06,  5.3443e-06],\n",
            "          [-1.1165e-06,  3.5008e-06,  5.1984e-06],\n",
            "          [-1.3272e-06,  3.4924e-06,  5.6492e-06]],\n",
            "\n",
            "         [[ 5.6643e-07,  4.7586e-06,  5.3443e-06],\n",
            "          [-1.1165e-06,  3.5008e-06,  5.1984e-06],\n",
            "          [-1.3272e-06,  3.4924e-06,  5.6492e-06]],\n",
            "\n",
            "         [[ 5.6643e-07,  4.7586e-06,  5.3443e-06],\n",
            "          [-1.1165e-06,  3.5008e-06,  5.1984e-06],\n",
            "          [-1.3272e-06,  3.4924e-06,  5.6492e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 1.1487e-06, -1.0750e-06, -1.3223e-06],\n",
            "          [ 1.0746e-06, -1.7661e-06, -1.8995e-06],\n",
            "          [ 8.8249e-07, -1.7489e-06, -3.4432e-06]],\n",
            "\n",
            "         [[ 1.1487e-06, -1.0750e-06, -1.3223e-06],\n",
            "          [ 1.0746e-06, -1.7661e-06, -1.8995e-06],\n",
            "          [ 8.8249e-07, -1.7489e-06, -3.4432e-06]],\n",
            "\n",
            "         [[ 1.1487e-06, -1.0750e-06, -1.3223e-06],\n",
            "          [ 1.0746e-06, -1.7661e-06, -1.8995e-06],\n",
            "          [ 8.8249e-07, -1.7489e-06, -3.4432e-06]]],\n",
            "\n",
            "\n",
            "        [[[ 2.4483e-06,  2.4874e-06,  3.7010e-06],\n",
            "          [ 3.4584e-06,  3.1332e-06,  3.8119e-06],\n",
            "          [ 4.7700e-06,  4.3336e-06,  4.5659e-06]],\n",
            "\n",
            "         [[ 2.4483e-06,  2.4874e-06,  3.7010e-06],\n",
            "          [ 3.4584e-06,  3.1332e-06,  3.8119e-06],\n",
            "          [ 4.7700e-06,  4.3336e-06,  4.5659e-06]],\n",
            "\n",
            "         [[ 2.4483e-06,  2.4874e-06,  3.7010e-06],\n",
            "          [ 3.4584e-06,  3.1332e-06,  3.8119e-06],\n",
            "          [ 4.7700e-06,  4.3336e-06,  4.5659e-06]]],\n",
            "\n",
            "\n",
            "        [[[-3.4200e-05, -3.6484e-05, -3.6151e-05],\n",
            "          [-3.1965e-05, -3.4976e-05, -3.4968e-05],\n",
            "          [-2.7874e-05, -3.1459e-05, -3.2560e-05]],\n",
            "\n",
            "         [[-3.4200e-05, -3.6484e-05, -3.6151e-05],\n",
            "          [-3.1965e-05, -3.4976e-05, -3.4968e-05],\n",
            "          [-2.7874e-05, -3.1459e-05, -3.2560e-05]],\n",
            "\n",
            "         [[-3.4200e-05, -3.6484e-05, -3.6151e-05],\n",
            "          [-3.1965e-05, -3.4976e-05, -3.4968e-05],\n",
            "          [-2.7874e-05, -3.1459e-05, -3.2560e-05]]],\n",
            "\n",
            "\n",
            "        [[[-3.2314e-06,  1.2041e-06,  6.4791e-06],\n",
            "          [-6.0680e-06, -1.7523e-06,  4.0849e-06],\n",
            "          [-5.9618e-06, -2.4209e-06,  2.7963e-06]],\n",
            "\n",
            "         [[-3.2314e-06,  1.2041e-06,  6.4791e-06],\n",
            "          [-6.0680e-06, -1.7523e-06,  4.0849e-06],\n",
            "          [-5.9618e-06, -2.4209e-06,  2.7963e-06]],\n",
            "\n",
            "         [[-3.2314e-06,  1.2041e-06,  6.4791e-06],\n",
            "          [-6.0680e-06, -1.7523e-06,  4.0849e-06],\n",
            "          [-5.9618e-06, -2.4209e-06,  2.7963e-06]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[6.9233e-04, 5.8245e-04, 5.6565e-04],\n",
            "          [7.5622e-04, 6.0840e-04, 5.7777e-04],\n",
            "          [7.3378e-04, 5.8325e-04, 5.5130e-04]],\n",
            "\n",
            "         [[6.9233e-04, 5.8245e-04, 5.6565e-04],\n",
            "          [7.5622e-04, 6.0840e-04, 5.7777e-04],\n",
            "          [7.3378e-04, 5.8325e-04, 5.5130e-04]],\n",
            "\n",
            "         [[6.9233e-04, 5.8245e-04, 5.6565e-04],\n",
            "          [7.5622e-04, 6.0840e-04, 5.7777e-04],\n",
            "          [7.3378e-04, 5.8325e-04, 5.5130e-04]]],\n",
            "\n",
            "\n",
            "        [[[3.6994e-03, 3.5468e-03, 3.5377e-03],\n",
            "          [4.1529e-03, 3.8985e-03, 3.7753e-03],\n",
            "          [4.4276e-03, 4.2201e-03, 3.9625e-03]],\n",
            "\n",
            "         [[3.6994e-03, 3.5468e-03, 3.5377e-03],\n",
            "          [4.1529e-03, 3.8985e-03, 3.7753e-03],\n",
            "          [4.4276e-03, 4.2201e-03, 3.9625e-03]],\n",
            "\n",
            "         [[3.6994e-03, 3.5468e-03, 3.5377e-03],\n",
            "          [4.1529e-03, 3.8985e-03, 3.7753e-03],\n",
            "          [4.4276e-03, 4.2201e-03, 3.9625e-03]]],\n",
            "\n",
            "\n",
            "        [[[4.1732e-03, 4.6608e-03, 5.3351e-03],\n",
            "          [4.4604e-03, 5.1009e-03, 5.9612e-03],\n",
            "          [4.5377e-03, 5.1495e-03, 5.9561e-03]],\n",
            "\n",
            "         [[4.1732e-03, 4.6608e-03, 5.3351e-03],\n",
            "          [4.4604e-03, 5.1009e-03, 5.9612e-03],\n",
            "          [4.5377e-03, 5.1495e-03, 5.9561e-03]],\n",
            "\n",
            "         [[4.1732e-03, 4.6608e-03, 5.3351e-03],\n",
            "          [4.4604e-03, 5.1009e-03, 5.9612e-03],\n",
            "          [4.5377e-03, 5.1495e-03, 5.9561e-03]]],\n",
            "\n",
            "\n",
            "        [[[6.7822e-05, 6.7819e-05, 6.6989e-05],\n",
            "          [6.7060e-05, 6.8859e-05, 6.9238e-05],\n",
            "          [6.3543e-05, 6.5032e-05, 6.6009e-05]],\n",
            "\n",
            "         [[6.7822e-05, 6.7819e-05, 6.6989e-05],\n",
            "          [6.7060e-05, 6.8859e-05, 6.9238e-05],\n",
            "          [6.3543e-05, 6.5032e-05, 6.6009e-05]],\n",
            "\n",
            "         [[6.7822e-05, 6.7819e-05, 6.6989e-05],\n",
            "          [6.7060e-05, 6.8859e-05, 6.9238e-05],\n",
            "          [6.3543e-05, 6.5032e-05, 6.6009e-05]]],\n",
            "\n",
            "\n",
            "        [[[1.0434e-03, 8.8144e-04, 9.1152e-04],\n",
            "          [1.1830e-03, 9.2586e-04, 9.0460e-04],\n",
            "          [1.1952e-03, 9.3165e-04, 8.8450e-04]],\n",
            "\n",
            "         [[1.0434e-03, 8.8144e-04, 9.1152e-04],\n",
            "          [1.1830e-03, 9.2586e-04, 9.0460e-04],\n",
            "          [1.1952e-03, 9.3165e-04, 8.8450e-04]],\n",
            "\n",
            "         [[1.0434e-03, 8.8144e-04, 9.1152e-04],\n",
            "          [1.1830e-03, 9.2586e-04, 9.0460e-04],\n",
            "          [1.1952e-03, 9.3165e-04, 8.8450e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.6098e-04, 3.0716e-04, 3.7253e-04],\n",
            "          [2.5475e-04, 3.0449e-04, 4.0372e-04],\n",
            "          [2.4034e-04, 2.9924e-04, 4.2411e-04]],\n",
            "\n",
            "         [[2.6098e-04, 3.0716e-04, 3.7253e-04],\n",
            "          [2.5475e-04, 3.0449e-04, 4.0372e-04],\n",
            "          [2.4034e-04, 2.9924e-04, 4.2411e-04]],\n",
            "\n",
            "         [[2.6098e-04, 3.0716e-04, 3.7253e-04],\n",
            "          [2.5475e-04, 3.0449e-04, 4.0372e-04],\n",
            "          [2.4034e-04, 2.9924e-04, 4.2411e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.3940e-03, 2.5708e-03, 2.8942e-03],\n",
            "          [2.6388e-03, 2.9595e-03, 3.2565e-03],\n",
            "          [2.7508e-03, 3.0936e-03, 3.2475e-03]],\n",
            "\n",
            "         [[2.3940e-03, 2.5708e-03, 2.8942e-03],\n",
            "          [2.6388e-03, 2.9595e-03, 3.2565e-03],\n",
            "          [2.7508e-03, 3.0936e-03, 3.2475e-03]],\n",
            "\n",
            "         [[2.3940e-03, 2.5708e-03, 2.8942e-03],\n",
            "          [2.6388e-03, 2.9595e-03, 3.2565e-03],\n",
            "          [2.7508e-03, 3.0936e-03, 3.2475e-03]]],\n",
            "\n",
            "\n",
            "        [[[1.1810e-04, 1.1746e-04, 1.2422e-04],\n",
            "          [1.2500e-04, 1.1630e-04, 1.2148e-04],\n",
            "          [1.2925e-04, 1.1410e-04, 1.1756e-04]],\n",
            "\n",
            "         [[1.1810e-04, 1.1746e-04, 1.2422e-04],\n",
            "          [1.2500e-04, 1.1630e-04, 1.2148e-04],\n",
            "          [1.2925e-04, 1.1410e-04, 1.1756e-04]],\n",
            "\n",
            "         [[1.1810e-04, 1.1746e-04, 1.2422e-04],\n",
            "          [1.2500e-04, 1.1630e-04, 1.2148e-04],\n",
            "          [1.2925e-04, 1.1410e-04, 1.1756e-04]]],\n",
            "\n",
            "\n",
            "        [[[1.6569e-03, 1.6560e-03, 1.6334e-03],\n",
            "          [1.9106e-03, 1.9243e-03, 1.8827e-03],\n",
            "          [1.9478e-03, 1.9783e-03, 1.9187e-03]],\n",
            "\n",
            "         [[1.6569e-03, 1.6560e-03, 1.6334e-03],\n",
            "          [1.9106e-03, 1.9243e-03, 1.8827e-03],\n",
            "          [1.9478e-03, 1.9783e-03, 1.9187e-03]],\n",
            "\n",
            "         [[1.6569e-03, 1.6560e-03, 1.6334e-03],\n",
            "          [1.9106e-03, 1.9243e-03, 1.8827e-03],\n",
            "          [1.9478e-03, 1.9783e-03, 1.9187e-03]]],\n",
            "\n",
            "\n",
            "        [[[6.9400e-05, 7.0356e-05, 7.0680e-05],\n",
            "          [7.8981e-05, 8.0176e-05, 8.0233e-05],\n",
            "          [8.3490e-05, 8.5053e-05, 8.4979e-05]],\n",
            "\n",
            "         [[6.9400e-05, 7.0356e-05, 7.0680e-05],\n",
            "          [7.8981e-05, 8.0176e-05, 8.0233e-05],\n",
            "          [8.3490e-05, 8.5053e-05, 8.4979e-05]],\n",
            "\n",
            "         [[6.9400e-05, 7.0356e-05, 7.0680e-05],\n",
            "          [7.8981e-05, 8.0176e-05, 8.0233e-05],\n",
            "          [8.3490e-05, 8.5053e-05, 8.4979e-05]]],\n",
            "\n",
            "\n",
            "        [[[2.8920e-04, 2.9655e-04, 2.9166e-04],\n",
            "          [2.8871e-04, 3.0215e-04, 3.0035e-04],\n",
            "          [2.6910e-04, 2.8434e-04, 2.9013e-04]],\n",
            "\n",
            "         [[2.8920e-04, 2.9655e-04, 2.9166e-04],\n",
            "          [2.8871e-04, 3.0215e-04, 3.0035e-04],\n",
            "          [2.6910e-04, 2.8434e-04, 2.9013e-04]],\n",
            "\n",
            "         [[2.8920e-04, 2.9655e-04, 2.9166e-04],\n",
            "          [2.8871e-04, 3.0215e-04, 3.0035e-04],\n",
            "          [2.6910e-04, 2.8434e-04, 2.9013e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.7688e-05, 2.6120e-05, 3.3637e-05],\n",
            "          [3.0652e-05, 2.5277e-05, 3.5234e-05],\n",
            "          [3.2862e-05, 2.6018e-05, 3.5418e-05]],\n",
            "\n",
            "         [[2.7688e-05, 2.6120e-05, 3.3637e-05],\n",
            "          [3.0652e-05, 2.5277e-05, 3.5234e-05],\n",
            "          [3.2862e-05, 2.6018e-05, 3.5418e-05]],\n",
            "\n",
            "         [[2.7688e-05, 2.6120e-05, 3.3637e-05],\n",
            "          [3.0652e-05, 2.5277e-05, 3.5234e-05],\n",
            "          [3.2862e-05, 2.6018e-05, 3.5418e-05]]],\n",
            "\n",
            "\n",
            "        [[[2.2384e-04, 1.8698e-04, 1.6375e-04],\n",
            "          [1.8729e-04, 1.5506e-04, 1.4727e-04],\n",
            "          [1.5879e-04, 1.5340e-04, 1.5883e-04]],\n",
            "\n",
            "         [[2.2384e-04, 1.8698e-04, 1.6375e-04],\n",
            "          [1.8729e-04, 1.5506e-04, 1.4727e-04],\n",
            "          [1.5879e-04, 1.5340e-04, 1.5883e-04]],\n",
            "\n",
            "         [[2.2384e-04, 1.8698e-04, 1.6375e-04],\n",
            "          [1.8729e-04, 1.5506e-04, 1.4727e-04],\n",
            "          [1.5879e-04, 1.5340e-04, 1.5883e-04]]],\n",
            "\n",
            "\n",
            "        [[[4.3867e-04, 4.4752e-04, 4.4704e-04],\n",
            "          [4.4700e-04, 4.5756e-04, 4.5253e-04],\n",
            "          [4.5103e-04, 4.5647e-04, 4.4832e-04]],\n",
            "\n",
            "         [[4.3867e-04, 4.4752e-04, 4.4704e-04],\n",
            "          [4.4700e-04, 4.5756e-04, 4.5253e-04],\n",
            "          [4.5103e-04, 4.5647e-04, 4.4832e-04]],\n",
            "\n",
            "         [[4.3867e-04, 4.4752e-04, 4.4704e-04],\n",
            "          [4.4700e-04, 4.5756e-04, 4.5253e-04],\n",
            "          [4.5103e-04, 4.5647e-04, 4.4832e-04]]],\n",
            "\n",
            "\n",
            "        [[[5.0045e-04, 5.4612e-04, 5.4447e-04],\n",
            "          [4.9700e-04, 5.4348e-04, 5.3879e-04],\n",
            "          [4.7367e-04, 5.1453e-04, 5.1433e-04]],\n",
            "\n",
            "         [[5.0045e-04, 5.4612e-04, 5.4447e-04],\n",
            "          [4.9700e-04, 5.4348e-04, 5.3879e-04],\n",
            "          [4.7367e-04, 5.1453e-04, 5.1433e-04]],\n",
            "\n",
            "         [[5.0045e-04, 5.4612e-04, 5.4447e-04],\n",
            "          [4.9700e-04, 5.4348e-04, 5.3879e-04],\n",
            "          [4.7367e-04, 5.1453e-04, 5.1433e-04]]],\n",
            "\n",
            "\n",
            "        [[[2.8639e-04, 2.5310e-04, 2.5728e-04],\n",
            "          [3.0973e-04, 2.6474e-04, 2.5774e-04],\n",
            "          [3.2370e-04, 2.7704e-04, 2.5908e-04]],\n",
            "\n",
            "         [[2.8639e-04, 2.5310e-04, 2.5728e-04],\n",
            "          [3.0973e-04, 2.6474e-04, 2.5774e-04],\n",
            "          [3.2370e-04, 2.7704e-04, 2.5908e-04]],\n",
            "\n",
            "         [[2.8639e-04, 2.5310e-04, 2.5728e-04],\n",
            "          [3.0973e-04, 2.6474e-04, 2.5774e-04],\n",
            "          [3.2370e-04, 2.7704e-04, 2.5908e-04]]]], device='cuda:0')}, 1: {'step': tensor(1920.), 'exp_avg': tensor([-9.7154e-06, -2.0288e-04, -7.9073e-05,  2.0579e-05, -6.1251e-05,\n",
            "        -1.0900e-05, -7.2585e-05, -6.8127e-06, -3.1924e-05, -2.7055e-06,\n",
            "        -3.3172e-05,  1.1339e-05,  7.9015e-07,  2.7123e-05,  1.6642e-04,\n",
            "        -9.2228e-06], device='cuda:0'), 'exp_avg_sq': tensor([5.7016e-03, 2.9535e-02, 1.6589e-02, 2.3251e-04, 5.9860e-03, 1.7322e-03,\n",
            "        1.2739e-02, 7.6791e-04, 6.9435e-03, 1.6062e-04, 4.6477e-03, 9.2923e-05,\n",
            "        6.1107e-04, 3.2373e-03, 5.5482e-03, 2.0157e-03], device='cuda:0')}, 2: {'step': tensor(1920.), 'exp_avg': tensor([[[[-4.3435e-06, -4.0039e-06, -1.9901e-06],\n",
            "          [-3.1395e-06, -3.8935e-06, -2.0991e-06],\n",
            "          [-1.8131e-06, -2.8209e-06, -1.3387e-06]],\n",
            "\n",
            "         [[-3.4029e-06, -2.7497e-06, -1.3980e-06],\n",
            "          [-4.8392e-06, -4.7800e-06, -3.5608e-06],\n",
            "          [-4.1998e-06, -5.3759e-06, -5.1059e-06]],\n",
            "\n",
            "         [[-5.5797e-07,  1.1877e-07,  5.0290e-07],\n",
            "          [-8.2814e-07,  1.1665e-07, -8.3020e-08],\n",
            "          [-1.5945e-06, -2.6390e-07,  1.1582e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-4.1854e-07, -1.7174e-07, -2.0854e-07],\n",
            "          [-2.1166e-06, -8.8267e-07, -4.2267e-07],\n",
            "          [-5.8705e-06, -4.3641e-06, -3.2761e-06]],\n",
            "\n",
            "         [[-3.4221e-06, -5.7409e-06, -5.3924e-06],\n",
            "          [-1.1999e-06, -4.0971e-06, -3.8425e-06],\n",
            "          [ 1.1243e-07, -1.0883e-06, -1.6002e-06]],\n",
            "\n",
            "         [[-3.2838e-06, -2.8302e-06, -1.3931e-06],\n",
            "          [-3.5091e-06, -3.3278e-06, -1.4386e-06],\n",
            "          [-3.0121e-06, -3.4070e-06, -2.0103e-06]]],\n",
            "\n",
            "\n",
            "        [[[-8.4926e-08, -2.3520e-07, -2.6415e-07],\n",
            "          [-2.5811e-07, -3.8437e-07, -4.5825e-07],\n",
            "          [-1.4653e-07, -2.9087e-07, -2.9993e-07]],\n",
            "\n",
            "         [[-5.2334e-08, -6.4037e-08, -2.5886e-09],\n",
            "          [-2.4205e-07, -1.8654e-07, -1.7894e-07],\n",
            "          [-3.3824e-07, -4.4560e-07, -4.3719e-07]],\n",
            "\n",
            "         [[ 1.0322e-08,  7.3865e-09, -1.5297e-08],\n",
            "          [-5.2055e-09,  1.1233e-08,  9.3980e-09],\n",
            "          [ 3.0144e-09,  1.2052e-08,  1.5149e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.8393e-06, -1.9252e-06, -2.0536e-06],\n",
            "          [-1.9648e-06, -1.9995e-06, -1.9772e-06],\n",
            "          [-2.1253e-06, -2.0437e-06, -2.0800e-06]],\n",
            "\n",
            "         [[-6.2839e-07, -6.5196e-07, -7.5317e-07],\n",
            "          [-6.4679e-07, -6.9613e-07, -8.2352e-07],\n",
            "          [-3.6406e-07, -4.3342e-07, -5.7060e-07]],\n",
            "\n",
            "         [[-5.6393e-07, -7.8381e-07, -8.1618e-07],\n",
            "          [-6.1319e-07, -8.1886e-07, -8.2973e-07],\n",
            "          [-6.5285e-07, -8.0454e-07, -7.8428e-07]]],\n",
            "\n",
            "\n",
            "        [[[-3.3833e-07, -5.4543e-07, -3.4149e-07],\n",
            "          [-1.9021e-07, -5.7855e-07, -3.8786e-07],\n",
            "          [ 1.8509e-07, -2.3147e-07, -3.5918e-08]],\n",
            "\n",
            "         [[-4.9805e-07, -4.9689e-07, -2.6734e-07],\n",
            "          [ 1.0092e-08,  2.8648e-08,  1.4603e-07],\n",
            "          [ 3.0478e-07,  1.6285e-07,  1.8456e-07]],\n",
            "\n",
            "         [[-1.2853e-06, -1.1739e-06, -9.1186e-07],\n",
            "          [-1.2292e-06, -1.0579e-06, -6.7037e-07],\n",
            "          [-1.1982e-06, -8.7880e-07, -6.3996e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-2.6475e-06, -3.0231e-06, -2.6339e-06],\n",
            "          [-2.3839e-06, -2.5956e-06, -2.0190e-06],\n",
            "          [-1.9728e-06, -2.2067e-06, -1.5277e-06]],\n",
            "\n",
            "         [[ 1.7683e-07,  1.4138e-07,  8.9545e-08],\n",
            "          [ 3.0693e-07,  2.4007e-07,  1.1634e-07],\n",
            "          [ 5.7590e-07,  6.8124e-07,  4.8484e-07]],\n",
            "\n",
            "         [[-4.3380e-07, -7.5987e-07, -7.6618e-07],\n",
            "          [-4.3821e-07, -8.3840e-07, -7.4160e-07],\n",
            "          [-3.5936e-07, -7.3961e-07, -5.9115e-07]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[ 1.0481e-07, -1.0513e-06, -2.8816e-06],\n",
            "          [ 5.8942e-07, -1.1755e-06, -2.7485e-06],\n",
            "          [-9.9040e-08, -1.4182e-06, -3.4497e-06]],\n",
            "\n",
            "         [[-2.8382e-07, -1.2709e-07,  1.2721e-07],\n",
            "          [-7.3051e-07, -7.2296e-07, -5.2357e-07],\n",
            "          [-1.2158e-07, -6.3448e-07, -9.8232e-07]],\n",
            "\n",
            "         [[-1.6734e-07, -1.5104e-06, -2.4915e-07],\n",
            "          [-6.2312e-07, -1.1420e-06, -2.1440e-07],\n",
            "          [-9.3473e-07, -1.1216e-06, -1.0077e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.2239e-06, -1.6143e-06, -1.1894e-06],\n",
            "          [-2.4977e-06, -2.6029e-06, -1.3724e-06],\n",
            "          [-2.5648e-06, -2.6680e-06, -1.3775e-06]],\n",
            "\n",
            "         [[ 2.6474e-06,  2.0968e-06,  8.8308e-08],\n",
            "          [ 2.7900e-06,  2.1766e-06, -1.0840e-07],\n",
            "          [ 1.9674e-06,  1.3813e-06, -1.0300e-06]],\n",
            "\n",
            "         [[ 2.2454e-07, -7.6989e-07, -1.8412e-06],\n",
            "          [-2.8005e-07, -1.1206e-06, -2.4323e-06],\n",
            "          [-2.0279e-07, -1.3359e-06, -2.5883e-06]]],\n",
            "\n",
            "\n",
            "        [[[-3.8835e-07, -4.8238e-07, -5.0633e-07],\n",
            "          [-5.4608e-07, -6.5752e-07, -4.5918e-07],\n",
            "          [-1.6925e-07, -3.9736e-07, -2.5859e-07]],\n",
            "\n",
            "         [[-1.3299e-06, -1.5035e-06, -2.0737e-06],\n",
            "          [-1.1050e-06, -1.1473e-06, -1.5054e-06],\n",
            "          [ 9.3300e-09,  6.3746e-08, -1.0616e-08]],\n",
            "\n",
            "         [[-4.2777e-07, -3.6246e-06, -2.3481e-06],\n",
            "          [-1.2486e-06, -4.9658e-06, -3.7074e-06],\n",
            "          [-1.2227e-06, -4.7366e-06, -3.3306e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-5.3059e-06, -7.7284e-06, -8.8004e-06],\n",
            "          [-6.8782e-06, -9.7401e-06, -1.1075e-05],\n",
            "          [-9.0978e-06, -1.2356e-05, -1.3728e-05]],\n",
            "\n",
            "         [[-6.1530e-07, -2.2803e-07, -1.6233e-07],\n",
            "          [-4.7878e-07, -1.0911e-07, -2.9810e-08],\n",
            "          [-3.4518e-07, -3.2235e-08, -1.4502e-08]],\n",
            "\n",
            "         [[-1.3107e-06, -1.6369e-06, -1.9103e-06],\n",
            "          [-8.7137e-07, -1.3883e-06, -1.6281e-06],\n",
            "          [-1.5764e-06, -2.0251e-06, -2.3442e-06]]],\n",
            "\n",
            "\n",
            "        [[[-2.0387e-07, -1.4503e-07, -2.3428e-07],\n",
            "          [-1.2098e-07,  2.3235e-08,  1.5016e-07],\n",
            "          [ 7.8501e-08,  1.6835e-07,  4.7619e-07]],\n",
            "\n",
            "         [[ 3.8793e-07,  4.2616e-07,  4.7380e-07],\n",
            "          [ 6.7335e-08,  2.3715e-07,  4.2699e-07],\n",
            "          [-5.1355e-08, -6.8583e-08,  1.9241e-07]],\n",
            "\n",
            "         [[-8.6834e-08,  1.5704e-07, -1.1895e-07],\n",
            "          [-3.9044e-08,  1.6301e-07,  8.5924e-08],\n",
            "          [-9.1763e-08,  2.0396e-07,  1.0518e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-9.1751e-08, -8.1914e-08, -1.0036e-07],\n",
            "          [-1.3705e-09, -3.1394e-08, -1.6460e-08],\n",
            "          [ 5.7716e-09, -8.1750e-08,  9.7859e-10]],\n",
            "\n",
            "         [[ 1.3408e-06,  1.4776e-06,  1.1085e-06],\n",
            "          [ 8.6693e-07,  8.3343e-07,  4.6222e-07],\n",
            "          [ 1.2823e-06,  9.4316e-07,  4.7014e-07]],\n",
            "\n",
            "         [[-1.6738e-07, -2.3623e-08, -1.5730e-07],\n",
            "          [-1.1287e-07,  2.4444e-09,  2.3175e-08],\n",
            "          [-1.0246e-07,  3.8064e-08,  1.7391e-07]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[1.0138e-05, 9.6506e-06, 5.0039e-06],\n",
            "          [8.2859e-06, 9.9527e-06, 5.8054e-06],\n",
            "          [5.6481e-06, 8.1714e-06, 5.8634e-06]],\n",
            "\n",
            "         [[3.5795e-06, 2.5180e-06, 1.8674e-06],\n",
            "          [7.1403e-06, 7.5306e-06, 6.3593e-06],\n",
            "          [4.8481e-06, 6.2643e-06, 6.1142e-06]],\n",
            "\n",
            "         [[4.0954e-07, 2.0774e-07, 2.0646e-06],\n",
            "          [7.7152e-07, 4.6130e-07, 2.1987e-06],\n",
            "          [1.3477e-06, 5.9053e-07, 1.7341e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[4.1998e-06, 2.1624e-06, 2.4349e-06],\n",
            "          [6.8380e-06, 4.6303e-06, 4.3965e-06],\n",
            "          [1.3513e-05, 1.0773e-05, 9.0189e-06]],\n",
            "\n",
            "         [[1.0981e-04, 1.2335e-04, 1.2478e-04],\n",
            "          [8.8024e-05, 1.0315e-04, 1.0832e-04],\n",
            "          [7.1975e-05, 8.3215e-05, 8.9918e-05]],\n",
            "\n",
            "         [[1.4426e-05, 1.2381e-05, 8.4951e-06],\n",
            "          [1.3568e-05, 1.2705e-05, 8.4219e-06],\n",
            "          [1.2017e-05, 1.3272e-05, 1.0497e-05]]],\n",
            "\n",
            "\n",
            "        [[[1.1617e-06, 1.2420e-06, 1.3282e-06],\n",
            "          [1.8101e-06, 1.6211e-06, 1.6455e-06],\n",
            "          [1.4124e-06, 1.1532e-06, 1.3458e-06]],\n",
            "\n",
            "         [[4.5476e-07, 5.1739e-07, 5.5399e-07],\n",
            "          [5.1504e-07, 5.1795e-07, 5.7107e-07],\n",
            "          [7.8314e-07, 8.1396e-07, 8.3315e-07]],\n",
            "\n",
            "         [[5.4752e-08, 5.1368e-08, 9.6474e-08],\n",
            "          [4.2240e-08, 5.3443e-08, 1.1430e-07],\n",
            "          [5.8875e-08, 7.4543e-08, 1.2304e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[3.9679e-05, 3.9987e-05, 3.9340e-05],\n",
            "          [3.8708e-05, 3.9367e-05, 3.9960e-05],\n",
            "          [3.8698e-05, 3.9022e-05, 3.9881e-05]],\n",
            "\n",
            "         [[2.4210e-05, 2.4568e-05, 2.4886e-05],\n",
            "          [2.6093e-05, 2.6351e-05, 2.6452e-05],\n",
            "          [2.5065e-05, 2.5396e-05, 2.5318e-05]],\n",
            "\n",
            "         [[5.8268e-06, 6.5398e-06, 6.4287e-06],\n",
            "          [5.9338e-06, 6.2966e-06, 6.2366e-06],\n",
            "          [5.9348e-06, 6.0778e-06, 6.1268e-06]]],\n",
            "\n",
            "\n",
            "        [[[1.1763e-06, 1.7764e-06, 1.2860e-06],\n",
            "          [7.6001e-07, 1.5752e-06, 1.2790e-06],\n",
            "          [1.1180e-06, 2.1833e-06, 1.6981e-06]],\n",
            "\n",
            "         [[4.0349e-06, 4.2567e-06, 4.0698e-06],\n",
            "          [1.2467e-06, 1.3211e-06, 1.2904e-06],\n",
            "          [8.1860e-07, 8.4620e-07, 8.0041e-07]],\n",
            "\n",
            "         [[4.0782e-06, 5.8231e-06, 4.0260e-06],\n",
            "          [4.0583e-06, 5.3769e-06, 3.0790e-06],\n",
            "          [4.1572e-06, 4.6564e-06, 2.8068e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.6124e-04, 1.6485e-04, 1.6440e-04],\n",
            "          [1.6795e-04, 1.7192e-04, 1.6899e-04],\n",
            "          [1.6856e-04, 1.7058e-04, 1.6650e-04]],\n",
            "\n",
            "         [[2.8032e-06, 3.3178e-06, 1.3168e-06],\n",
            "          [1.5704e-06, 2.1853e-06, 5.2076e-07],\n",
            "          [1.7816e-06, 2.5684e-06, 7.6473e-07]],\n",
            "\n",
            "         [[1.0733e-05, 1.1433e-05, 1.1515e-05],\n",
            "          [1.0936e-05, 1.2150e-05, 1.2396e-05],\n",
            "          [1.1291e-05, 1.2693e-05, 1.2654e-05]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[5.1421e-06, 1.1047e-05, 1.9255e-05],\n",
            "          [7.5156e-06, 1.1262e-05, 1.5425e-05],\n",
            "          [7.0631e-06, 1.4995e-05, 1.7925e-05]],\n",
            "\n",
            "         [[4.1054e-06, 4.9498e-06, 4.1206e-06],\n",
            "          [3.5994e-06, 3.3598e-06, 2.7467e-06],\n",
            "          [2.0495e-06, 2.3873e-06, 2.4927e-06]],\n",
            "\n",
            "         [[5.8439e-06, 3.8166e-06, 1.4400e-06],\n",
            "          [7.7339e-06, 3.0310e-06, 1.1911e-06],\n",
            "          [7.2825e-06, 2.1789e-06, 1.3135e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[5.8455e-05, 5.2687e-05, 4.3942e-05],\n",
            "          [6.3790e-05, 5.5155e-05, 4.2339e-05],\n",
            "          [6.6948e-05, 5.5030e-05, 4.0906e-05]],\n",
            "\n",
            "         [[6.9197e-05, 7.1869e-05, 9.3801e-05],\n",
            "          [7.4140e-05, 7.9880e-05, 9.1033e-05],\n",
            "          [8.7207e-05, 9.4985e-05, 1.0480e-04]],\n",
            "\n",
            "         [[1.0240e-05, 1.4442e-05, 1.9470e-05],\n",
            "          [1.0555e-05, 1.6541e-05, 2.0611e-05],\n",
            "          [1.0928e-05, 1.7777e-05, 1.8117e-05]]],\n",
            "\n",
            "\n",
            "        [[[4.7209e-07, 2.6094e-07, 3.1059e-07],\n",
            "          [3.5455e-07, 1.6157e-07, 1.7976e-07],\n",
            "          [3.6248e-07, 2.0620e-07, 4.0222e-07]],\n",
            "\n",
            "         [[1.1034e-06, 1.2988e-06, 1.7132e-06],\n",
            "          [1.2982e-06, 1.5119e-06, 1.8013e-06],\n",
            "          [2.5248e-07, 2.2032e-07, 1.6289e-07]],\n",
            "\n",
            "         [[1.3513e-06, 5.4573e-06, 7.2280e-06],\n",
            "          [1.6950e-06, 7.5237e-06, 1.0145e-05],\n",
            "          [1.9642e-06, 7.3255e-06, 9.3949e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[2.1684e-05, 2.8909e-05, 4.1949e-05],\n",
            "          [2.6058e-05, 3.5191e-05, 5.3961e-05],\n",
            "          [2.8936e-05, 4.0513e-05, 6.1979e-05]],\n",
            "\n",
            "         [[1.7516e-06, 9.9606e-07, 5.1160e-07],\n",
            "          [1.1775e-06, 4.1215e-07, 4.1482e-08],\n",
            "          [1.0556e-06, 2.8770e-07, 3.9262e-08]],\n",
            "\n",
            "         [[2.0779e-06, 1.8608e-06, 2.0155e-06],\n",
            "          [2.6419e-06, 2.3195e-06, 2.5091e-06],\n",
            "          [2.4620e-06, 2.1932e-06, 2.4930e-06]]],\n",
            "\n",
            "\n",
            "        [[[1.4374e-06, 1.3822e-06, 9.9189e-07],\n",
            "          [9.5783e-07, 9.6794e-07, 7.5770e-07],\n",
            "          [7.5952e-07, 8.9836e-07, 7.6015e-07]],\n",
            "\n",
            "         [[6.0055e-07, 5.4021e-07, 4.6347e-07],\n",
            "          [1.4791e-06, 1.4472e-06, 1.2375e-06],\n",
            "          [5.5538e-07, 5.2945e-07, 4.8851e-07]],\n",
            "\n",
            "         [[8.0763e-08, 3.6256e-07, 2.9388e-07],\n",
            "          [1.2674e-07, 4.5034e-07, 2.1487e-07],\n",
            "          [1.9332e-07, 4.2445e-07, 1.4743e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[9.1578e-08, 1.0331e-07, 3.4859e-07],\n",
            "          [1.8441e-07, 1.9336e-07, 4.5645e-07],\n",
            "          [4.6759e-07, 4.8492e-07, 7.0060e-07]],\n",
            "\n",
            "         [[2.6129e-05, 2.5941e-05, 2.3308e-05],\n",
            "          [2.1182e-05, 2.1264e-05, 1.9137e-05],\n",
            "          [1.5790e-05, 1.5712e-05, 1.4426e-05]],\n",
            "\n",
            "         [[1.2552e-06, 1.1837e-06, 9.5939e-07],\n",
            "          [1.2510e-06, 1.1801e-06, 1.0793e-06],\n",
            "          [1.3037e-06, 1.3585e-06, 1.2955e-06]]]], device='cuda:0')}, 3: {'step': tensor(1920.), 'exp_avg': tensor([-5.1136e-05, -2.4200e-05, -3.1779e-06, -2.2542e-06, -3.9742e-07,\n",
            "        -1.0388e-04, -1.6043e-06,  8.9924e-06, -6.4465e-05, -1.3418e-05,\n",
            "        -3.5301e-05, -6.1819e-06,  3.5276e-05, -9.4045e-05,  5.6052e-45,\n",
            "        -4.6637e-05, -8.5094e-05, -1.0310e-05, -1.1297e-05, -1.2486e-05,\n",
            "        -3.0521e-06, -4.9658e-05,  1.6254e-05, -1.1409e-05,  7.7661e-05,\n",
            "         7.0988e-05,  4.2465e-05,  7.0732e-06, -1.0954e-04,  6.0816e-06,\n",
            "        -2.7444e-05,  1.2922e-06], device='cuda:0'), 'exp_avg_sq': tensor([2.4064e-03, 8.9549e-04, 1.2565e-03, 3.8580e-03, 1.3887e-03, 3.2604e-03,\n",
            "        1.5234e-03, 1.2455e-03, 2.1971e-03, 8.7378e-04, 1.4263e-03, 2.4618e-03,\n",
            "        1.7102e-03, 5.8110e-03, 1.8795e-12, 9.0029e-04, 1.4850e-03, 7.3742e-04,\n",
            "        7.2691e-04, 1.4205e-03, 7.7476e-04, 8.5718e-04, 3.6230e-03, 1.3762e-03,\n",
            "        3.8263e-03, 2.8874e-03, 6.1353e-03, 1.5042e-03, 1.4887e-02, 9.4641e-04,\n",
            "        7.6206e-04, 4.2047e-04], device='cuda:0')}, 4: {'step': tensor(1920.), 'exp_avg': tensor([[[[ 3.0059e-07,  4.7616e-07,  6.0556e-07],\n",
            "          [-6.7258e-08, -3.4168e-08,  1.7690e-07],\n",
            "          [ 9.4438e-08,  1.9426e-08,  8.9327e-08]],\n",
            "\n",
            "         [[ 1.0385e-07,  1.3760e-07,  1.8709e-07],\n",
            "          [ 5.6259e-08,  7.7400e-08,  5.5574e-08],\n",
            "          [ 3.3463e-08,  3.2491e-08,  4.8734e-09]],\n",
            "\n",
            "         [[-5.3414e-08, -5.9144e-07, -3.1166e-07],\n",
            "          [-2.2983e-07, -7.3011e-07, -3.9629e-07],\n",
            "          [-2.7291e-07, -6.7441e-07, -2.3776e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 1.3720e-08, -1.7877e-07,  1.4882e-07],\n",
            "          [-4.9543e-08, -3.7442e-07,  1.2156e-08],\n",
            "          [-1.2523e-07, -4.3908e-07, -4.2232e-08]],\n",
            "\n",
            "         [[ 1.2121e-08, -1.3748e-07,  2.9744e-07],\n",
            "          [-3.4135e-07, -3.6088e-07,  7.9111e-08],\n",
            "          [-3.2196e-07, -3.4221e-07,  4.6904e-08]],\n",
            "\n",
            "         [[ 4.0355e-08,  2.7258e-08,  8.4939e-08],\n",
            "          [-8.4836e-08, -1.1418e-08,  8.0068e-10],\n",
            "          [ 2.8853e-08,  9.1820e-08,  4.5756e-08]]],\n",
            "\n",
            "\n",
            "        [[[-1.8092e-06, -1.9930e-06, -8.7557e-07],\n",
            "          [-2.4374e-06, -2.7148e-06, -1.4424e-06],\n",
            "          [-9.2081e-07, -1.3339e-06, -1.6481e-06]],\n",
            "\n",
            "         [[-8.4976e-08, -1.4224e-07, -1.5988e-07],\n",
            "          [ 4.0233e-08, -8.3550e-08, -2.5349e-07],\n",
            "          [ 4.5435e-09, -1.4118e-07, -1.9185e-07]],\n",
            "\n",
            "         [[-1.3479e-06, -1.1273e-06, -6.1016e-07],\n",
            "          [-2.1986e-06, -1.6633e-06, -1.0740e-06],\n",
            "          [-2.5827e-06, -1.5100e-06, -5.8932e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-9.6270e-07, -8.6572e-07, -5.2132e-07],\n",
            "          [-1.8401e-06, -2.1220e-06, -9.9582e-07],\n",
            "          [-2.0691e-06, -1.9122e-06, -6.0085e-07]],\n",
            "\n",
            "         [[-9.7654e-07, -5.1967e-07, -3.5136e-07],\n",
            "          [-1.9177e-06, -9.4124e-07, -5.7106e-07],\n",
            "          [-1.7740e-06, -5.6429e-07, -3.3532e-07]],\n",
            "\n",
            "         [[-4.2836e-07, -6.6483e-07, -8.3868e-08],\n",
            "          [-1.0051e-06, -7.5719e-07, -1.0986e-07],\n",
            "          [-4.3866e-07, -1.7846e-07, -4.7641e-08]]],\n",
            "\n",
            "\n",
            "        [[[ 7.7976e-08,  1.1412e-07,  3.0421e-07],\n",
            "          [-1.2039e-07, -1.6760e-07, -1.0744e-07],\n",
            "          [ 1.0017e-08,  2.4038e-08, -3.1274e-08]],\n",
            "\n",
            "         [[ 5.0399e-08,  2.4806e-08,  8.0995e-09],\n",
            "          [-3.1644e-08, -7.8961e-08, -8.1404e-08],\n",
            "          [-6.8682e-08, -1.2425e-07, -1.2674e-07]],\n",
            "\n",
            "         [[-1.1692e-06, -6.2484e-07, -2.8058e-07],\n",
            "          [-1.0892e-06, -5.1788e-07, -1.2564e-07],\n",
            "          [-1.0479e-06, -5.5497e-07, -2.2234e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-6.7852e-07, -4.1353e-07,  3.7647e-08],\n",
            "          [-4.9274e-07, -4.0064e-07,  3.0730e-09],\n",
            "          [-3.3078e-07, -2.3322e-07, -4.8996e-08]],\n",
            "\n",
            "         [[-6.1457e-07,  1.6240e-07,  2.7088e-07],\n",
            "          [-6.7416e-07,  1.4019e-07,  1.6285e-07],\n",
            "          [-9.2445e-07, -1.7903e-07, -1.6439e-07]],\n",
            "\n",
            "         [[-8.7036e-08, -2.0941e-08,  1.9585e-08],\n",
            "          [ 3.4440e-09, -2.3430e-08, -5.5674e-08],\n",
            "          [ 1.2074e-08, -2.0153e-09, -1.6753e-08]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[-3.6056e-06, -3.3371e-06, -2.7013e-06],\n",
            "          [-3.6809e-06, -4.0269e-06, -3.4050e-06],\n",
            "          [-1.0735e-06, -1.6603e-06, -1.8903e-06]],\n",
            "\n",
            "         [[-4.9401e-07, -5.8745e-07, -6.4158e-07],\n",
            "          [-4.1510e-07, -5.6842e-07, -6.6189e-07],\n",
            "          [-2.7122e-07, -4.2964e-07, -5.6902e-07]],\n",
            "\n",
            "         [[-5.4347e-08,  2.7431e-08, -4.0623e-08],\n",
            "          [-9.3876e-08,  6.3839e-08,  4.7578e-08],\n",
            "          [-4.7424e-07, -3.6068e-07, -3.7075e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.2296e-06, -6.2533e-07, -4.4859e-07],\n",
            "          [-9.4293e-07, -8.1387e-07, -8.1697e-07],\n",
            "          [-6.3351e-07, -6.8098e-07, -6.2740e-07]],\n",
            "\n",
            "         [[-2.5964e-07, -1.5619e-07, -4.2652e-07],\n",
            "          [-6.5156e-07, -5.2130e-07, -7.2202e-07],\n",
            "          [-8.4307e-07, -7.2163e-07, -9.7266e-07]],\n",
            "\n",
            "         [[-8.7645e-07, -1.2087e-06, -1.3460e-06],\n",
            "          [-1.0165e-06, -1.4123e-06, -1.4688e-06],\n",
            "          [-4.3977e-07, -5.9864e-07, -8.2557e-07]]],\n",
            "\n",
            "\n",
            "        [[[-1.0285e-06, -1.4328e-06, -1.6928e-06],\n",
            "          [-3.0150e-07, -1.0041e-06, -1.5412e-06],\n",
            "          [ 2.2714e-07, -3.7972e-07, -1.3796e-08]],\n",
            "\n",
            "         [[ 8.3556e-08,  9.7392e-08,  1.0550e-07],\n",
            "          [ 1.3640e-07,  1.1363e-07,  1.0838e-07],\n",
            "          [ 1.0423e-07,  1.6866e-07,  1.4314e-07]],\n",
            "\n",
            "         [[-4.3796e-07, -1.5621e-06, -1.0896e-06],\n",
            "          [-9.0123e-07, -2.1369e-06, -1.4202e-06],\n",
            "          [-4.8204e-07, -1.6613e-06, -1.0877e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-4.2869e-07, -7.8049e-07, -1.7604e-06],\n",
            "          [-4.6595e-07, -9.5089e-07, -1.8376e-06],\n",
            "          [-5.6434e-08, -7.2680e-07, -1.4024e-06]],\n",
            "\n",
            "         [[-5.7920e-07, -1.3648e-06, -1.7290e-07],\n",
            "          [-1.3405e-06, -2.0354e-06, -5.1354e-07],\n",
            "          [-4.5816e-07, -1.2807e-06, -3.0637e-07]],\n",
            "\n",
            "         [[-3.6498e-07, -6.5751e-07, -6.9189e-07],\n",
            "          [-1.4228e-07, -2.2080e-07, -2.4081e-07],\n",
            "          [ 1.1686e-09, -1.7642e-07,  2.1551e-08]]],\n",
            "\n",
            "\n",
            "        [[[ 4.3383e-07, -2.0390e-07,  6.1283e-07],\n",
            "          [ 5.1478e-07,  6.0942e-08,  1.4488e-07],\n",
            "          [ 4.8648e-07,  2.7399e-07,  5.2170e-07]],\n",
            "\n",
            "         [[ 1.0258e-07,  6.0315e-08,  4.9209e-08],\n",
            "          [ 1.0096e-07,  6.9129e-08,  9.3155e-09],\n",
            "          [ 7.9968e-08,  8.5747e-08,  4.4098e-08]],\n",
            "\n",
            "         [[-3.8217e-07, -3.3596e-07,  2.5401e-07],\n",
            "          [-2.2527e-07, -2.0031e-07,  3.1686e-07],\n",
            "          [-3.0356e-08, -1.2407e-07,  1.4117e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-3.5128e-07, -3.9424e-07,  1.4358e-08],\n",
            "          [-3.1681e-07, -4.2122e-07, -2.9662e-08],\n",
            "          [ 1.9228e-07, -3.0733e-08, -3.4364e-08]],\n",
            "\n",
            "         [[-2.1283e-07,  1.7567e-07,  1.4757e-07],\n",
            "          [-1.2858e-07,  7.9798e-08,  2.8749e-07],\n",
            "          [-7.1309e-09, -1.0530e-07,  1.2048e-07]],\n",
            "\n",
            "         [[ 1.2962e-07,  1.4245e-07,  4.9233e-07],\n",
            "          [ 2.5310e-07,  8.7884e-08,  2.7362e-07],\n",
            "          [ 1.5017e-07,  1.8763e-08,  2.7561e-07]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[3.0986e-07, 5.0486e-07, 4.3215e-07],\n",
            "          [5.0103e-07, 6.3634e-07, 6.1051e-07],\n",
            "          [3.7969e-07, 4.5772e-07, 4.5321e-07]],\n",
            "\n",
            "         [[1.0178e-06, 1.0324e-06, 1.0589e-06],\n",
            "          [1.0802e-06, 1.1217e-06, 1.1301e-06],\n",
            "          [1.0396e-06, 1.0767e-06, 1.0520e-06]],\n",
            "\n",
            "         [[6.1690e-06, 6.2448e-06, 6.1362e-06],\n",
            "          [6.9972e-06, 7.2321e-06, 6.9911e-06],\n",
            "          [7.0142e-06, 7.1314e-06, 6.8859e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.0819e-07, 2.9283e-07, 2.7160e-07],\n",
            "          [2.8226e-07, 7.0786e-07, 5.7503e-07],\n",
            "          [2.6868e-07, 6.2549e-07, 4.4282e-07]],\n",
            "\n",
            "         [[1.6506e-06, 1.6729e-06, 1.5655e-06],\n",
            "          [1.9853e-06, 1.8794e-06, 1.6465e-06],\n",
            "          [1.8015e-06, 1.6718e-06, 1.5526e-06]],\n",
            "\n",
            "         [[1.3321e-07, 1.5187e-07, 1.0967e-07],\n",
            "          [2.0816e-07, 2.3125e-07, 1.7401e-07],\n",
            "          [1.8635e-07, 1.9862e-07, 1.4252e-07]]],\n",
            "\n",
            "\n",
            "        [[[1.0756e-06, 1.9083e-06, 1.2683e-06],\n",
            "          [3.1716e-06, 4.1632e-06, 3.3950e-06],\n",
            "          [1.7462e-06, 2.3284e-06, 2.2983e-06]],\n",
            "\n",
            "         [[5.2871e-07, 6.8506e-07, 9.4858e-07],\n",
            "          [4.0534e-07, 5.2489e-07, 9.0116e-07],\n",
            "          [3.0260e-07, 4.0541e-07, 7.3144e-07]],\n",
            "\n",
            "         [[2.1143e-06, 2.1738e-06, 1.6893e-06],\n",
            "          [4.9137e-06, 4.9207e-06, 4.0438e-06],\n",
            "          [3.8378e-06, 3.8233e-06, 3.0215e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[7.8938e-07, 1.7921e-06, 7.1009e-07],\n",
            "          [1.9573e-06, 3.6864e-06, 1.7413e-06],\n",
            "          [2.7657e-06, 3.9429e-06, 1.5092e-06]],\n",
            "\n",
            "         [[6.1035e-07, 2.0292e-07, 1.2288e-07],\n",
            "          [2.1489e-06, 9.6252e-07, 6.5814e-07],\n",
            "          [1.4078e-06, 3.2400e-07, 2.1061e-07]],\n",
            "\n",
            "         [[1.9444e-07, 2.4507e-07, 1.6389e-07],\n",
            "          [7.4828e-07, 8.1175e-07, 5.1219e-07],\n",
            "          [7.1043e-07, 5.3790e-07, 3.1591e-07]]],\n",
            "\n",
            "\n",
            "        [[[2.2980e-07, 2.6904e-07, 2.2405e-07],\n",
            "          [1.4393e-07, 1.5195e-07, 1.2099e-07],\n",
            "          [9.0121e-08, 1.3389e-07, 1.3966e-07]],\n",
            "\n",
            "         [[2.5247e-07, 2.5147e-07, 3.2711e-07],\n",
            "          [2.0703e-07, 2.1175e-07, 2.8700e-07],\n",
            "          [1.8960e-07, 1.9438e-07, 2.6713e-07]],\n",
            "\n",
            "         [[2.4119e-06, 2.5794e-06, 2.3677e-06],\n",
            "          [2.6932e-06, 2.8439e-06, 2.6014e-06],\n",
            "          [2.8887e-06, 3.0486e-06, 2.8140e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[2.8320e-07, 4.3752e-07, 3.6082e-07],\n",
            "          [2.8279e-07, 4.4190e-07, 3.2086e-07],\n",
            "          [2.9908e-07, 4.0579e-07, 3.3097e-07]],\n",
            "\n",
            "         [[4.9660e-07, 1.6464e-07, 9.6420e-08],\n",
            "          [4.6451e-07, 9.0318e-08, 6.7278e-08],\n",
            "          [3.9022e-07, 8.0797e-08, 6.5659e-08]],\n",
            "\n",
            "         [[7.8489e-08, 4.1697e-08, 3.3699e-08],\n",
            "          [5.5767e-08, 1.5282e-08, 1.0196e-08],\n",
            "          [4.7327e-08, 1.2179e-08, 1.0862e-08]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[2.3084e-06, 2.9247e-06, 2.1862e-06],\n",
            "          [1.8648e-06, 2.9698e-06, 2.3249e-06],\n",
            "          [8.2831e-07, 1.5234e-06, 1.4582e-06]],\n",
            "\n",
            "         [[2.0476e-06, 2.3108e-06, 2.2741e-06],\n",
            "          [1.6280e-06, 1.8519e-06, 1.8818e-06],\n",
            "          [1.3229e-06, 1.4606e-06, 1.5529e-06]],\n",
            "\n",
            "         [[5.9832e-08, 5.0528e-08, 6.5236e-08],\n",
            "          [1.6998e-07, 1.7043e-07, 2.0453e-07],\n",
            "          [1.6592e-07, 1.6756e-07, 1.7974e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[6.9077e-07, 3.6556e-07, 3.2328e-07],\n",
            "          [4.5761e-07, 3.3651e-07, 3.3824e-07],\n",
            "          [2.9983e-07, 2.9496e-07, 3.0526e-07]],\n",
            "\n",
            "         [[1.0172e-07, 1.3537e-07, 2.2414e-07],\n",
            "          [2.1198e-07, 2.6737e-07, 3.5675e-07],\n",
            "          [2.6026e-07, 3.0682e-07, 3.1430e-07]],\n",
            "\n",
            "         [[7.5996e-07, 9.8080e-07, 1.0724e-06],\n",
            "          [7.4225e-07, 9.8199e-07, 1.0167e-06],\n",
            "          [3.0622e-07, 4.0273e-07, 4.2823e-07]]],\n",
            "\n",
            "\n",
            "        [[[2.8246e-06, 2.5238e-06, 3.9448e-06],\n",
            "          [3.7247e-06, 3.2848e-06, 4.1341e-06],\n",
            "          [2.6409e-06, 1.5174e-06, 1.9994e-06]],\n",
            "\n",
            "         [[4.2484e-06, 2.7158e-06, 2.4847e-06],\n",
            "          [3.9989e-06, 2.2013e-06, 2.0364e-06],\n",
            "          [3.7706e-06, 1.9313e-06, 1.7130e-06]],\n",
            "\n",
            "         [[1.7942e-06, 3.0794e-06, 3.8144e-06],\n",
            "          [4.3002e-06, 6.4998e-06, 7.7424e-06],\n",
            "          [2.8627e-06, 4.5337e-06, 5.5250e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[3.8692e-07, 9.5095e-07, 5.7628e-06],\n",
            "          [6.7215e-07, 2.0049e-06, 7.2305e-06],\n",
            "          [5.0038e-07, 2.0641e-06, 6.1134e-06]],\n",
            "\n",
            "         [[1.6375e-06, 3.6833e-06, 1.4122e-06],\n",
            "          [3.4152e-06, 6.3498e-06, 2.5667e-06],\n",
            "          [2.2949e-06, 4.5443e-06, 1.6056e-06]],\n",
            "\n",
            "         [[1.1258e-06, 1.3492e-06, 1.1334e-06],\n",
            "          [1.7630e-06, 2.1747e-06, 1.4524e-06],\n",
            "          [1.5230e-06, 1.3681e-06, 6.2619e-07]]],\n",
            "\n",
            "\n",
            "        [[[1.8527e-06, 2.3828e-06, 2.4356e-06],\n",
            "          [1.5804e-06, 2.0176e-06, 2.0242e-06],\n",
            "          [1.2950e-06, 1.9714e-06, 2.1253e-06]],\n",
            "\n",
            "         [[1.3299e-06, 1.5096e-06, 1.8384e-06],\n",
            "          [1.4856e-06, 1.6976e-06, 2.0180e-06],\n",
            "          [1.5270e-06, 1.8189e-06, 2.1312e-06]],\n",
            "\n",
            "         [[1.3515e-06, 1.3901e-06, 1.1768e-06],\n",
            "          [9.7207e-07, 9.2083e-07, 7.6455e-07],\n",
            "          [5.6625e-07, 5.2454e-07, 4.5077e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.1708e-06, 2.3532e-06, 1.3373e-06],\n",
            "          [1.1982e-06, 2.2509e-06, 1.0022e-06],\n",
            "          [8.3523e-07, 1.4529e-06, 5.6391e-07]],\n",
            "\n",
            "         [[5.0479e-07, 3.3014e-07, 2.3018e-07],\n",
            "          [2.3365e-07, 1.1127e-07, 1.0889e-07],\n",
            "          [1.0840e-07, 5.8786e-08, 7.4479e-08]],\n",
            "\n",
            "         [[5.8606e-07, 7.0011e-07, 6.6834e-07],\n",
            "          [6.0011e-07, 5.5318e-07, 4.9396e-07],\n",
            "          [4.9188e-07, 5.1123e-07, 5.6457e-07]]]], device='cuda:0')}, 5: {'step': tensor(1920.), 'exp_avg': tensor([ 2.0983e-05, -3.2798e-05,  6.5683e-06, -6.7589e-05, -9.9508e-06,\n",
            "         7.8585e-06, -3.8804e-05, -1.9757e-05, -2.2632e-05, -7.5030e-06,\n",
            "        -2.1895e-05,  1.2668e-05,  3.4285e-05,  2.1819e-34, -1.5227e-05,\n",
            "        -3.9344e-06, -2.7476e-05,  2.1728e-05, -7.8376e-06, -8.9848e-06,\n",
            "         2.7580e-05,  2.2019e-05,  2.2518e-05,  7.5749e-06, -5.9262e-06,\n",
            "        -1.2595e-08,  2.0754e-06, -3.4399e-06, -1.5217e-05,  1.5407e-05,\n",
            "        -2.9236e-06,  1.1009e-05,  1.7371e-05, -1.0557e-05,  2.8318e-05,\n",
            "         3.5323e-06,  1.5618e-05, -3.6110e-07, -8.6270e-06,  7.9632e-06,\n",
            "         1.6127e-05,  5.6052e-45,  7.1318e-06, -1.9162e-06,  8.6521e-06,\n",
            "         3.6459e-06, -1.2222e-05, -2.9192e-05, -7.9990e-06, -7.6261e-06,\n",
            "        -5.5924e-06,  5.6830e-06,  5.6052e-45,  1.4698e-05, -1.5483e-05,\n",
            "        -7.1776e-06, -7.2419e-06,  3.2914e-05,  2.4507e-06, -5.2265e-05,\n",
            "        -3.5098e-09, -3.9663e-05, -1.5736e-06,  6.7930e-06], device='cuda:0'), 'exp_avg_sq': tensor([1.1794e-03, 4.8262e-04, 6.6513e-04, 3.8137e-03, 2.9459e-05, 9.5889e-04,\n",
            "        1.1062e-03, 1.1778e-03, 1.8178e-04, 8.6370e-04, 5.5693e-04, 5.3871e-04,\n",
            "        1.0662e-03, 1.3509e-08, 5.4360e-04, 1.1467e-03, 3.7691e-04, 6.0956e-04,\n",
            "        2.1478e-03, 3.0017e-04, 5.9228e-04, 4.2019e-04, 1.1980e-03, 7.2360e-04,\n",
            "        9.3373e-04, 4.2667e-05, 1.9273e-04, 1.5298e-04, 3.0365e-04, 6.2724e-04,\n",
            "        1.3470e-03, 3.8241e-05, 3.7018e-04, 1.6139e-04, 5.4610e-04, 6.4091e-05,\n",
            "        4.1172e-04, 3.5114e-03, 2.5144e-04, 1.4497e-03, 4.0199e-04, 3.9180e-14,\n",
            "        1.8480e-03, 4.9104e-04, 4.9557e-04, 1.0075e-03, 5.5726e-04, 4.0668e-04,\n",
            "        2.1804e-04, 1.4240e-03, 4.2266e-04, 6.7197e-04, 4.0351e-10, 7.7840e-04,\n",
            "        1.3055e-03, 6.7740e-04, 7.7970e-05, 2.1432e-03, 2.6664e-04, 1.7025e-03,\n",
            "        1.4073e-06, 5.4636e-04, 1.5152e-03, 2.2769e-04], device='cuda:0')}, 6: {'step': tensor(1920.), 'exp_avg': tensor([[[[-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45]],\n",
            "\n",
            "         [[ 5.6052e-45,  5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45]],\n",
            "\n",
            "         [[-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [-5.6052e-45, -5.6052e-45, -5.6052e-45]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [ 5.6052e-45, -5.6052e-45, -5.6052e-45],\n",
            "          [ 5.6052e-45, -5.6052e-45, -5.6052e-45]],\n",
            "\n",
            "         [[ 5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45]],\n",
            "\n",
            "         [[ 5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [ 5.6052e-45,  5.6052e-45,  5.6052e-45],\n",
            "          [-5.6052e-45,  5.6052e-45,  5.6052e-45]]],\n",
            "\n",
            "\n",
            "        [[[ 9.2136e-08,  4.9005e-09, -1.4286e-08],\n",
            "          [ 7.3352e-08,  1.1220e-10, -2.3532e-08],\n",
            "          [ 3.2180e-08, -4.5505e-08, -1.2694e-07]],\n",
            "\n",
            "         [[ 1.0536e-07, -3.2324e-08, -2.8300e-08],\n",
            "          [-5.2195e-08, -6.5654e-08, -2.1735e-07],\n",
            "          [-5.2578e-08, -1.9249e-07, -3.0899e-07]],\n",
            "\n",
            "         [[ 2.7587e-07,  5.3450e-08,  6.4270e-09],\n",
            "          [ 2.2729e-07,  6.5263e-08,  3.3260e-09],\n",
            "          [ 1.4962e-07, -1.2119e-08, -1.1699e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 2.5067e-07,  1.4892e-07,  1.6348e-07],\n",
            "          [ 1.3544e-07,  1.0947e-07, -2.4641e-07],\n",
            "          [ 6.7831e-08,  1.8886e-07, -1.2827e-08]],\n",
            "\n",
            "         [[-9.2936e-08, -4.9206e-09, -1.1609e-08],\n",
            "          [-8.4117e-08, -9.5346e-08, -1.6818e-07],\n",
            "          [-1.5198e-07, -1.6973e-07, -2.3024e-07]],\n",
            "\n",
            "         [[-8.1424e-08, -1.0984e-07, -1.2274e-08],\n",
            "          [-1.3845e-07,  1.4390e-08, -9.4605e-08],\n",
            "          [-1.9605e-07, -1.1670e-07, -6.6048e-08]]],\n",
            "\n",
            "\n",
            "        [[[-1.1333e-07, -1.1823e-07, -2.0388e-09],\n",
            "          [-1.1187e-07, -1.6921e-07, -1.0234e-08],\n",
            "          [-9.2609e-08, -5.2410e-08,  8.0651e-09]],\n",
            "\n",
            "         [[-1.9446e-07, -2.8977e-07, -7.4939e-08],\n",
            "          [-7.5458e-08, -3.6791e-07, -3.5900e-08],\n",
            "          [-4.0229e-08, -1.1936e-07, -1.0399e-08]],\n",
            "\n",
            "         [[-2.6812e-07, -3.6485e-07, -5.7667e-08],\n",
            "          [-2.4232e-07, -3.3320e-07, -2.1198e-08],\n",
            "          [-2.1309e-07, -2.7231e-07,  2.1888e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-5.7873e-09, -2.2913e-07, -3.2275e-07],\n",
            "          [ 4.3397e-09, -2.4209e-07, -5.9901e-07],\n",
            "          [ 2.2783e-08, -1.2853e-07, -4.7748e-07]],\n",
            "\n",
            "         [[-5.3392e-07, -1.4711e-07,  3.1707e-10],\n",
            "          [-5.7432e-07, -1.0823e-07, -3.0410e-08],\n",
            "          [-4.5670e-07, -7.5690e-08, -1.0827e-07]],\n",
            "\n",
            "         [[-1.6508e-07, -7.8601e-07, -5.8405e-07],\n",
            "          [-1.6706e-07, -8.3801e-07, -5.1533e-07],\n",
            "          [-2.0678e-07, -6.5474e-07, -4.6938e-07]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[-4.7889e-10, -1.3688e-09,  2.0237e-10],\n",
            "          [ 1.9597e-10, -2.7334e-09, -1.8780e-09],\n",
            "          [ 1.1409e-08,  2.1130e-08,  2.0304e-08]],\n",
            "\n",
            "         [[-4.8125e-12, -1.5526e-09,  1.2919e-09],\n",
            "          [-6.3917e-10, -9.4602e-09, -1.1182e-08],\n",
            "          [ 2.2400e-08,  4.8062e-08,  6.0262e-08]],\n",
            "\n",
            "         [[ 9.7002e-11, -3.7821e-09,  1.1248e-09],\n",
            "          [ 4.8117e-11, -1.7468e-09, -4.9483e-09],\n",
            "          [ 4.3301e-10,  7.0587e-09,  1.5749e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-1.4954e-09, -3.2439e-09, -5.6342e-11],\n",
            "          [ 9.8633e-09,  1.6433e-08,  1.1083e-08],\n",
            "          [ 1.2556e-08,  1.9735e-08,  1.7079e-08]],\n",
            "\n",
            "         [[-7.7268e-10,  1.0097e-10,  7.5618e-10],\n",
            "          [-2.7000e-09, -9.3635e-09, -9.1553e-09],\n",
            "          [ 2.9995e-08,  3.0147e-08,  1.9098e-08]],\n",
            "\n",
            "         [[ 4.4682e-09, -8.3979e-09,  5.1345e-10],\n",
            "          [ 5.0934e-09, -1.6989e-08, -1.7709e-08],\n",
            "          [ 7.4914e-09,  2.3626e-12,  8.2032e-09]]],\n",
            "\n",
            "\n",
            "        [[[-5.4461e-07, -6.6592e-07, -4.8451e-07],\n",
            "          [-4.7478e-07, -4.7850e-07, -4.5005e-07],\n",
            "          [-1.3387e-07, -4.8090e-08, -6.6253e-08]],\n",
            "\n",
            "         [[-3.4860e-07, -7.7531e-07, -4.8479e-07],\n",
            "          [-3.9957e-07, -5.5046e-07, -4.7437e-07],\n",
            "          [-1.0324e-07, -1.7860e-07, -1.3482e-07]],\n",
            "\n",
            "         [[-3.5626e-07, -7.6731e-07, -4.8835e-07],\n",
            "          [-2.4641e-07, -4.9117e-07, -3.7531e-07],\n",
            "          [-1.0796e-07, -1.6871e-07, -1.3348e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[-4.3979e-07, -4.6934e-07, -2.0787e-07],\n",
            "          [-6.5225e-08, -7.3386e-08, -3.5065e-08],\n",
            "          [-4.1476e-08, -4.6756e-08,  3.2974e-08]],\n",
            "\n",
            "         [[-6.5020e-07, -5.2303e-07, -1.6003e-07],\n",
            "          [-1.9725e-07, -1.9664e-07, -1.0443e-07],\n",
            "          [-2.7314e-08, -6.5722e-08, -7.2950e-08]],\n",
            "\n",
            "         [[-2.6032e-07, -4.8246e-07, -1.1110e-07],\n",
            "          [-1.6267e-07, -1.9504e-07, -9.8405e-08],\n",
            "          [-9.3157e-09,  3.3535e-09,  7.2539e-09]]],\n",
            "\n",
            "\n",
            "        [[[-1.3894e-09, -6.1859e-10,  3.9112e-10],\n",
            "          [ 3.2214e-09, -1.8399e-08,  8.5492e-12],\n",
            "          [ 3.9735e-09,  3.4735e-09,  1.5439e-09]],\n",
            "\n",
            "         [[-1.5486e-08, -2.5509e-09,  5.6423e-10],\n",
            "          [ 9.8815e-09,  2.3548e-09, -1.8390e-09],\n",
            "          [-2.9704e-09, -1.0806e-09, -7.2918e-10]],\n",
            "\n",
            "         [[-1.9437e-09, -4.3093e-10, -1.6421e-09],\n",
            "          [-3.2569e-10, -4.2787e-15, -1.1704e-09],\n",
            "          [ 2.4795e-10,  4.4694e-10, -3.6261e-10]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[ 1.5145e-07,  6.1059e-08,  8.8054e-08],\n",
            "          [ 5.6936e-09, -6.1526e-08, -2.3307e-08],\n",
            "          [ 2.7961e-07,  1.9581e-07,  2.3849e-07]],\n",
            "\n",
            "         [[ 2.9143e-09, -3.6420e-08, -1.3159e-08],\n",
            "          [-2.4936e-08, -8.8255e-08, -8.2967e-09],\n",
            "          [ 3.4761e-08, -4.7745e-08,  3.4459e-08]],\n",
            "\n",
            "         [[-1.3636e-07, -6.4964e-08, -2.4185e-08],\n",
            "          [-1.9346e-07, -9.5570e-08, -3.4726e-08],\n",
            "          [-5.5813e-08, -4.8931e-08, -1.8465e-08]]]], device='cuda:0'), 'exp_avg_sq': tensor([[[[5.9860e-16, 1.8735e-15, 1.6150e-17],\n",
            "          [1.3010e-15, 9.3373e-16, 3.1585e-17],\n",
            "          [1.5083e-16, 1.4147e-16, 5.2339e-17]],\n",
            "\n",
            "         [[4.0832e-16, 5.9786e-19, 7.9552e-33],\n",
            "          [4.1533e-16, 3.7826e-18, 2.5241e-21],\n",
            "          [3.9514e-16, 4.8049e-20, 4.1994e-18]],\n",
            "\n",
            "         [[1.7602e-15, 6.7050e-16, 4.0796e-17],\n",
            "          [3.3528e-15, 3.2493e-15, 3.9524e-17],\n",
            "          [1.8685e-15, 3.9281e-16, 5.0720e-17]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.9122e-15, 1.5950e-15, 5.1166e-19],\n",
            "          [3.7152e-15, 7.3042e-15, 1.3601e-18],\n",
            "          [5.7760e-16, 6.6202e-16, 1.6486e-28]],\n",
            "\n",
            "         [[8.5795e-17, 7.3037e-17, 2.4246e-17],\n",
            "          [4.8801e-17, 3.4017e-16, 3.7477e-17],\n",
            "          [4.8280e-17, 2.2959e-17, 8.0384e-17]],\n",
            "\n",
            "         [[1.1233e-15, 1.0492e-18, 1.4275e-20],\n",
            "          [1.2010e-15, 1.0664e-17, 1.1416e-31],\n",
            "          [1.3019e-15, 3.1131e-19, 2.9437e-18]]],\n",
            "\n",
            "\n",
            "        [[[1.5086e-07, 1.3603e-07, 1.3997e-07],\n",
            "          [1.6365e-07, 1.3894e-07, 1.3928e-07],\n",
            "          [1.6670e-07, 1.4083e-07, 1.4270e-07]],\n",
            "\n",
            "         [[4.8344e-08, 2.3558e-08, 1.5725e-08],\n",
            "          [1.4736e-07, 7.3072e-08, 7.6969e-08],\n",
            "          [1.0693e-07, 4.8924e-08, 5.5885e-08]],\n",
            "\n",
            "         [[1.5463e-07, 1.2029e-07, 1.1936e-07],\n",
            "          [1.9499e-07, 1.3911e-07, 1.3442e-07],\n",
            "          [2.2869e-07, 1.6096e-07, 1.5943e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[3.2141e-07, 3.6153e-07, 2.7463e-07],\n",
            "          [4.9226e-07, 7.3107e-07, 7.4887e-07],\n",
            "          [3.1933e-07, 4.8966e-07, 6.8186e-07]],\n",
            "\n",
            "         [[3.6985e-08, 2.9307e-08, 8.0743e-08],\n",
            "          [9.0078e-08, 8.7640e-08, 1.3811e-07],\n",
            "          [7.8107e-08, 6.4462e-08, 1.0407e-07]],\n",
            "\n",
            "         [[2.1319e-07, 3.2151e-07, 2.4022e-07],\n",
            "          [2.2050e-07, 3.1268e-07, 2.2309e-07],\n",
            "          [2.3403e-07, 3.2926e-07, 2.0748e-07]]],\n",
            "\n",
            "\n",
            "        [[[2.8045e-07, 3.6239e-07, 1.0406e-07],\n",
            "          [2.4963e-07, 3.5358e-07, 8.9025e-08],\n",
            "          [1.8530e-07, 2.5305e-07, 7.2708e-08]],\n",
            "\n",
            "         [[1.7925e-07, 6.7849e-07, 1.1560e-07],\n",
            "          [1.2056e-07, 7.8397e-07, 8.0455e-08],\n",
            "          [8.1151e-08, 6.6577e-07, 1.1919e-07]],\n",
            "\n",
            "         [[2.7272e-07, 6.5601e-07, 1.3852e-07],\n",
            "          [2.7775e-07, 6.8312e-07, 1.1529e-07],\n",
            "          [2.7112e-07, 5.5859e-07, 1.1311e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[6.3198e-08, 1.2378e-07, 8.9696e-08],\n",
            "          [4.2143e-08, 1.5449e-07, 1.4076e-07],\n",
            "          [3.6162e-08, 1.6000e-07, 1.9476e-07]],\n",
            "\n",
            "         [[6.5243e-07, 6.0663e-07, 4.0083e-08],\n",
            "          [6.5165e-07, 5.9538e-07, 2.9479e-08],\n",
            "          [5.5846e-07, 5.4456e-07, 2.9781e-08]],\n",
            "\n",
            "         [[1.2797e-07, 1.6139e-06, 7.4917e-07],\n",
            "          [1.6347e-07, 1.6742e-06, 6.9567e-07],\n",
            "          [2.1881e-07, 1.6765e-06, 7.1421e-07]]],\n",
            "\n",
            "\n",
            "        ...,\n",
            "\n",
            "\n",
            "        [[[1.3630e-08, 1.8300e-08, 1.9622e-08],\n",
            "          [5.8047e-08, 9.5604e-08, 9.7507e-08],\n",
            "          [6.1707e-08, 8.5235e-08, 7.4948e-08]],\n",
            "\n",
            "         [[4.1728e-09, 1.1620e-08, 2.1630e-08],\n",
            "          [1.1695e-07, 1.9523e-07, 2.8359e-07],\n",
            "          [7.7561e-08, 1.2108e-07, 1.5460e-07]],\n",
            "\n",
            "         [[1.0317e-08, 1.5204e-08, 1.6579e-08],\n",
            "          [1.8110e-08, 2.9849e-08, 5.2996e-08],\n",
            "          [3.0371e-08, 4.9643e-08, 7.8487e-08]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[2.5048e-08, 2.1136e-08, 2.2617e-08],\n",
            "          [9.0909e-08, 1.0700e-07, 1.0934e-07],\n",
            "          [8.9629e-08, 1.2692e-07, 1.5823e-07]],\n",
            "\n",
            "         [[6.1682e-09, 1.6523e-08, 1.0720e-08],\n",
            "          [3.7392e-08, 1.2995e-07, 5.3401e-08],\n",
            "          [5.0416e-08, 1.3520e-07, 4.2383e-08]],\n",
            "\n",
            "         [[9.6730e-08, 1.4848e-07, 1.8791e-07],\n",
            "          [2.8987e-07, 4.6802e-07, 6.2227e-07],\n",
            "          [3.4712e-07, 6.0836e-07, 8.1144e-07]]],\n",
            "\n",
            "\n",
            "        [[[7.0991e-07, 9.1545e-07, 7.9270e-07],\n",
            "          [7.1611e-07, 8.5759e-07, 8.0351e-07],\n",
            "          [9.2220e-07, 1.0653e-06, 1.0727e-06]],\n",
            "\n",
            "         [[7.0944e-08, 2.4252e-07, 1.3896e-07],\n",
            "          [3.6869e-08, 8.9221e-08, 4.9681e-08],\n",
            "          [2.3615e-08, 3.5906e-08, 3.0586e-08]],\n",
            "\n",
            "         [[7.1816e-07, 1.0884e-06, 9.8572e-07],\n",
            "          [7.4225e-07, 9.9708e-07, 9.3805e-07],\n",
            "          [8.0721e-07, 9.7382e-07, 1.0269e-06]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[7.2880e-08, 7.7855e-08, 7.5584e-08],\n",
            "          [1.9834e-08, 1.6277e-08, 2.4748e-08],\n",
            "          [1.1066e-08, 1.0474e-08, 1.3252e-08]],\n",
            "\n",
            "         [[3.4273e-07, 3.7462e-07, 9.3878e-08],\n",
            "          [1.1106e-07, 1.3529e-07, 3.1757e-08],\n",
            "          [5.7180e-08, 7.5242e-08, 3.9954e-08]],\n",
            "\n",
            "         [[3.0110e-07, 5.7246e-07, 3.1811e-07],\n",
            "          [9.3908e-08, 2.0123e-07, 1.1076e-07],\n",
            "          [8.4572e-09, 1.7304e-08, 9.8436e-09]]],\n",
            "\n",
            "\n",
            "        [[[2.5278e-07, 2.1247e-07, 1.9869e-07],\n",
            "          [2.3492e-07, 1.9880e-07, 1.8669e-07],\n",
            "          [1.9118e-07, 1.6378e-07, 1.5848e-07]],\n",
            "\n",
            "         [[8.2057e-08, 3.3389e-08, 2.0788e-08],\n",
            "          [7.7195e-08, 2.9837e-08, 1.8751e-08],\n",
            "          [5.5661e-08, 2.0544e-08, 1.8268e-08]],\n",
            "\n",
            "         [[4.2082e-07, 3.8312e-07, 3.5275e-07],\n",
            "          [4.2635e-07, 3.8617e-07, 3.5470e-07],\n",
            "          [2.8763e-07, 2.5042e-07, 2.2980e-07]],\n",
            "\n",
            "         ...,\n",
            "\n",
            "         [[1.7494e-07, 1.7038e-07, 1.3201e-07],\n",
            "          [2.2482e-07, 2.5907e-07, 2.0146e-07],\n",
            "          [2.8533e-07, 3.3244e-07, 3.0709e-07]],\n",
            "\n",
            "         [[6.0903e-08, 3.5425e-08, 6.5333e-08],\n",
            "          [4.9032e-08, 3.4633e-08, 6.9175e-08],\n",
            "          [6.2794e-08, 5.4130e-08, 8.9772e-08]],\n",
            "\n",
            "         [[7.7502e-07, 5.8260e-07, 2.9527e-07],\n",
            "          [9.1601e-07, 7.1091e-07, 4.0783e-07],\n",
            "          [7.2122e-07, 5.3753e-07, 2.8227e-07]]]], device='cuda:0')}, 7: {'step': tensor(1920.), 'exp_avg': tensor([-5.6052e-45,  8.8870e-06, -2.1876e-06, -1.4117e-05, -5.4881e-06,\n",
            "         2.6558e-06,  5.6052e-45, -3.3807e-05,  1.3301e-08, -6.6602e-07,\n",
            "         5.6052e-45, -4.8954e-13, -1.2110e-05, -2.2201e-05, -2.4244e-05,\n",
            "         7.2223e-06, -4.6394e-07, -1.1403e-05,  5.6052e-45, -1.3877e-05,\n",
            "        -3.8452e-06,  5.6052e-45, -4.0417e-06, -1.2874e-15,  1.1011e-07,\n",
            "        -3.4786e-06, -8.3651e-06, -1.1794e-05, -3.5751e-06, -2.4884e-06,\n",
            "         5.0838e-06,  3.3271e-06, -5.8541e-06, -1.2587e-10, -1.3990e-06,\n",
            "        -7.6781e-06, -2.7685e-05, -1.8685e-09,  3.1101e-08,  2.3770e-07,\n",
            "         2.0977e-31,  1.2237e-05,  6.2244e-06, -5.4248e-06, -2.2026e-06,\n",
            "        -1.0984e-05, -1.7434e-05, -3.2445e-06,  7.4926e-06, -2.2724e-08,\n",
            "        -6.9169e-06, -9.0819e-06,  1.2041e-06, -7.3995e-06, -9.5825e-07,\n",
            "        -1.1623e-05, -2.2787e-05, -1.1211e-05, -3.4276e-20, -4.7408e-26,\n",
            "         6.1888e-07,  2.0888e-06, -4.4916e-06, -7.4829e-06, -1.0699e-05,\n",
            "         5.7207e-06, -8.0564e-06, -2.5436e-05, -8.9859e-08,  1.4452e-05,\n",
            "         1.3983e-09, -1.0837e-06, -3.9824e-06,  4.4645e-06, -1.3065e-11,\n",
            "         5.6052e-45, -3.3434e-06, -3.7534e-06,  1.1045e-05,  6.1194e-06,\n",
            "        -7.2387e-06, -4.0982e-06,  4.2513e-07, -4.2582e-06, -1.0815e-05,\n",
            "        -5.2303e-06,  3.7236e-07, -5.2501e-10, -1.5556e-05, -2.1080e-05,\n",
            "         3.0508e-07, -1.4794e-05, -1.3174e-05, -1.7390e-05,  5.6052e-45,\n",
            "         7.8723e-06, -1.5940e-06, -1.3368e-06, -1.5934e-05, -4.4487e-06,\n",
            "        -1.1401e-06,  1.1367e-08,  5.6052e-45, -1.0414e-05,  1.0618e-05,\n",
            "         5.6052e-45,  3.2715e-06, -1.2412e-05, -5.0524e-06, -1.3184e-05,\n",
            "        -5.7470e-06,  8.8274e-06, -3.1882e-06, -7.9111e-06, -1.5292e-05,\n",
            "         1.1359e-33, -1.7243e-12, -5.6052e-45, -5.2980e-06, -2.5388e-07,\n",
            "         5.8017e-06, -5.8510e-07, -7.6242e-38,  3.7967e-31,  2.2307e-05,\n",
            "         6.6128e-08, -6.5315e-06,  3.7510e-06], device='cuda:0'), 'exp_avg_sq': tensor([5.1631e-13, 2.1746e-04, 5.6281e-05, 2.1517e-03, 2.9253e-04, 3.1785e-04,\n",
            "        2.5445e-08, 5.9172e-04, 6.4631e-06, 1.5734e-04, 4.3226e-13, 1.4513e-05,\n",
            "        5.9479e-04, 1.0651e-03, 9.9570e-04, 8.1083e-04, 1.9891e-05, 7.2594e-04,\n",
            "        5.8221e-07, 3.7270e-04, 1.0081e-04, 8.7206e-07, 8.4821e-05, 9.7949e-05,\n",
            "        1.7400e-04, 1.6241e-04, 1.1922e-04, 6.3980e-04, 1.7515e-04, 4.5291e-05,\n",
            "        3.4263e-04, 2.6211e-04, 3.2651e-04, 1.5376e-05, 5.0572e-05, 2.5399e-04,\n",
            "        5.0241e-04, 4.5698e-05, 2.5244e-06, 1.3930e-04, 1.5724e-07, 5.1849e-04,\n",
            "        1.4752e-04, 6.2816e-05, 2.8721e-05, 6.7027e-05, 7.0230e-05, 2.8002e-04,\n",
            "        3.9358e-04, 7.8467e-06, 1.2077e-04, 2.9407e-04, 1.0311e-04, 5.7000e-04,\n",
            "        2.6183e-04, 1.2045e-04, 1.2458e-03, 6.9745e-04, 2.8346e-07, 3.1369e-07,\n",
            "        1.6065e-04, 4.3904e-04, 2.3570e-04, 1.5289e-04, 4.7216e-04, 1.7600e-03,\n",
            "        6.5281e-04, 1.8267e-03, 6.2636e-05, 2.8696e-04, 1.4921e-06, 1.8300e-04,\n",
            "        1.8129e-04, 1.2594e-04, 1.4493e-08, 4.1941e-05, 1.8718e-04, 4.8401e-04,\n",
            "        2.6417e-04, 9.4321e-05, 1.0782e-03, 2.2164e-04, 5.1378e-04, 8.2616e-05,\n",
            "        3.8294e-04, 9.1665e-05, 1.0390e-04, 2.7038e-07, 2.0514e-03, 1.0549e-03,\n",
            "        3.8168e-05, 2.2824e-04, 3.1583e-04, 5.6601e-04, 1.1953e-12, 1.6108e-04,\n",
            "        6.3969e-04, 8.6334e-06, 6.8710e-03, 2.4462e-04, 5.5193e-05, 1.3665e-06,\n",
            "        8.6563e-11, 5.2075e-04, 2.8554e-04, 2.0063e-16, 5.8663e-05, 9.2561e-04,\n",
            "        2.3647e-04, 4.6786e-04, 2.5829e-04, 1.5419e-04, 8.7421e-05, 3.5077e-04,\n",
            "        1.6264e-04, 4.0930e-05, 1.6755e-05, 5.2979e-11, 1.6292e-04, 1.7760e-04,\n",
            "        1.2864e-04, 1.1824e-04, 1.3551e-06, 6.2619e-06, 8.1790e-04, 1.5693e-05,\n",
            "        1.6016e-04, 1.9500e-04], device='cuda:0')}, 8: {'step': tensor(1920.), 'exp_avg': tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 3.2151e-10, 4.7531e-09,\n",
            "         3.3656e-09],\n",
            "        ...,\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 5.6052e-45, 5.6052e-45,\n",
            "         5.6052e-45]], device='cuda:0'), 'exp_avg_sq': tensor([[0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 1.6201e-13, 2.4327e-13,\n",
            "         3.8381e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.5525e-13, 3.6317e-13,\n",
            "         3.4274e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 2.3927e-09, 4.5701e-09,\n",
            "         5.3099e-09],\n",
            "        ...,\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 4.7059e-13, 4.0209e-13,\n",
            "         3.0205e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 9.0464e-14, 1.7059e-13,\n",
            "         3.5410e-13],\n",
            "        [0.0000e+00, 0.0000e+00, 0.0000e+00,  ..., 3.6768e-11, 1.0212e-10,\n",
            "         1.3188e-10]], device='cuda:0')}, 9: {'step': tensor(1920.), 'exp_avg': tensor([ 5.6052e-45,  5.6052e-45, -2.8088e-07,  5.6052e-45,  3.5345e-07,\n",
            "         5.6052e-45, -8.8008e-17, -5.2413e-09, -2.9867e-08, -5.5406e-08,\n",
            "         5.6052e-45,  5.6052e-45, -1.3142e-07,  5.6052e-45, -1.8576e-08,\n",
            "         5.6052e-45, -3.7381e-08, -2.5968e-07, -1.0773e-08,  5.6052e-45,\n",
            "         5.6052e-45,  4.1422e-08,  5.6052e-45, -1.2963e-07,  4.4890e-07,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -7.5288e-08,  4.0809e-08,\n",
            "         5.6052e-45,  5.6052e-45, -3.7727e-08,  5.6052e-45,  5.6052e-45,\n",
            "         7.7274e-08,  5.6052e-45, -1.3067e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -2.1280e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -7.7154e-08, -4.1719e-07,  5.6052e-45,  5.6052e-45,\n",
            "         2.6260e-07,  5.6052e-45, -1.4027e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -9.4945e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.0635e-07, -1.9612e-07,  1.1029e-07, -8.6121e-09,  5.6052e-45,\n",
            "         4.3321e-36,  5.6052e-45,  5.6052e-45,  5.6052e-45, -6.8288e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -8.9211e-08,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  3.1726e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -7.2555e-08, -1.8095e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -2.0694e-07,  5.6052e-45,  5.6052e-45,  2.9903e-40,\n",
            "        -1.1179e-16,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  1.3550e-07,  1.9022e-29, -2.1126e-08, -5.0369e-08,\n",
            "         5.6052e-45, -3.1718e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         6.8802e-38,  4.7559e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -8.0513e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  7.7301e-08,  1.8170e-08,\n",
            "         1.0970e-08,  5.6052e-45, -3.6103e-07, -4.8644e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -3.1252e-09,  2.7887e-09,  9.4459e-08,\n",
            "         5.6052e-45,  5.6052e-45, -1.2406e-07,  5.6052e-45,  5.6052e-45,\n",
            "        -2.3496e-07,  5.6052e-45, -3.7453e-07, -1.4366e-07,  5.6052e-45,\n",
            "         5.6052e-45, -2.5462e-07,  5.6052e-45,  5.6052e-45, -1.5947e-07,\n",
            "        -2.0019e-07,  5.6052e-45, -5.3826e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -4.8504e-08, -1.5304e-34,  4.1630e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  1.1712e-07,\n",
            "         5.6052e-45,  5.6052e-45, -2.5880e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -6.8192e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -5.4777e-08,  1.9605e-36,  8.2112e-08,  5.6052e-45, -1.0364e-08,\n",
            "         5.6052e-45, -3.0110e-07,  5.6052e-45,  1.5900e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -2.3397e-07,\n",
            "         5.6052e-45, -7.1517e-09, -1.1536e-07,  5.6052e-45,  5.6052e-45,\n",
            "        -2.9520e-08,  5.6052e-45,  5.6052e-45, -2.2537e-07, -1.2070e-07,\n",
            "         5.6052e-45,  5.6052e-45, -9.6025e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  2.8289e-12,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -8.8401e-08, -8.7024e-08, -2.5848e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45, -1.9390e-08, -1.0072e-07, -1.8746e-09,\n",
            "        -5.5343e-08,  5.6052e-45, -2.3316e-07,  5.6052e-45, -2.8512e-07,\n",
            "         5.6052e-45,  5.6052e-45, -1.1579e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.3652e-08,  5.6052e-45, -6.5299e-08, -7.0888e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -6.7796e-07,  5.6052e-45,\n",
            "         5.6052e-45, -9.4855e-09, -1.3231e-08, -1.1078e-07, -4.4066e-16,\n",
            "         5.6052e-45,  7.4767e-08, -9.5429e-08,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  8.8176e-08, -1.6111e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.3712e-07,  5.6052e-45,  5.6052e-45,  3.6645e-08,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -3.0244e-08,  5.6052e-45,  5.6052e-45, -4.0009e-08,\n",
            "         5.6052e-45, -2.4618e-09, -2.8133e-08, -3.4168e-09,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -1.5991e-08,  5.6052e-45,\n",
            "        -7.5107e-08,  5.6052e-45,  5.6052e-45, -4.8448e-08,  1.2800e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -9.8662e-08,\n",
            "        -3.0874e-08, -1.8804e-07,  5.6052e-45,  5.6052e-45, -2.0318e-07,\n",
            "         5.6052e-45, -7.5433e-07, -1.9811e-15,  5.6052e-45, -4.9909e-07,\n",
            "        -3.7516e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         7.0205e-09,  5.6052e-45,  4.4506e-12,  5.6052e-45, -4.3186e-07,\n",
            "        -1.0606e-07, -4.1156e-07, -1.0645e-07,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -5.4061e-08, -8.8578e-08, -1.1805e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.3649e-42,  7.9227e-09,  5.6052e-45,  5.6052e-45, -1.5674e-07,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -3.0310e-08,  1.3830e-07,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -5.3725e-08,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -1.6927e-08,\n",
            "        -9.1075e-08,  4.1109e-08,  5.6052e-45, -1.6740e-07, -4.9385e-08,\n",
            "        -1.3888e-07, -9.7392e-08,  5.6052e-45,  5.6052e-45, -1.8026e-07,\n",
            "         5.6052e-45, -4.6457e-09,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  1.5486e-07,  5.6052e-45, -7.9703e-09,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45, -2.4018e-07,\n",
            "         5.6052e-45, -1.2072e-08, -5.3552e-08, -6.6488e-09,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -8.8307e-08,  5.6052e-45, -1.0845e-07,  5.6052e-45,  5.6052e-45,\n",
            "        -5.8368e-11,  7.3434e-08,  1.0983e-07, -8.0880e-08, -3.5850e-18,\n",
            "         5.6052e-45,  5.6052e-45,  9.8721e-08,  5.6052e-45, -8.5457e-07,\n",
            "        -2.3591e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.3703e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45, -2.6119e-07,\n",
            "        -7.4409e-42, -9.7621e-08, -1.2425e-17, -1.8263e-07, -5.5764e-08,\n",
            "         5.6052e-45, -2.6744e-07,  5.6052e-45, -8.1387e-08,  5.6052e-45,\n",
            "        -1.6563e-09, -5.1245e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -8.3644e-08,  1.6989e-20,  9.0780e-08,  5.6052e-45,  5.6052e-45,\n",
            "        -1.7714e-10, -1.6859e-08,  6.0406e-08, -8.6140e-08,  5.6052e-45,\n",
            "        -1.7005e-07, -2.9557e-20,  5.6052e-45, -3.8010e-07,  2.7280e-08,\n",
            "        -4.6774e-08,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "        -1.8815e-08, -9.4587e-08, -1.6428e-07,  5.6052e-45, -5.4576e-08,\n",
            "         5.6052e-45,  5.6052e-45, -1.3043e-06,  5.6052e-45, -4.9776e-08,\n",
            "        -1.1922e-07, -2.7546e-07, -2.3192e-07,  5.6052e-45, -4.3974e-08,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45,  5.6052e-45, -1.2224e-07, -1.1323e-07,\n",
            "         5.6052e-45,  2.1149e-07, -1.3909e-07,  5.6052e-45, -1.1117e-07,\n",
            "        -3.0972e-07,  5.6052e-45,  5.6052e-45,  5.6052e-45,  5.6052e-45,\n",
            "         5.6052e-45, -1.1805e-07,  5.6052e-45,  1.5881e-07,  5.6052e-45,\n",
            "         5.6052e-45,  5.6052e-45], device='cuda:0'), 'exp_avg_sq': tensor([1.8175e-10, 1.1196e-10, 7.1843e-07, 7.1927e-12, 1.0045e-06, 3.2982e-10,\n",
            "        1.1819e-08, 3.7170e-08, 1.7375e-07, 3.2047e-08, 1.6811e-09, 6.1353e-11,\n",
            "        3.3786e-07, 3.2943e-10, 5.6780e-08, 2.4521e-09, 8.1126e-08, 2.9364e-07,\n",
            "        1.0133e-07, 1.5154e-12, 2.9167e-13, 3.6427e-07, 4.6549e-11, 1.3816e-07,\n",
            "        6.0473e-07, 2.3099e-13, 4.6199e-09, 9.0676e-12, 1.9594e-09, 2.5258e-12,\n",
            "        9.3337e-12, 4.4748e-11, 8.4461e-11, 6.6691e-07, 9.7378e-07, 1.9565e-12,\n",
            "        8.1170e-10, 1.6270e-08, 1.1967e-10, 1.9722e-10, 1.4629e-07, 8.3807e-10,\n",
            "        2.0802e-07, 2.5020e-10, 7.4600e-10, 2.6600e-09, 2.0196e-12, 1.2354e-12,\n",
            "        1.0673e-10, 1.1090e-09, 1.5411e-14, 1.2915e-07, 5.4990e-12, 1.2012e-11,\n",
            "        2.6593e-10, 7.3222e-12, 3.3601e-07, 3.0776e-07, 6.6609e-14, 4.9000e-09,\n",
            "        2.9222e-07, 8.8280e-10, 1.4777e-07, 3.9350e-12, 3.0584e-12, 1.7689e-10,\n",
            "        4.3176e-07, 8.7164e-11, 1.0408e-12, 4.2906e-11, 8.6088e-08, 3.0501e-07,\n",
            "        1.3506e-07, 4.6614e-07, 2.1738e-09, 2.7586e-09, 1.9142e-09, 4.4801e-10,\n",
            "        2.8018e-12, 4.5561e-07, 2.4359e-14, 7.0407e-11, 5.5491e-12, 4.8432e-07,\n",
            "        3.0638e-10, 1.6738e-09, 1.8710e-09, 8.3112e-09, 1.3235e-10, 9.2069e-11,\n",
            "        2.4966e-13, 6.3768e-11, 1.3381e-06, 5.2963e-11, 3.0314e-13, 1.0094e-09,\n",
            "        3.3883e-07, 2.3848e-07, 2.7675e-09, 1.1404e-09, 4.7400e-13, 7.1340e-08,\n",
            "        4.4979e-11, 1.5204e-09, 1.2619e-08, 2.0172e-08, 7.2272e-12, 9.6916e-12,\n",
            "        3.1574e-12, 3.7109e-09, 4.9893e-15, 1.5001e-07, 5.5829e-09, 1.1421e-07,\n",
            "        4.1055e-07, 6.3459e-12, 2.4453e-07, 1.0719e-10, 5.8161e-09, 9.7510e-13,\n",
            "        8.1119e-10, 1.3237e-07, 9.7787e-10, 5.7638e-12, 3.1876e-10, 2.7311e-10,\n",
            "        9.6378e-10, 4.6527e-08, 8.7848e-11, 4.7338e-11, 6.4376e-11, 3.4736e-12,\n",
            "        1.0660e-09, 5.5185e-07, 8.6038e-08, 1.0393e-07, 3.7276e-10, 1.1982e-07,\n",
            "        1.2465e-07, 9.2043e-11, 4.1756e-09, 1.1948e-11, 9.5559e-08, 5.2748e-08,\n",
            "        1.0651e-07, 7.4982e-11, 3.3579e-10, 2.0648e-07, 1.5845e-09, 2.8503e-14,\n",
            "        3.9808e-07, 1.7101e-09, 6.5325e-07, 1.2656e-07, 3.8740e-12, 3.6673e-13,\n",
            "        2.6642e-07, 2.0770e-09, 1.2671e-09, 6.1093e-07, 1.6483e-07, 1.1471e-09,\n",
            "        5.4675e-08, 7.9149e-10, 1.2436e-09, 2.1868e-13, 8.9554e-13, 1.1598e-07,\n",
            "        5.4408e-09, 1.8283e-07, 5.6142e-11, 1.3629e-10, 2.0430e-12, 4.0562e-10,\n",
            "        1.2403e-07, 5.2884e-10, 1.9159e-13, 4.7881e-08, 1.5216e-12, 6.7029e-11,\n",
            "        2.5687e-12, 1.6576e-10, 2.9868e-11, 2.9231e-07, 1.8286e-10, 5.2981e-10,\n",
            "        4.1780e-11, 4.9847e-11, 7.4507e-10, 8.4047e-10, 1.3654e-07, 3.3831e-10,\n",
            "        4.5475e-08, 5.4344e-10, 1.3483e-07, 1.9088e-11, 8.3741e-07, 1.3044e-10,\n",
            "        3.6507e-07, 7.4308e-14, 8.0144e-09, 2.1320e-11, 8.7917e-12, 8.5668e-12,\n",
            "        6.6994e-08, 6.7024e-12, 3.2250e-08, 2.4319e-07, 1.3866e-11, 2.4112e-10,\n",
            "        1.7753e-07, 3.9954e-10, 1.7584e-10, 2.2913e-07, 2.0428e-07, 1.0321e-11,\n",
            "        1.3044e-09, 3.6242e-07, 7.0789e-10, 3.6249e-12, 6.6525e-11, 3.9823e-12,\n",
            "        2.3799e-08, 2.9694e-11, 2.1889e-12, 2.1690e-10, 8.6927e-08, 3.5083e-07,\n",
            "        1.4259e-07, 1.3001e-09, 4.4068e-11, 3.9787e-12, 1.4067e-08, 1.8961e-07,\n",
            "        2.8926e-08, 1.1151e-07, 1.1961e-10, 3.3740e-07, 3.6861e-11, 1.0437e-07,\n",
            "        6.3893e-11, 1.0292e-10, 1.6145e-07, 2.0704e-09, 7.8648e-10, 5.5267e-10,\n",
            "        1.4825e-07, 3.8940e-10, 1.1123e-07, 5.0301e-08, 3.3289e-10, 3.3161e-09,\n",
            "        1.1694e-09, 3.7892e-07, 2.1457e-10, 6.6675e-10, 4.1203e-08, 1.6028e-07,\n",
            "        1.8425e-07, 8.1798e-09, 1.8576e-10, 1.1372e-07, 6.5299e-08, 1.4079e-09,\n",
            "        3.5512e-10, 8.3230e-10, 1.9688e-10, 2.9267e-07, 2.2528e-07, 3.2408e-12,\n",
            "        8.5253e-09, 3.9427e-09, 3.1691e-10, 2.9713e-13, 1.9134e-12, 1.5744e-09,\n",
            "        1.1899e-10, 8.4111e-13, 6.0599e-10, 2.2400e-09, 5.9309e-08, 1.4135e-10,\n",
            "        2.8683e-09, 8.0040e-07, 2.8083e-13, 1.2178e-09, 4.7913e-10, 1.1080e-09,\n",
            "        8.6273e-10, 4.4805e-09, 8.4133e-10, 1.1912e-07, 1.2181e-11, 5.4007e-13,\n",
            "        1.0766e-07, 1.2858e-12, 1.9983e-08, 6.0205e-07, 7.4677e-09, 1.3852e-09,\n",
            "        1.0227e-09, 3.0579e-11, 1.0451e-09, 1.0831e-07, 1.8995e-12, 1.4634e-07,\n",
            "        1.9380e-10, 8.0997e-10, 2.9543e-08, 5.7872e-07, 2.2177e-10, 2.3276e-10,\n",
            "        9.8717e-11, 8.5489e-10, 1.1030e-07, 1.4184e-07, 3.2851e-07, 2.7047e-11,\n",
            "        5.6914e-11, 6.7272e-07, 8.3687e-11, 5.8490e-07, 3.5221e-09, 7.1403e-11,\n",
            "        3.7204e-07, 1.7962e-07, 3.2907e-11, 8.5262e-11, 4.0045e-11, 1.4579e-12,\n",
            "        6.1293e-08, 1.6667e-12, 4.9582e-09, 1.5275e-13, 5.2577e-07, 3.7649e-07,\n",
            "        3.5198e-09, 3.2420e-07, 1.4220e-08, 1.9954e-11, 3.0781e-10, 7.4702e-08,\n",
            "        2.7275e-07, 1.4945e-07, 6.8537e-11, 1.3062e-09, 6.4105e-12, 3.6779e-10,\n",
            "        4.0695e-11, 2.7636e-10, 1.0896e-12, 1.7365e-07, 5.4300e-11, 1.7637e-13,\n",
            "        1.0240e-07, 3.8291e-10, 3.2219e-10, 6.4651e-11, 4.5746e-07, 1.5749e-07,\n",
            "        9.3676e-12, 6.1561e-10, 6.7153e-10, 4.3075e-08, 8.3952e-10, 3.3720e-11,\n",
            "        6.1937e-10, 8.0353e-12, 1.1318e-09, 2.4816e-07, 2.1253e-07, 1.8406e-07,\n",
            "        3.1753e-11, 5.0488e-07, 9.3754e-08, 3.0746e-07, 8.0960e-07, 4.7360e-11,\n",
            "        2.3098e-10, 2.6794e-07, 6.0479e-11, 7.7424e-08, 3.3792e-10, 5.7836e-11,\n",
            "        7.0772e-14, 2.4015e-11, 2.6388e-10, 3.7243e-12, 2.3119e-10, 9.5944e-12,\n",
            "        4.0236e-11, 2.0744e-12, 3.0749e-07, 4.7402e-10, 1.6073e-07, 6.4971e-12,\n",
            "        2.2928e-13, 1.0345e-11, 2.6869e-10, 2.7131e-07, 1.0980e-10, 4.2275e-08,\n",
            "        3.0284e-08, 2.5457e-08, 3.4984e-10, 1.0081e-10, 8.6345e-10, 1.8833e-10,\n",
            "        2.9048e-12, 8.2459e-10, 3.7348e-07, 8.8852e-11, 4.1992e-07, 2.0794e-11,\n",
            "        3.9484e-10, 1.4892e-08, 4.5895e-07, 4.6250e-07, 3.3415e-07, 5.3606e-09,\n",
            "        9.1213e-10, 4.1314e-10, 1.0849e-07, 1.0515e-12, 5.2706e-07, 2.4340e-07,\n",
            "        6.2064e-14, 3.1942e-10, 1.3517e-09, 3.0711e-11, 3.4649e-07, 4.7045e-09,\n",
            "        5.6463e-10, 2.7361e-09, 4.2932e-07, 6.8742e-09, 1.2498e-07, 3.0723e-09,\n",
            "        2.4816e-07, 2.2397e-07, 5.3841e-11, 2.4450e-07, 6.1444e-12, 1.7662e-07,\n",
            "        9.7853e-10, 3.0358e-08, 3.4887e-07, 7.7247e-12, 7.3748e-10, 3.4767e-12,\n",
            "        3.2877e-07, 9.9482e-09, 7.7772e-08, 3.2904e-09, 1.0398e-13, 1.8493e-08,\n",
            "        9.0804e-08, 1.0259e-07, 2.6572e-07, 5.1397e-10, 3.2234e-07, 2.2748e-09,\n",
            "        3.1418e-13, 3.1522e-06, 1.0374e-07, 8.1409e-08, 5.4961e-10, 7.0916e-11,\n",
            "        1.7342e-13, 1.8807e-09, 2.6229e-07, 4.0787e-08, 2.0345e-07, 2.2053e-10,\n",
            "        1.0024e-07, 1.3358e-11, 7.3021e-10, 1.1006e-06, 1.0286e-12, 1.5862e-07,\n",
            "        4.6932e-07, 3.2849e-07, 2.5973e-07, 3.0591e-14, 7.1764e-08, 1.3796e-10,\n",
            "        2.4791e-09, 1.3703e-11, 6.9228e-11, 1.0095e-11, 2.4073e-11, 3.6116e-10,\n",
            "        7.8101e-12, 2.1516e-08, 1.5498e-07, 3.4176e-11, 4.0481e-07, 1.9188e-07,\n",
            "        1.5351e-10, 1.3218e-07, 2.6437e-08, 5.8659e-10, 3.5837e-11, 2.9644e-11,\n",
            "        1.4594e-11, 6.4354e-11, 2.8798e-07, 4.9237e-11, 2.0112e-07, 3.1845e-11,\n",
            "        3.1369e-10, 3.8239e-09], device='cuda:0')}, 10: {'step': tensor(1920.), 'exp_avg': tensor([[ 0.0000e+00,  0.0000e+00, -5.0238e-30,  ...,  0.0000e+00,\n",
            "          0.0000e+00,  5.6052e-45],\n",
            "        [-5.6052e-45, -5.6052e-45, -8.2683e-07,  ..., -5.6052e-45,\n",
            "         -5.6052e-45,  5.6052e-45],\n",
            "        [ 0.0000e+00,  0.0000e+00, -9.4504e-08,  ...,  0.0000e+00,\n",
            "          5.6052e-45,  5.6052e-45],\n",
            "        ...,\n",
            "        [ 5.6052e-45, -5.6052e-45, -2.1020e-06,  ..., -5.6052e-45,\n",
            "          5.6052e-45,  5.6052e-45],\n",
            "        [ 5.6052e-45,  5.6052e-45,  2.1046e-07,  ...,  5.6052e-45,\n",
            "         -5.6052e-45,  5.6052e-45],\n",
            "        [ 5.6052e-45,  5.6052e-45,  5.6052e-45,  ...,  5.6052e-45,\n",
            "          5.6052e-45,  5.6052e-45]], device='cuda:0'), 'exp_avg_sq': tensor([[0.0000e+00, 0.0000e+00, 9.6366e-13,  ..., 0.0000e+00, 0.0000e+00,\n",
            "         2.2277e-09],\n",
            "        [2.8315e-15, 1.2421e-14, 1.1266e-06,  ..., 5.1316e-13, 2.0743e-13,\n",
            "         4.4270e-08],\n",
            "        [0.0000e+00, 0.0000e+00, 8.0259e-07,  ..., 0.0000e+00, 7.0816e-16,\n",
            "         5.8378e-09],\n",
            "        ...,\n",
            "        [1.5060e-14, 4.3494e-14, 5.3897e-06,  ..., 8.4541e-13, 3.2791e-13,\n",
            "         3.5727e-07],\n",
            "        [2.4110e-13, 6.3793e-15, 2.1063e-06,  ..., 2.0340e-12, 1.0879e-12,\n",
            "         7.3120e-08],\n",
            "        [2.6509e-13, 3.4684e-14, 1.0709e-11,  ..., 4.0154e-12, 1.7295e-12,\n",
            "         1.7083e-13]], device='cuda:0')}, 11: {'step': tensor(1920.), 'exp_avg': tensor([-2.5140e-11, -1.5801e-07, -5.4711e-08, -1.1062e-08, -3.9508e-10,\n",
            "        -3.8151e-07,  5.6052e-45, -5.6052e-45, -2.3397e-07, -3.5800e-07,\n",
            "         1.5782e-07,  2.3309e-36,  1.5668e-36, -1.9553e-07,  3.2582e-09,\n",
            "        -9.6083e-08, -3.5740e-10, -4.5451e-08, -2.8519e-07,  2.0646e-40,\n",
            "        -4.4676e-07, -1.6577e-07, -6.9549e-08, -1.2115e-08, -1.2608e-07,\n",
            "        -8.1256e-08, -6.2316e-07, -1.8267e-07,  5.5718e-08,  6.8844e-08,\n",
            "        -6.5610e-08,  2.5582e-07, -1.4824e-08,  1.0860e-22, -2.8974e-07,\n",
            "         5.6052e-45, -3.0075e-10, -1.8219e-07,  1.0823e-29,  5.6052e-45,\n",
            "        -2.7834e-07, -8.9397e-09, -4.8525e-07,  5.6052e-45, -3.9529e-07,\n",
            "        -1.0970e-07, -3.8112e-07,  5.6052e-45, -2.3831e-08,  5.6052e-45,\n",
            "        -1.5114e-07, -1.2972e-08, -1.6677e-07, -2.0791e-07, -2.3424e-08,\n",
            "        -3.5181e-08, -2.7034e-07, -3.2246e-08,  1.0267e-07, -4.9816e-09,\n",
            "        -5.8542e-08, -1.5545e-07, -9.2013e-07,  1.8014e-39, -1.5150e-12,\n",
            "         7.9472e-08, -5.2231e-07, -2.6458e-07, -2.1976e-07,  9.9819e-38,\n",
            "        -1.8105e-08, -2.5617e-20, -6.3803e-08, -6.2247e-09, -3.4210e-08,\n",
            "        -4.1557e-07, -4.0805e-07, -1.3457e-07, -8.1225e-08, -1.0693e-08,\n",
            "        -2.6101e-07,  6.9877e-08, -4.8446e-08,  1.2019e-08, -4.3635e-08,\n",
            "        -6.7550e-08, -6.7262e-07, -4.4926e-08,  6.2161e-09,  1.8347e-08,\n",
            "        -2.4085e-08,  5.6052e-45,  5.6052e-45, -2.6615e-14, -4.0488e-07,\n",
            "         6.3401e-08,  5.6052e-45, -2.4430e-08, -3.0418e-11, -1.1541e-07,\n",
            "        -1.0858e-07, -1.3711e-08,  2.2615e-39,  5.3314e-36, -3.0377e-08,\n",
            "         5.6052e-45, -1.6591e-07,  2.5585e-07, -1.7524e-07, -1.1278e-08,\n",
            "        -5.5820e-10, -1.7429e-08, -1.9367e-07, -1.1265e-07, -8.6009e-08,\n",
            "         5.6052e-45,  2.0526e-07,  1.3741e-07, -6.2071e-08, -6.0983e-08,\n",
            "         5.6052e-45, -1.7457e-07, -1.4901e-11, -7.4724e-12, -1.5544e-07,\n",
            "        -7.6967e-08,  5.4454e-16, -1.3218e-07, -1.5188e-07, -3.3889e-08,\n",
            "        -2.6306e-07, -4.5664e-07, -7.6783e-07, -1.2370e-07, -9.6043e-10,\n",
            "         5.6052e-45, -3.2016e-10,  5.6052e-45, -7.0995e-07, -3.3867e-07,\n",
            "        -1.6645e-07, -1.7496e-07, -7.4140e-07, -2.2127e-07, -6.3641e-08,\n",
            "         5.6052e-45, -3.0967e-07, -2.9952e-07, -1.5708e-07, -1.4480e-07,\n",
            "        -2.3088e-08, -6.0877e-10,  2.7769e-09,  1.5384e-30, -1.4687e-09,\n",
            "         1.8246e-07,  5.6052e-45, -2.1727e-07,  1.2164e-08, -6.1657e-44,\n",
            "        -7.3462e-11,  1.0808e-07, -3.1706e-07,  5.8836e-12, -2.4552e-12,\n",
            "        -1.8263e-07, -3.1480e-07, -2.6697e-08,  1.8904e-08, -2.0418e-07,\n",
            "         5.6052e-45,  8.1429e-10, -2.6622e-08, -1.3536e-07,  5.6052e-45,\n",
            "        -1.5417e-13, -8.2829e-08, -8.1149e-08, -2.8307e-09,  6.6344e-10,\n",
            "         5.6052e-45, -2.0120e-07,  1.6414e-07, -7.4694e-08, -1.1007e-08,\n",
            "        -4.2050e-08, -1.4888e-07,  2.8988e-32, -7.2581e-08, -6.0462e-10,\n",
            "        -5.4825e-07,  5.6052e-45, -2.8995e-08, -1.3868e-07, -4.8723e-08,\n",
            "         1.7200e-07, -3.3733e-08, -1.6681e-10, -8.4286e-08,  3.8818e-38,\n",
            "        -1.3776e-08, -3.0763e-11, -2.2994e-07, -2.0031e-07, -2.7236e-07,\n",
            "        -3.7058e-07, -9.2267e-09, -3.2288e-07, -1.7942e-07, -3.0790e-07,\n",
            "        -2.2409e-07, -9.5553e-11, -1.9941e-08, -1.6743e-07, -1.2216e-07,\n",
            "         1.2610e-07, -9.3054e-08,  2.0738e-18, -1.1640e-07,  4.6908e-08,\n",
            "        -1.6416e-10, -3.0619e-19,  8.5092e-35, -1.8265e-17, -1.0500e-07,\n",
            "        -2.2201e-07, -9.9618e-09, -7.1318e-10,  5.6052e-45, -7.2106e-08,\n",
            "         5.6052e-45, -2.2509e-07,  1.3620e-08, -2.5145e-08, -5.7494e-08,\n",
            "         4.0989e-14, -1.7695e-07, -2.4308e-09, -1.4172e-07,  8.6976e-26,\n",
            "        -1.2119e-07, -1.9608e-07, -2.3635e-08, -6.6048e-07,  1.8722e-11,\n",
            "         5.6052e-45,  5.9364e-08, -2.9305e-08, -3.8504e-08,  2.7709e-32,\n",
            "        -2.1504e-08, -1.0996e-07, -1.8373e-08, -4.7312e-07, -2.1291e-08,\n",
            "        -5.6052e-45], device='cuda:0'), 'exp_avg_sq': tensor([4.8186e-09, 1.9786e-07, 1.0181e-07, 3.3121e-08, 1.9950e-09, 1.4681e-06,\n",
            "        3.2836e-09, 1.4280e-08, 5.5806e-07, 3.6307e-07, 8.5814e-07, 1.2523e-09,\n",
            "        1.4653e-09, 3.0636e-07, 4.7887e-07, 3.5072e-07, 1.5220e-08, 2.0036e-06,\n",
            "        1.0527e-06, 4.4126e-11, 1.6446e-06, 1.4438e-07, 6.5334e-08, 6.0400e-09,\n",
            "        3.9584e-07, 1.3341e-07, 2.0436e-06, 7.2513e-07, 7.9297e-07, 6.4857e-07,\n",
            "        8.0234e-07, 2.3353e-06, 2.3097e-09, 3.0035e-09, 1.5867e-07, 3.0887e-09,\n",
            "        6.7285e-09, 3.0242e-08, 1.1566e-08, 3.7663e-09, 1.9597e-07, 3.9681e-07,\n",
            "        1.6995e-06, 2.9161e-10, 2.1327e-07, 8.1816e-07, 6.4745e-07, 4.9244e-10,\n",
            "        1.0899e-09, 1.2165e-09, 5.2704e-08, 2.8541e-08, 6.9546e-08, 4.2895e-09,\n",
            "        1.3691e-07, 2.2150e-08, 7.2632e-08, 5.9336e-08, 1.8061e-07, 5.8468e-09,\n",
            "        1.1583e-07, 4.0433e-07, 1.0976e-06, 2.1945e-10, 6.0740e-09, 5.0538e-07,\n",
            "        2.2823e-06, 2.9615e-07, 1.2272e-06, 2.3365e-08, 1.7658e-08, 1.7871e-08,\n",
            "        4.2300e-07, 2.7165e-07, 3.1740e-09, 1.1618e-06, 1.1266e-07, 4.2498e-08,\n",
            "        6.3214e-08, 3.1549e-08, 3.7186e-07, 4.8105e-07, 1.6925e-07, 3.4258e-07,\n",
            "        7.5741e-09, 2.8115e-07, 3.9666e-06, 3.0577e-07, 4.9913e-09, 6.7999e-07,\n",
            "        1.9902e-08, 1.1625e-08, 7.5782e-11, 4.9736e-09, 1.5249e-06, 7.5528e-07,\n",
            "        4.0502e-09, 2.8700e-08, 2.1063e-10, 1.2694e-07, 4.7964e-07, 1.3873e-07,\n",
            "        2.9453e-08, 2.9426e-09, 1.6311e-07, 2.8414e-08, 8.3369e-07, 2.3779e-06,\n",
            "        3.6029e-07, 1.6573e-09, 2.1718e-09, 2.7562e-07, 1.3586e-07, 1.9397e-08,\n",
            "        5.0360e-07, 2.6892e-10, 1.6692e-09, 8.1604e-07, 1.9002e-09, 6.1962e-08,\n",
            "        2.4408e-08, 6.5380e-07, 6.7344e-10, 1.4639e-08, 2.1625e-07, 4.0758e-07,\n",
            "        1.2574e-08, 4.5956e-07, 1.7927e-07, 2.7383e-08, 6.7540e-08, 4.8237e-07,\n",
            "        7.0889e-07, 7.8608e-07, 1.2987e-08, 4.5319e-10, 2.6481e-09, 1.1918e-08,\n",
            "        2.1268e-06, 6.7390e-08, 2.9367e-07, 4.9207e-07, 3.1487e-06, 6.8530e-08,\n",
            "        8.2976e-08, 1.8940e-09, 8.1913e-07, 3.4041e-07, 4.4659e-08, 2.4092e-06,\n",
            "        1.0751e-07, 2.4521e-08, 9.0339e-09, 6.1232e-12, 5.6566e-10, 1.5946e-06,\n",
            "        2.5504e-10, 3.8015e-07, 4.7192e-07, 8.6493e-09, 4.7383e-08, 1.7744e-06,\n",
            "        1.3713e-06, 8.7011e-09, 2.9893e-10, 3.7546e-07, 6.4665e-07, 2.3062e-08,\n",
            "        1.9148e-07, 1.2685e-07, 1.6429e-12, 1.8928e-08, 9.9374e-10, 2.7167e-07,\n",
            "        4.8450e-09, 4.0783e-09, 2.0168e-09, 4.5523e-08, 2.1869e-09, 9.6731e-11,\n",
            "        2.1830e-10, 8.1907e-07, 2.1682e-06, 3.0953e-10, 2.8030e-07, 7.5130e-08,\n",
            "        3.4218e-08, 3.9819e-09, 1.1366e-07, 1.1426e-08, 1.8486e-06, 1.3628e-08,\n",
            "        3.3172e-08, 3.8195e-08, 1.5549e-07, 1.3511e-06, 1.1287e-07, 5.2454e-09,\n",
            "        1.4207e-07, 2.6834e-10, 2.1750e-08, 5.6198e-10, 5.1474e-07, 8.6447e-07,\n",
            "        3.9847e-07, 3.4368e-07, 7.8729e-09, 2.1230e-07, 3.8500e-08, 5.4713e-07,\n",
            "        5.7793e-07, 1.4451e-08, 3.8232e-08, 6.7450e-07, 3.0505e-07, 2.3727e-06,\n",
            "        2.9903e-08, 1.4763e-08, 5.7740e-08, 3.9846e-07, 1.3806e-09, 1.0520e-10,\n",
            "        1.7068e-09, 3.6566e-08, 2.1062e-08, 3.7427e-07, 1.2357e-07, 5.4582e-09,\n",
            "        1.3656e-09, 4.6517e-07, 3.2824e-08, 8.9528e-07, 3.6479e-07, 2.8883e-09,\n",
            "        4.3501e-08, 1.0607e-09, 1.6861e-07, 4.3746e-09, 1.8903e-07, 1.4658e-08,\n",
            "        2.3366e-08, 9.6536e-08, 2.2619e-08, 3.0825e-06, 1.0439e-08, 7.9096e-09,\n",
            "        2.8393e-07, 1.3197e-06, 2.2237e-07, 4.1128e-09, 2.4161e-08, 1.4414e-07,\n",
            "        3.9619e-07, 1.1538e-06, 5.9279e-07, 2.6610e-08], device='cuda:0')}, 12: {'step': tensor(1920.), 'exp_avg': tensor([[ 4.0412e-10,  2.7548e-05,  2.1088e-06,  9.3537e-08, -1.8357e-08,\n",
            "          5.3770e-06,  5.6052e-45,  5.6052e-45,  2.6638e-05,  4.6122e-06,\n",
            "         -8.9158e-06,  1.2124e-37, -1.4411e-35,  3.1222e-05,  2.1600e-06,\n",
            "          1.1914e-05,  3.1921e-09, -1.2245e-05, -7.4120e-06,  2.7143e-42,\n",
            "          2.0993e-05,  5.9992e-07,  2.5727e-05,  5.1539e-09,  2.5886e-06,\n",
            "         -1.4350e-06,  2.5808e-05,  3.0342e-06, -1.7401e-05,  8.6468e-08,\n",
            "         -1.6310e-05, -5.8057e-06, -1.2132e-07,  4.6948e-22,  2.5175e-05,\n",
            "          5.6052e-45,  2.1788e-08, -1.9149e-06,  3.7463e-32,  5.6052e-44,\n",
            "          1.8732e-07,  8.7083e-06,  2.6206e-05,  5.6052e-45, -1.1704e-05,\n",
            "          2.1112e-06,  1.5944e-05,  5.6052e-45,  1.3655e-06, -5.6052e-45,\n",
            "          3.3261e-06, -6.2228e-07,  2.7552e-05,  7.8291e-06,  3.0376e-06,\n",
            "          2.2714e-06,  1.2335e-05, -3.9325e-06, -3.8371e-06,  7.0457e-10,\n",
            "         -1.2439e-07,  1.8692e-06,  2.7760e-05,  7.0203e-39,  1.2796e-11,\n",
            "         -3.9614e-07, -1.0988e-05,  2.6467e-05,  2.5446e-06, -2.6468e-37,\n",
            "         -8.7679e-07,  1.0194e-24, -4.0639e-07,  2.5699e-06,  2.3826e-08,\n",
            "         -7.5885e-06,  8.4254e-06, -2.5316e-07,  3.3544e-06,  3.8282e-06,\n",
            "          1.6176e-05, -7.9387e-06, -9.2516e-06,  1.5649e-05, -1.1435e-07,\n",
            "          1.2102e-07,  2.7412e-05,  3.5116e-06,  5.7560e-07,  2.9585e-06,\n",
            "          1.3265e-06,  5.6052e-45, -5.6052e-45,  3.9812e-12,  3.0659e-05,\n",
            "         -9.3502e-06,  5.6052e-45, -8.0102e-06,  2.1842e-09,  2.1326e-05,\n",
            "          2.6608e-05,  2.1508e-07,  1.8778e-39,  3.8804e-35,  2.3224e-06,\n",
            "         -5.6052e-45,  9.5939e-07, -7.2690e-06,  3.6339e-06, -2.3365e-06,\n",
            "         -4.8607e-09,  3.0874e-05,  3.3859e-06,  4.2306e-07,  3.2273e-06,\n",
            "         -5.6052e-45, -1.3885e-05, -5.9665e-06, -4.6883e-06,  1.3378e-05,\n",
            "          5.6052e-45,  4.2359e-07,  6.2958e-11,  8.1592e-10,  3.6891e-05,\n",
            "          4.0393e-06,  1.0226e-14, -6.8200e-06,  2.6752e-05,  2.6599e-08,\n",
            "          1.2743e-05,  1.1867e-05,  6.3114e-06, -5.2396e-06,  1.3832e-08,\n",
            "          5.6052e-45,  7.3546e-09,  5.6052e-45,  2.0569e-05,  2.6355e-05,\n",
            "         -2.5196e-06,  4.0829e-06,  2.7302e-05, -1.4553e-07,  3.5001e-06,\n",
            "          5.6052e-45,  2.4477e-05,  1.8377e-05,  1.2032e-05, -9.0317e-06,\n",
            "          6.5268e-07,  1.3725e-08,  6.2404e-09,  1.5936e-30,  3.5189e-08,\n",
            "         -1.1223e-05,  5.6052e-45,  2.2862e-06, -9.6213e-06,  2.3262e-43,\n",
            "          1.2067e-09, -1.2514e-05,  1.7176e-05,  9.2740e-13, -3.5621e-11,\n",
            "          1.6083e-06,  1.4752e-06,  2.3657e-06,  9.3892e-06,  2.0411e-05,\n",
            "          5.6052e-45, -5.8596e-08,  1.1915e-07,  5.9076e-07, -5.6052e-45,\n",
            "          8.4874e-13, -3.4689e-07, -1.1067e-08,  8.9440e-07,  5.3151e-10,\n",
            "         -5.6052e-45,  2.8310e-05, -4.3298e-06,  2.3106e-07,  1.3730e-06,\n",
            "          1.3983e-06,  1.3989e-05,  5.8810e-34,  2.0097e-05, -3.6359e-08,\n",
            "          2.4225e-05,  5.6052e-45,  1.1697e-05,  1.2269e-06,  2.2105e-05,\n",
            "         -4.9890e-06,  2.9432e-06,  1.3139e-07,  2.3510e-06, -1.3498e-38,\n",
            "         -2.3468e-09,  1.2872e-12,  2.9158e-05,  2.7108e-05, -3.2889e-06,\n",
            "          1.6928e-05,  3.9532e-09,  1.7167e-05, -8.4102e-06,  1.9764e-05,\n",
            "          2.7959e-05,  2.0804e-09,  2.1380e-06, -1.1228e-05,  2.6878e-06,\n",
            "         -1.2001e-05, -1.8008e-07,  2.3176e-17, -1.3832e-05, -7.0420e-06,\n",
            "         -2.2172e-10,  7.8528e-20,  1.2501e-37,  2.3608e-22,  8.9564e-07,\n",
            "          3.5463e-06, -1.1799e-07, -3.6387e-08, -5.6052e-45,  5.3417e-06,\n",
            "         -5.6052e-45,  3.0781e-05,  3.2382e-06,  2.2256e-09,  5.4047e-07,\n",
            "          1.3036e-14,  1.1764e-05,  4.0640e-09,  1.2460e-05,  1.5129e-25,\n",
            "          1.4062e-05,  1.7742e-05, -3.5554e-08,  2.5902e-05,  4.8513e-13,\n",
            "          5.6052e-45, -2.6356e-06,  5.2381e-06,  3.8892e-06,  1.9984e-37,\n",
            "          5.2024e-08,  3.0815e-06,  8.3539e-06,  1.7745e-05,  3.0788e-06,\n",
            "          2.7746e-43],\n",
            "        [-5.6056e-10, -2.7583e-05, -1.3476e-05, -2.8675e-07,  1.1015e-08,\n",
            "         -6.2527e-06, -5.6052e-45,  5.6052e-45, -3.8822e-05, -1.1377e-05,\n",
            "          8.4447e-06, -2.6780e-36,  1.7370e-35, -3.1404e-05, -1.0134e-05,\n",
            "         -2.1406e-05, -1.2246e-07,  1.5661e-05,  1.3801e-05, -7.3441e-39,\n",
            "         -2.1738e-05,  1.8670e-06, -2.5873e-05, -5.6281e-07, -1.5275e-05,\n",
            "          2.5067e-06, -3.8795e-05,  5.3205e-06,  1.8897e-05, -9.8675e-07,\n",
            "          1.7173e-05,  4.9625e-06,  7.3631e-07,  2.2339e-23, -2.5256e-05,\n",
            "         -5.6052e-45, -2.1823e-08,  2.1309e-06,  2.4182e-28, -5.3249e-44,\n",
            "          8.6421e-06, -9.7994e-06, -2.6277e-05, -5.6052e-45,  1.4739e-05,\n",
            "          3.5605e-06, -2.1655e-05, -5.6052e-45,  2.1416e-06,  5.6052e-45,\n",
            "          6.6815e-06,  6.2003e-07, -2.7677e-05, -1.2733e-05, -1.4585e-05,\n",
            "         -7.6273e-06, -1.2435e-05,  3.5807e-06,  3.7789e-06, -4.6803e-08,\n",
            "          3.3388e-07, -1.5230e-05, -2.8000e-05, -6.6286e-39,  9.1205e-18,\n",
            "         -7.0360e-07,  1.5393e-05, -2.7182e-05,  4.4971e-06,  2.6310e-37,\n",
            "          7.8866e-07,  4.1983e-22,  4.9611e-06, -9.7324e-06, -6.1855e-07,\n",
            "          1.1513e-05, -8.6458e-06,  8.2590e-06, -1.2376e-05, -4.1769e-06,\n",
            "         -2.0826e-05,  8.0907e-06,  7.5914e-06, -1.7205e-05,  1.3552e-06,\n",
            "          3.5268e-06, -3.3491e-05, -1.5562e-05, -5.7545e-07, -9.6618e-07,\n",
            "         -1.1680e-06, -5.6052e-45,  5.6052e-45, -5.1888e-12, -4.3745e-05,\n",
            "          8.6680e-06, -5.6052e-45,  7.4376e-06, -2.2584e-09, -2.1450e-05,\n",
            "         -2.9891e-05,  2.1708e-06,  1.6818e-38, -3.8804e-35,  3.4911e-06,\n",
            "          5.6052e-45, -1.2270e-05,  7.7599e-06, -2.3931e-05,  2.4347e-06,\n",
            "          4.0730e-09, -3.2177e-05,  5.8757e-06, -7.8045e-06, -1.7777e-05,\n",
            "          5.6052e-45,  1.1880e-05,  5.0946e-06,  4.4623e-06, -1.4018e-05,\n",
            "         -5.6052e-45, -1.2426e-05, -6.3036e-11, -8.3740e-10, -4.0565e-05,\n",
            "         -1.8329e-05, -9.5460e-26,  8.4565e-06, -2.7135e-05, -1.0362e-06,\n",
            "         -1.3104e-05, -1.4058e-05, -6.0252e-06,  8.4716e-06,  7.7250e-10,\n",
            "         -5.6052e-45, -7.2761e-09, -5.6052e-45, -2.7562e-05, -2.6367e-05,\n",
            "          2.3641e-06, -2.1228e-05, -3.9041e-05,  6.0762e-06, -1.2854e-05,\n",
            "          5.6052e-45, -3.4399e-05, -2.4592e-05, -1.1884e-05,  1.5474e-05,\n",
            "          1.1180e-06, -7.5767e-08, -8.9341e-09, -1.6154e-30,  4.5140e-07,\n",
            "          1.2125e-05, -5.6052e-45,  3.7517e-06,  1.0201e-05, -2.3262e-43,\n",
            "         -1.3670e-09,  1.2475e-05, -2.1374e-05,  3.1161e-11,  3.4609e-11,\n",
            "          3.0439e-06, -1.5215e-05,  3.5246e-06, -1.0258e-05, -2.0718e-05,\n",
            "         -5.6052e-45,  1.9869e-08,  6.5848e-07, -1.6241e-05,  5.6052e-45,\n",
            "         -9.9748e-13,  3.9958e-06,  1.3046e-06, -1.0988e-06,  1.9315e-08,\n",
            "          5.6052e-45, -4.4754e-05,  1.6598e-06,  5.0494e-06, -6.4287e-06,\n",
            "          2.9116e-06, -1.3996e-05,  1.6242e-32, -2.0718e-05,  2.9304e-08,\n",
            "         -3.1956e-05,  5.6052e-45, -1.2150e-05,  8.7743e-06, -2.3424e-05,\n",
            "          3.8249e-06,  3.5756e-06, -1.3371e-07,  3.8277e-06, -6.9202e-38,\n",
            "         -6.0722e-07,  1.9165e-08, -3.8077e-05, -2.7453e-05,  7.5933e-06,\n",
            "         -1.6985e-05, -6.5791e-07, -1.9327e-05,  7.5515e-06, -3.0876e-05,\n",
            "         -3.7640e-05, -2.1240e-09, -4.1861e-06,  1.1509e-05,  4.7283e-06,\n",
            "          1.2219e-05,  3.9100e-06, -5.9135e-43,  1.3615e-05,  6.0593e-06,\n",
            "          1.3274e-10, -5.6052e-45,  1.4548e-33,  7.9292e-18,  2.8348e-06,\n",
            "         -1.4185e-05,  9.1269e-08,  2.1807e-08,  5.6052e-45, -2.0129e-05,\n",
            "          5.6052e-45, -4.7010e-05,  1.6346e-06,  3.3443e-07,  1.6859e-06,\n",
            "         -2.3152e-25, -1.0327e-05, -7.2505e-08, -1.1827e-05, -1.6146e-25,\n",
            "         -1.4092e-05, -1.8787e-05,  4.6469e-07, -3.2735e-05,  7.9866e-10,\n",
            "          5.6052e-45,  1.9889e-06, -1.1091e-05, -6.6829e-06, -2.9123e-32,\n",
            "          1.3580e-06,  5.9506e-06, -1.3968e-05, -1.7898e-05, -4.6474e-06,\n",
            "         -3.0408e-43],\n",
            "        [ 1.7436e-10,  4.3354e-08,  1.1372e-05,  1.9335e-07,  7.3332e-09,\n",
            "          8.7911e-07,  5.6052e-45, -5.6052e-45,  1.2201e-05,  6.7697e-06,\n",
            "          4.8315e-07,  2.5568e-36, -2.9582e-36,  1.9333e-07,  7.9787e-06,\n",
            "          9.5080e-06,  1.1934e-07, -3.3989e-06, -6.3750e-06,  7.3414e-39,\n",
            "          7.5092e-07, -2.4657e-06,  1.5665e-07,  5.5806e-07,  1.2691e-05,\n",
            "         -1.0702e-06,  1.3000e-05, -8.3526e-06, -1.4779e-06,  9.1169e-07,\n",
            "         -8.4406e-07,  8.6087e-07, -6.1437e-07, -4.9181e-22,  8.6373e-08,\n",
            "          5.6052e-45,  2.1572e-10, -2.1581e-07, -2.4170e-28, -5.6052e-45,\n",
            "         -8.8226e-06,  1.1091e-06,  7.8348e-08,  5.6052e-45, -3.0203e-06,\n",
            "         -5.6698e-06,  5.7187e-06,  5.6052e-45, -3.5066e-06,  5.6052e-45,\n",
            "         -1.0004e-05,  2.3383e-09,  1.3315e-07,  4.9097e-06,  1.1561e-05,\n",
            "          5.3588e-06,  1.0369e-07,  3.5502e-07,  5.9409e-08,  4.6046e-08,\n",
            "         -2.0935e-07,  1.3364e-05,  2.4701e-07, -3.9165e-40,  1.4404e-13,\n",
            "          1.1148e-06, -4.3869e-06,  7.2418e-07, -7.0396e-06,  1.5761e-39,\n",
            "          8.9462e-08, -4.2534e-22, -4.5459e-06,  7.1684e-06,  5.9538e-07,\n",
            "         -3.9069e-06,  2.2311e-07, -7.9997e-06,  9.0259e-06,  3.5076e-07,\n",
            "          4.6563e-06, -1.4042e-07,  1.6765e-06,  1.5760e-06, -1.2399e-06,\n",
            "         -3.6427e-06,  6.0888e-06,  1.2056e-05,  5.1977e-10, -1.9896e-06,\n",
            "         -1.5690e-07,  5.6052e-45,  5.6052e-45,  5.3401e-13,  1.3103e-05,\n",
            "          6.9260e-07,  5.6052e-45,  5.8091e-07,  3.9424e-11,  1.3073e-07,\n",
            "          3.3127e-06, -2.3819e-06, -1.8696e-38,  6.1657e-44, -5.8118e-06,\n",
            "          5.6052e-45,  1.1312e-05, -4.7498e-07,  2.0303e-05, -9.6489e-08,\n",
            "          8.0256e-10,  1.3266e-06, -9.2588e-06,  7.3846e-06,  1.4555e-05,\n",
            "          5.6052e-45,  2.0285e-06,  8.8534e-07,  2.2696e-07,  6.4429e-07,\n",
            "          5.6052e-45,  1.2005e-05,  4.6097e-14,  2.5454e-11,  3.7024e-06,\n",
            "          1.4297e-05,  2.6005e-16, -1.6227e-06,  3.9562e-07,  1.0109e-06,\n",
            "          3.6546e-07,  2.1947e-06, -2.8317e-07, -3.2168e-06, -1.4606e-08,\n",
            "          5.6052e-45,  4.4721e-11,  5.6052e-45,  7.0006e-06,  1.8151e-08,\n",
            "          1.6006e-07,  1.7152e-05,  1.1751e-05, -5.9277e-06,  9.3585e-06,\n",
            "         -5.6052e-45,  9.9356e-06,  6.2240e-06, -1.4445e-07, -6.4291e-06,\n",
            "         -1.7692e-06,  6.2215e-08,  2.6564e-09,  3.7876e-32, -4.8657e-07,\n",
            "         -8.8370e-07,  5.6052e-45, -6.0361e-06, -5.6815e-07,  5.6052e-45,\n",
            "          1.8454e-10,  5.6029e-08,  4.2065e-06, -2.9837e-11,  4.1392e-13,\n",
            "         -4.6507e-06,  1.3742e-05, -5.8888e-06,  8.8065e-07,  3.1299e-07,\n",
            "          5.6052e-45,  3.8984e-08, -7.7703e-07,  1.5655e-05,  5.6052e-45,\n",
            "          1.0956e-13, -3.6471e-06, -1.2933e-06,  2.0485e-07, -1.9838e-08,\n",
            "          5.6052e-45,  1.6474e-05,  2.6852e-06, -5.2788e-06,  5.0577e-06,\n",
            "         -4.3083e-06,  1.1445e-08, -1.6830e-32,  6.3395e-07,  7.0556e-09,\n",
            "          7.7421e-06, -5.6052e-45,  4.5993e-07, -9.9965e-06,  1.3359e-06,\n",
            "          1.1771e-06, -6.5175e-06,  2.5424e-09, -6.1770e-06,  8.2700e-38,\n",
            "          6.1037e-07, -1.9055e-08,  8.9344e-06,  3.5401e-07, -4.2973e-06,\n",
            "          6.1253e-08,  6.5438e-07,  2.1687e-06,  8.6212e-07,  1.1121e-05,\n",
            "          9.7069e-06,  6.4005e-11,  2.0500e-06, -2.7122e-07, -7.4146e-06,\n",
            "         -2.0715e-07, -3.7276e-06,  1.5499e-23,  2.3194e-07,  9.9404e-07,\n",
            "          8.8894e-11,  8.8467e-19, -5.6052e-45, -2.3994e-19, -3.7290e-06,\n",
            "          1.0642e-05,  2.6813e-08,  1.4567e-08,  5.6052e-45,  1.4794e-05,\n",
            "          5.6052e-45,  1.6257e-05, -4.8710e-06, -3.3664e-07, -2.2257e-06,\n",
            "          4.2998e-14, -1.4325e-06,  6.8476e-08, -6.2828e-07,  1.0561e-26,\n",
            "          3.3767e-08,  1.0502e-06, -4.2892e-07,  6.8421e-06, -7.9871e-10,\n",
            "         -5.6052e-45,  6.5639e-07,  5.8695e-06,  2.7975e-06,  2.9124e-32,\n",
            "         -1.4097e-06, -9.0293e-06,  5.6334e-06,  1.5759e-07,  1.5705e-06,\n",
            "          5.6052e-45]], device='cuda:0'), 'exp_avg_sq': tensor([[2.0564e-08, 9.5689e-04, 1.9027e-05, 2.0620e-06, 3.9586e-08, 8.2411e-05,\n",
            "         2.6194e-09, 8.8574e-12, 1.6380e-03, 1.5191e-05, 5.1492e-04, 2.2084e-09,\n",
            "         2.5153e-06, 1.0350e-03, 4.5325e-05, 1.5228e-03, 1.9714e-08, 9.0862e-04,\n",
            "         4.3064e-04, 2.2715e-09, 3.0101e-04, 1.3427e-05, 1.1597e-03, 1.6216e-06,\n",
            "         4.3198e-05, 1.6626e-05, 8.2528e-04, 7.6085e-05, 5.3503e-04, 5.1190e-04,\n",
            "         1.1974e-03, 2.4969e-03, 2.0153e-05, 1.0580e-08, 1.7018e-04, 3.8593e-09,\n",
            "         1.5029e-05, 5.4735e-06, 8.4564e-10, 8.4965e-07, 2.8044e-04, 2.3429e-03,\n",
            "         7.4290e-04, 1.1636e-11, 3.2848e-04, 5.0211e-05, 1.5923e-04, 2.4523e-10,\n",
            "         1.7634e-06, 6.5616e-09, 3.8476e-05, 1.7198e-05, 4.6631e-04, 2.3174e-06,\n",
            "         1.2033e-03, 9.4070e-05, 1.4483e-04, 1.5844e-04, 5.3914e-05, 4.5596e-09,\n",
            "         1.9126e-06, 1.8856e-05, 1.8981e-04, 1.5215e-11, 3.5013e-05, 1.1583e-03,\n",
            "         5.4197e-04, 5.0745e-04, 8.7369e-05, 1.1780e-09, 3.0812e-05, 4.3173e-09,\n",
            "         3.8774e-04, 5.1447e-05, 4.8602e-08, 3.4474e-04, 4.2162e-05, 1.9210e-04,\n",
            "         4.8134e-05, 5.3172e-04, 2.3860e-04, 1.0387e-03, 3.4068e-04, 2.2950e-03,\n",
            "         1.4999e-06, 2.5330e-04, 5.9963e-04, 6.3672e-05, 2.5239e-05, 4.2861e-05,\n",
            "         1.7489e-05, 1.1119e-09, 1.8107e-10, 5.8322e-09, 1.5421e-03, 3.9769e-04,\n",
            "         3.7104e-10, 7.2391e-05, 5.2798e-08, 7.3906e-04, 3.8805e-03, 1.1472e-04,\n",
            "         7.5388e-09, 2.4570e-08, 4.4964e-05, 1.1698e-08, 2.4733e-05, 1.3709e-03,\n",
            "         2.7665e-05, 4.3139e-05, 9.4055e-10, 2.9562e-03, 5.2010e-05, 1.3105e-06,\n",
            "         6.4121e-05, 3.7502e-10, 5.3632e-06, 2.0638e-03, 9.7132e-07, 4.2057e-04,\n",
            "         1.4350e-09, 3.7811e-06, 1.9289e-09, 4.7837e-07, 4.3292e-03, 3.2596e-05,\n",
            "         2.2223e-07, 3.1298e-04, 1.5545e-03, 1.5367e-07, 2.2038e-05, 6.2850e-05,\n",
            "         7.2239e-05, 4.2688e-04, 2.0474e-10, 3.7933e-10, 4.0233e-06, 1.8452e-07,\n",
            "         2.7888e-04, 1.3928e-04, 5.3671e-05, 3.8001e-05, 6.2250e-04, 6.6687e-05,\n",
            "         2.3292e-05, 4.2988e-09, 1.3181e-03, 1.6464e-04, 2.1195e-05, 6.6068e-04,\n",
            "         4.6526e-05, 1.1437e-05, 3.1819e-09, 3.8372e-10, 2.3733e-06, 7.4544e-04,\n",
            "         6.6759e-13, 3.5340e-05, 5.6964e-04, 9.2722e-10, 2.4913e-08, 1.0954e-03,\n",
            "         2.4360e-04, 1.4798e-08, 6.5503e-10, 8.3302e-05, 9.1194e-06, 3.1814e-05,\n",
            "         1.1588e-03, 2.7801e-04, 7.5050e-13, 5.9678e-05, 7.5701e-06, 7.3361e-06,\n",
            "         6.3698e-10, 2.9855e-08, 8.5197e-07, 9.8900e-09, 2.1263e-05, 1.0815e-06,\n",
            "         4.8543e-09, 4.1901e-03, 1.9573e-03, 1.5241e-06, 1.3056e-05, 4.6050e-05,\n",
            "         6.1732e-05, 1.0279e-09, 1.4733e-03, 5.7847e-06, 8.2072e-04, 5.0610e-09,\n",
            "         1.0250e-03, 8.6415e-05, 2.2532e-03, 7.6523e-04, 4.5290e-05, 9.7882e-05,\n",
            "         6.5860e-05, 4.5352e-11, 1.3463e-06, 3.8976e-07, 1.6650e-03, 9.8983e-04,\n",
            "         3.2868e-04, 2.8032e-04, 3.5847e-07, 1.3465e-04, 1.9452e-05, 6.7373e-04,\n",
            "         2.9926e-03, 6.6029e-08, 2.1682e-05, 2.8422e-04, 4.9095e-05, 6.3672e-04,\n",
            "         8.0547e-10, 9.2969e-09, 2.6402e-04, 5.3494e-04, 9.7129e-07, 4.6188e-09,\n",
            "         2.9516e-10, 6.0926e-10, 9.8431e-06, 1.4513e-05, 3.0254e-06, 1.1305e-07,\n",
            "         8.2080e-11, 7.5525e-05, 1.0898e-09, 3.6903e-03, 9.4210e-05, 1.3767e-09,\n",
            "         1.3447e-07, 1.8258e-08, 6.9562e-05, 1.8380e-06, 2.2480e-04, 6.2654e-10,\n",
            "         6.2448e-05, 2.6624e-04, 1.4965e-06, 3.5803e-04, 8.3690e-09, 5.9005e-09,\n",
            "         1.2503e-04, 2.3371e-03, 1.1629e-04, 8.4688e-07, 1.9390e-07, 7.3117e-05,\n",
            "         1.9354e-03, 5.5616e-04, 5.3068e-05, 1.2262e-09],\n",
            "        [2.1083e-08, 9.5348e-04, 4.9460e-04, 1.9047e-05, 1.1126e-07, 1.6400e-04,\n",
            "         4.3983e-08, 8.4956e-11, 1.7825e-03, 1.2670e-04, 4.0965e-04, 6.4127e-09,\n",
            "         1.2897e-06, 1.0327e-03, 5.0334e-04, 1.4845e-03, 6.2038e-07, 6.4861e-04,\n",
            "         2.7313e-04, 2.8956e-08, 2.9919e-04, 8.1262e-06, 1.1573e-03, 1.2000e-05,\n",
            "         1.0046e-03, 8.0813e-06, 9.7427e-04, 2.0711e-04, 3.4541e-04, 4.7281e-04,\n",
            "         9.5619e-04, 2.1532e-03, 2.0888e-05, 4.9037e-08, 1.7229e-04, 2.7104e-08,\n",
            "         1.4966e-05, 7.8263e-06, 1.5597e-08, 7.9788e-07, 1.0090e-04, 2.2463e-03,\n",
            "         7.4316e-04, 3.4713e-13, 1.8054e-04, 5.9135e-05, 4.2344e-04, 2.7787e-09,\n",
            "         1.3135e-05, 6.6640e-09, 2.3853e-05, 1.7045e-05, 4.6552e-04, 8.6246e-06,\n",
            "         1.1239e-03, 1.3140e-04, 1.3790e-04, 1.4731e-04, 4.8975e-05, 4.0708e-07,\n",
            "         1.1705e-06, 6.2029e-04, 1.9273e-04, 3.1227e-12, 3.5749e-05, 1.0864e-03,\n",
            "         3.6385e-04, 5.2891e-04, 1.8237e-04, 1.2592e-09, 1.7726e-05, 4.8849e-09,\n",
            "         4.7338e-04, 5.7357e-04, 1.2726e-07, 1.3674e-04, 4.2493e-05, 1.0005e-04,\n",
            "         1.2189e-04, 5.2038e-04, 2.7338e-04, 8.7043e-04, 3.2142e-04, 2.2236e-03,\n",
            "         2.2046e-06, 2.7709e-04, 6.8764e-04, 1.0406e-03, 2.4040e-05, 2.6888e-04,\n",
            "         1.7683e-05, 1.6197e-09, 2.0506e-10, 1.1404e-09, 1.6622e-03, 3.7691e-04,\n",
            "         8.8593e-10, 6.7711e-05, 6.3411e-08, 7.3937e-04, 3.7258e-03, 8.1424e-05,\n",
            "         3.7609e-08, 2.7369e-08, 1.3260e-04, 1.1796e-08, 1.5735e-04, 1.1461e-03,\n",
            "         1.4894e-03, 3.0836e-05, 2.4474e-09, 2.9070e-03, 3.6717e-05, 1.3028e-04,\n",
            "         1.3843e-03, 2.6523e-10, 5.6156e-06, 1.8759e-03, 7.2868e-07, 4.2558e-04,\n",
            "         3.2027e-09, 1.5906e-04, 2.2412e-09, 3.9652e-07, 4.1273e-03, 8.1668e-04,\n",
            "         2.2552e-07, 2.2743e-04, 1.5431e-03, 4.2827e-06, 2.5620e-05, 2.1261e-04,\n",
            "         8.3190e-05, 3.9167e-04, 7.1636e-09, 2.0709e-09, 4.1637e-06, 2.3841e-07,\n",
            "         3.6004e-04, 1.3923e-04, 3.2660e-05, 1.1533e-03, 8.6175e-04, 4.5445e-05,\n",
            "         6.0285e-04, 4.5106e-09, 1.4037e-03, 4.9646e-04, 2.1733e-05, 4.4316e-04,\n",
            "         1.4865e-04, 1.6436e-05, 7.3752e-09, 8.1324e-11, 6.3640e-06, 5.3872e-04,\n",
            "         2.6645e-10, 5.7346e-05, 4.1532e-04, 1.2442e-09, 7.2634e-08, 8.1991e-04,\n",
            "         5.8334e-04, 3.2449e-06, 3.0689e-08, 8.1878e-05, 1.7820e-04, 3.3083e-05,\n",
            "         1.1054e-03, 2.7354e-04, 4.0864e-12, 5.4164e-05, 7.2561e-06, 3.7929e-04,\n",
            "         6.5771e-10, 4.0192e-08, 1.7279e-06, 3.4192e-07, 2.3933e-05, 4.9690e-07,\n",
            "         2.3613e-07, 4.1630e-03, 1.7574e-03, 1.7953e-06, 2.8606e-04, 6.6884e-05,\n",
            "         6.2977e-05, 7.7660e-08, 1.4371e-03, 6.4651e-06, 8.8207e-04, 1.8177e-08,\n",
            "         9.7341e-04, 3.2449e-05, 2.1811e-03, 6.3823e-04, 1.1374e-04, 8.9559e-05,\n",
            "         1.8941e-04, 1.1020e-09, 1.8645e-05, 1.8044e-06, 1.7328e-03, 9.8925e-04,\n",
            "         2.2656e-04, 2.8115e-04, 7.9182e-06, 2.9115e-04, 1.1972e-05, 8.0165e-04,\n",
            "         2.9223e-03, 8.2194e-08, 3.0346e-04, 1.9289e-04, 1.0086e-04, 5.3468e-04,\n",
            "         1.2146e-09, 1.0506e-08, 1.9112e-04, 4.8135e-04, 8.7191e-07, 4.3137e-09,\n",
            "         1.9167e-10, 5.1013e-08, 1.2401e-05, 1.5984e-04, 2.2073e-06, 1.1318e-07,\n",
            "         1.7688e-10, 1.5591e-03, 1.0520e-09, 3.8859e-03, 4.4371e-04, 3.0876e-09,\n",
            "         1.6879e-07, 3.0206e-08, 1.7017e-04, 1.8427e-06, 3.5089e-04, 8.6284e-10,\n",
            "         6.2627e-05, 3.8905e-04, 1.7959e-06, 5.4893e-04, 1.2339e-06, 5.9871e-08,\n",
            "         1.2019e-04, 2.2464e-03, 7.8987e-04, 1.0063e-06, 6.8999e-06, 1.9611e-04,\n",
            "         1.8104e-03, 5.6927e-04, 5.5413e-04, 1.1638e-09],\n",
            "        [6.0407e-11, 1.8660e-06, 5.0309e-04, 1.7219e-05, 1.0249e-07, 8.6351e-05,\n",
            "         4.4723e-08, 1.1316e-10, 1.5538e-04, 9.8826e-05, 6.3290e-05, 1.4619e-08,\n",
            "         5.4991e-07, 3.3297e-06, 5.5803e-04, 5.5841e-05, 5.9207e-07, 2.4534e-04,\n",
            "         2.7138e-04, 2.6609e-08, 8.4196e-06, 2.1396e-05, 4.2459e-06, 1.0844e-05,\n",
            "         1.0251e-03, 1.7066e-06, 1.3890e-04, 2.8683e-04, 1.4624e-04, 1.5714e-05,\n",
            "         1.7809e-04, 2.0127e-04, 1.1398e-06, 1.0147e-07, 1.2973e-06, 4.9536e-08,\n",
            "         2.0352e-09, 3.4348e-06, 1.8193e-08, 2.2515e-08, 2.6465e-04, 2.9158e-05,\n",
            "         3.9263e-06, 1.5823e-11, 1.3630e-04, 1.0173e-04, 2.6450e-04, 3.5917e-09,\n",
            "         1.4283e-05, 4.4022e-11, 6.1352e-05, 1.8254e-08, 1.1844e-06, 6.2706e-06,\n",
            "         9.7493e-05, 4.6568e-05, 7.9912e-06, 4.5869e-06, 5.3334e-06, 4.1472e-07,\n",
            "         1.1842e-06, 6.0542e-04, 1.8137e-06, 2.4931e-11, 3.4007e-07, 2.5305e-05,\n",
            "         1.9121e-04, 1.9757e-05, 2.6056e-04, 1.8752e-09, 9.3609e-06, 6.7431e-09,\n",
            "         6.0373e-04, 6.1885e-04, 8.1306e-08, 2.2512e-04, 5.5510e-07, 1.2135e-04,\n",
            "         7.0784e-05, 6.5129e-06, 4.6494e-05, 8.0232e-05, 1.7204e-05, 3.2157e-05,\n",
            "         8.3596e-07, 3.2367e-04, 7.0532e-05, 1.0992e-03, 6.4697e-07, 3.2079e-04,\n",
            "         1.7738e-07, 5.4157e-09, 2.9772e-11, 9.8696e-09, 1.3442e-04, 1.0408e-05,\n",
            "         2.3412e-09, 1.9749e-06, 1.1659e-08, 5.0806e-06, 6.7238e-05, 9.6834e-05,\n",
            "         2.2048e-08, 3.9000e-10, 1.6468e-04, 4.5303e-10, 1.3160e-04, 1.3658e-04,\n",
            "         1.4878e-03, 6.0265e-06, 6.4094e-09, 2.3743e-05, 9.2173e-05, 1.3126e-04,\n",
            "         1.4553e-03, 3.1201e-11, 6.5691e-08, 7.5753e-05, 1.1180e-07, 1.0673e-05,\n",
            "         8.3149e-09, 1.5335e-04, 7.7373e-09, 2.0589e-07, 6.9592e-05, 8.4716e-04,\n",
            "         1.1809e-09, 1.2073e-04, 6.3338e-06, 3.6619e-06, 3.4068e-06, 1.4835e-04,\n",
            "         9.2103e-06, 4.6631e-04, 7.2831e-09, 8.2617e-10, 1.3502e-07, 9.8678e-09,\n",
            "         7.4146e-05, 3.4402e-07, 1.0862e-05, 1.1538e-03, 2.1437e-04, 7.8278e-05,\n",
            "         6.1929e-04, 4.1381e-09, 9.2017e-05, 3.2168e-04, 1.8847e-07, 3.9889e-04,\n",
            "         1.1034e-04, 4.2630e-06, 1.1006e-08, 8.1782e-10, 5.7825e-06, 1.6232e-04,\n",
            "         2.6554e-10, 9.6720e-05, 1.6911e-04, 4.3545e-10, 4.2809e-08, 2.4102e-04,\n",
            "         3.4313e-04, 3.2561e-06, 3.0446e-08, 1.1330e-04, 1.6388e-04, 5.6356e-05,\n",
            "         2.4743e-05, 6.2148e-06, 1.3836e-12, 5.3020e-06, 5.4954e-06, 3.6479e-04,\n",
            "         2.5870e-09, 1.6858e-08, 2.9222e-06, 3.3724e-07, 5.5283e-06, 1.8642e-07,\n",
            "         2.4695e-07, 2.5243e-04, 1.1653e-04, 2.8468e-07, 2.8917e-04, 9.9886e-05,\n",
            "         1.1150e-06, 8.1423e-08, 1.3436e-05, 2.4828e-07, 6.8355e-05, 3.9898e-08,\n",
            "         1.8202e-05, 9.5609e-05, 2.2791e-05, 7.7612e-05, 1.6297e-04, 5.2551e-06,\n",
            "         2.5207e-04, 1.2576e-09, 1.8803e-05, 2.1870e-06, 9.6929e-05, 2.7980e-06,\n",
            "         1.7544e-04, 3.8863e-06, 7.6472e-06, 1.5451e-04, 6.0329e-06, 1.0640e-04,\n",
            "         1.0321e-04, 2.1553e-08, 3.2699e-04, 6.7489e-05, 1.5385e-04, 5.9080e-05,\n",
            "         3.7505e-09, 5.7037e-10, 5.5295e-05, 2.1253e-05, 1.0369e-07, 2.9688e-10,\n",
            "         7.6497e-10, 5.2850e-08, 2.2440e-05, 1.4061e-04, 1.8581e-06, 4.2336e-09,\n",
            "         4.9994e-10, 1.6125e-03, 4.2832e-09, 2.3468e-04, 5.4762e-04, 5.0418e-09,\n",
            "         1.9781e-08, 1.2127e-08, 1.0731e-04, 6.6998e-09, 1.6838e-04, 2.8148e-09,\n",
            "         1.1579e-07, 1.5959e-04, 5.1810e-07, 1.6169e-04, 1.2468e-06, 5.9095e-08,\n",
            "         2.1610e-05, 1.5289e-04, 9.0570e-04, 1.5933e-07, 6.5836e-06, 2.5824e-04,\n",
            "         7.9146e-05, 2.3815e-05, 6.1656e-04, 7.0927e-11]], device='cuda:0')}, 13: {'step': tensor(1920.), 'exp_avg': tensor([ 2.8580e-06, -3.6963e-06,  8.4277e-07], device='cuda:0'), 'exp_avg_sq': tensor([2.8075e-04, 2.8888e-04, 9.1226e-05], device='cuda:0')}}\n",
            "param_groups \n",
            "\n",
            " [{'lr': 0.001, 'betas': (0.9, 0.999), 'eps': 1e-08, 'weight_decay': 0.01, 'amsgrad': False, 'foreach': None, 'maximize': False, 'capturable': False, 'differentiable': False, 'fused': False, 'params': [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13]}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# correct = 0\n",
        "# total = 0\n",
        "# with torch.no_grad():\n",
        "#     for data in test_loader:\n",
        "#         images, labels = data\n",
        "#         images,labels = images.to(device),labels.to(device)\n",
        "#         outputs = model2(images)\n",
        "#         _, predicted = torch.max(outputs.data, 1)\n",
        "#         total += labels.size(0)\n",
        "#         correct += (predicted == labels).sum().item()\n",
        "\n",
        "# print('Accuracy of the network on the test images: %f %%'% (100 * correct / total))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "L_LREm7csngU",
        "outputId": "ac52f111-ef6a-4342-c42a-20714c2acdd2"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Accuracy of the network on the test images: 96.216569 %\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# model = TheModelClass(*args, **kwargs)\n",
        "# optimizer = TheOptimizerClass(*args, **kwargs)\n",
        "\n",
        "# checkpoint = torch.load(PATH)\n",
        "# model.load_state_dict(checkpoint['model_state_dict'])\n",
        "# optimizer.load_state_dict(checkpoint['optimizer_state_dict'])\n",
        "# epoch = checkpoint['epoch']\n",
        "# loss = checkpoint['loss']\n",
        "\n",
        "# model.eval()\n",
        "# # - or -\n",
        "# model.train()"
      ],
      "metadata": {
        "id": "ZN1RlvVKcGWB"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}