component_swap_test_gt.ipynb

{
  "cells": [
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "rbHij6RQTFny"
      },
      "source": [
        "# Component Swapping Experiments"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "/tmp/ipykernel_2703113/1711627342.py:5: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n",
            "  ipython.magic(\"load_ext autoreload\")\n",
            "/tmp/ipykernel_2703113/1711627342.py:6: DeprecationWarning: `magic(...)` is deprecated since IPython 0.13 (warning added in 8.1), use run_line_magic(magic_name, parameter_s).\n",
            "  ipython.magic(\"autoreload 2\")\n"
          ]
        }
      ],
      "source": [
        "from IPython import get_ipython\n",
        "from IPython.display import clear_output, display\n",
        "\n",
        "ipython = get_ipython()\n",
        "ipython.magic(\"load_ext autoreload\")\n",
        "ipython.magic(\"autoreload 2\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {},
      "outputs": [],
      "source": [
        "import os\n",
        "from typing import List, Optional, Union, Dict, Tuple\n",
        "\n",
        "import torch\n",
        "from torch import Tensor\n",
        "import numpy as np\n",
        "import einops\n",
        "from fancy_einsum import einsum\n",
        "import circuitsvis as cv\n",
        "\n",
        "import transformer_lens.utils as utils\n",
        "\n",
        "from transformer_lens import HookedTransformer\n",
        "import transformer_lens.patching as patching\n",
        "\n",
        "from transformers import AutoModelForCausalLM\n",
        "\n",
        "from torch import Tensor\n",
        "from jaxtyping import Float\n",
        "import plotly.express as px\n",
        "\n",
        "from functools import partial\n",
        "\n",
        "from torchtyping import TensorType as TT\n",
        "\n",
        "from path_patching_cm.path_patching import Node, IterNode, path_patch, act_patch\n",
        "from path_patching_cm.ioi_dataset import IOIDataset, NAMES\n",
        "from neel_plotly import imshow as imshow_n\n",
        "\n",
        "from utils.visualization import imshow_p, plot_attention_heads\n",
        "from utils.data_utils import generate_data_and_caches\n",
        "from utils.metrics import _logits_to_mean_logit_diff\n",
        "from utils.visualization import (\n",
        "    plot_attention_heads,\n",
        "    scatter_attention_and_contribution,\n",
        "    get_attn_head_patterns\n",
        ")\n",
        "\n",
        "from utils.circuit_utils import (\n",
        "    ComponentDict,\n",
        "    get_components_to_swap,\n",
        "    load_swapped_params\n",
        ")\n",
        "\n",
        "if torch.cuda.is_available():\n",
        "    device = \"cuda\"\n",
        "else:\n",
        "    device = \"cpu\""
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {},
      "outputs": [],
      "source": [
        "def imshow(tensor, renderer=None, xaxis=\"\", yaxis=\"\", **kwargs):\n",
        "    px.imshow(utils.to_numpy(tensor), color_continuous_midpoint=0.0, color_continuous_scale=\"RdBu\", labels={\"x\":xaxis, \"y\":yaxis}, **kwargs).show(renderer)\n",
        "\n",
        "def line(tensor, renderer=None, **kwargs):\n",
        "    px.line(y=utils.to_numpy(tensor), **kwargs).show(renderer)\n",
        "\n",
        "def two_lines(tensor1, tensor2, renderer=None, **kwargs):\n",
        "    px.line(y=[utils.to_numpy(tensor1), utils.to_numpy(tensor2)], **kwargs).show(renderer)\n",
        "\n",
        "def scatter(x, y, xaxis=\"\", yaxis=\"\", caxis=\"\", renderer=None, **kwargs):\n",
        "    x = utils.to_numpy(x)\n",
        "    y = utils.to_numpy(y)\n",
        "    px.scatter(y=y, x=x, labels={\"x\":xaxis, \"y\":yaxis, \"color\":caxis}, **kwargs).show(renderer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<torch.autograd.grad_mode.set_grad_enabled at 0x7fdd80f5ae80>"
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "torch.set_grad_enabled(False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {},
      "outputs": [],
      "source": [
        "from utils.data_utils import UniversalPatchingDataset\n",
        "from utils.circuit_utils import run_with_batches, get_data_and_metrics"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        }
      ],
      "source": [
        "\n",
        "model = HookedTransformer.from_pretrained(\n",
        "        \"EleutherAI/pythia-160m\",\n",
        "        center_unembed=True,\n",
        "        center_writing_weights=True,\n",
        "        fold_ln=True,\n",
        "        refactor_factored_attn_matrices=False,\n",
        "    )\n",
        "model.set_use_hook_mlp_in(True)\n",
        "model.tokenizer.add_bos_token = False"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [],
      "source": [
        "ds, metric = get_data_and_metrics(model, 'ioi')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {},
      "outputs": [],
      "source": [
        "ds = UniversalPatchingDataset.from_ioi(model, 100)\n",
        "batch_size = 100"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {},
      "outputs": [],
      "source": [
        "clean_logits = run_with_batches(model, ds.toks, batch_size, ds.max_seq_len)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "tensor(4.1922, device='cuda:0')"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "from utils.metrics import compute_logit_diff\n",
        "\n",
        "compute_logit_diff(clean_logits, ds.answer_toks, positions=ds.positions, flags_tensor=ds.group_flags)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Q5t_LmW3Tuej"
      },
      "source": [
        "## Model Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {},
      "outputs": [],
      "source": [
        "from typing import Callable\n",
        "\n",
        "def get_ioi_logit_diff_for_model(model: HookedTransformer):\n",
        "    N = 70\n",
        "    model.tokenizer.add_bos_token = False\n",
        "    ioi_dataset, abc_dataset = generate_data_and_caches(model, N, verbose=True)\n",
        "    clean_logits, clean_cache = model.run_with_cache(ioi_dataset.toks)\n",
        "    return _logits_to_mean_logit_diff(clean_logits, ioi_dataset)\n",
        "    \n",
        "\n",
        "def get_gt_prob_diff_for_model(model: HookedTransformer):\n",
        "    N = 1000\n",
        "    ds = UniversalPatchingDataset.from_greater_than(model, N)\n",
        "    batch_size = 100\n",
        "    logits = run_with_batches(model, ds.toks, batch_size, ds.max_seq_len)\n",
        "    return compute_probability_diff(logits, ds.answer_toks, positions=ds.positions, flags_tensor=ds.group_flags, mode=\"group_sum\")\n",
        "\n",
        "\n",
        "def run_swap_experiment_test(\n",
        "    model_name: str,\n",
        "    dataset_eval_fn: Callable, \n",
        "    source_checkpoint: int, \n",
        "    target_checkpoint: int, \n",
        "    heads: List[Tuple[int, int]],\n",
        "    include_ln: bool = False,\n",
        "    include_mlps: bool = False,\n",
        "    cache_dir: str = \"/mnt/hdd-0/circuits-over-time/model_cache/ct\"\n",
        "    ):\n",
        "\n",
        "    # get baseline logit diff metrics\n",
        "    model = HookedTransformer.from_pretrained(\n",
        "        \"EleutherAI/pythia-160m\",\n",
        "        checkpoint_value=target_checkpoint,\n",
        "        center_unembed=True,\n",
        "        center_writing_weights=True,\n",
        "        fold_ln=True,\n",
        "        refactor_factored_attn_matrices=False,\n",
        "    )\n",
        "    model.set_use_hook_mlp_in(True)\n",
        "    model.tokenizer.add_bos_token = False\n",
        "\n",
        "    baseline_metric = dataset_eval_fn(model)\n",
        "\n",
        "    source_model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_name, revision=f\"step{source_checkpoint}\", cache_dir=cache_dir\n",
        "    )\n",
        "\n",
        "    target_model = AutoModelForCausalLM.from_pretrained(\n",
        "        model_name, revision=f\"step{target_checkpoint}\", cache_dir=cache_dir\n",
        "    )\n",
        "\n",
        "    component_dict = ComponentDict(layer_heads=heads, include_ln=include_ln, include_mlps=include_mlps)\n",
        "\n",
        "    component_params = get_components_to_swap(source_model, component_dict, cache_dir)\n",
        "    load_swapped_params(target_model, component_params)\n",
        "\n",
        "    model = HookedTransformer.from_pretrained(\n",
        "        \"EleutherAI/pythia-160m\",\n",
        "        hf_model=target_model,\n",
        "        center_unembed=True,\n",
        "        center_writing_weights=True,\n",
        "        fold_ln=True,\n",
        "        refactor_factored_attn_matrices=False,\n",
        "    )\n",
        "    model.set_use_hook_mlp_in(True)\n",
        "    model.tokenizer.add_bos_token = False\n",
        "\n",
        "    swapped_metric = dataset_eval_fn(model)\n",
        "\n",
        "    return baseline_metric.item(), swapped_metric.item()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "\n",
        "def run_swap_grid_search(\n",
        "    model_name: str,\n",
        "    dataset_eval_fn: Callable,\n",
        "    source_range: Tuple[int, int],\n",
        "    target_range: Tuple[int, int],\n",
        "    heads: List[Tuple[int, int]]\n",
        "    ) -> pd.DataFrame:\n",
        "    \n",
        "    # Initialize an empty list to store the results\n",
        "    results = []\n",
        "\n",
        "    # Loop through the target and source ranges\n",
        "    for target in range(target_range[0], target_range[1], 1000):\n",
        "        for source in range(max(source_range[0], (target+1000)), source_range[1], 1000):\n",
        "            \n",
        "            # Run the experiment to get baseline and swapped values\n",
        "            baseline_ld, swapped_ld = run_swap_experiment_test(model_name, dataset_eval_fn, source, target, heads=heads)\n",
        "            \n",
        "            # Calculate the difference\n",
        "            difference = swapped_ld - baseline_ld\n",
        "            \n",
        "            # Append the results as a dictionary\n",
        "            results.append({\n",
        "                \"source\": source,\n",
        "                \"target\": target,\n",
        "                \"baseline_val\": baseline_ld,\n",
        "                \"swapped_val\": swapped_ld,\n",
        "                \"difference\": difference\n",
        "            })\n",
        "            \n",
        "            print(f\"Source: {source} -> Target: {target}\")\n",
        "\n",
        "    # Convert the results list to a DataFrame\n",
        "    results_df = pd.DataFrame(results)\n",
        "    \n",
        "    return results_df"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### IOI"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 27,
      "metadata": {},
      "outputs": [],
      "source": [
        "model_name = \"EleutherAI/pythia-160m\"\n",
        "whole_circuit = [(4, 6), (4, 11), (6, 6), (7, 2), (7, 9), (8, 9), (8, 2), (8, 10)]\n",
        "nmh = [(8, 2), (8, 10)]\n",
        "top_3 = nmh + [(8, 9)]\n",
        "s2i = [(6, 6), (7, 2), (7, 9)]\n",
        "csh = [(8, 9)]\n",
        "idh = [(4, 6), (4, 11)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 5000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 7000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 8000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 9000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 12000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 8000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 10000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 11000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 18000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 7000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 8000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 15000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 11000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 9000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 10000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 12000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 15000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 10000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 11000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 14000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 19000 -> Target: 9000\n"
          ]
        }
      ],
      "source": [
        "results = run_swap_grid_search(model_name, get_ioi_logit_diff_for_model, source_range=(4000, 20000), target_range=(1000, 10000), heads=whole_circuit)\n",
        "results['pct_change'] = results['difference'] / results['baseline_val']"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>target</th>\n",
              "      <th>baseline_val</th>\n",
              "      <th>swapped_val</th>\n",
              "      <th>difference</th>\n",
              "      <th>pct_change</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>4000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.315800</td>\n",
              "      <td>-0.013590</td>\n",
              "      <td>0.044967</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>5000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.316972</td>\n",
              "      <td>-0.014762</td>\n",
              "      <td>0.048848</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>6000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.316227</td>\n",
              "      <td>-0.014017</td>\n",
              "      <td>0.046382</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>7000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.310158</td>\n",
              "      <td>-0.007948</td>\n",
              "      <td>0.026300</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>8000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.308751</td>\n",
              "      <td>-0.006541</td>\n",
              "      <td>0.021645</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>9000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.306486</td>\n",
              "      <td>-0.004276</td>\n",
              "      <td>0.014150</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>10000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.310836</td>\n",
              "      <td>-0.008626</td>\n",
              "      <td>0.028543</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>11000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.309256</td>\n",
              "      <td>-0.007046</td>\n",
              "      <td>0.023314</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>12000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.313383</td>\n",
              "      <td>-0.011173</td>\n",
              "      <td>0.036972</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>13000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.307827</td>\n",
              "      <td>-0.005617</td>\n",
              "      <td>0.018588</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>14000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.301453</td>\n",
              "      <td>0.000757</td>\n",
              "      <td>-0.002504</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>15000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.303023</td>\n",
              "      <td>-0.000813</td>\n",
              "      <td>0.002689</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>16000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.296638</td>\n",
              "      <td>0.005572</td>\n",
              "      <td>-0.018436</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>17000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.287602</td>\n",
              "      <td>0.014608</td>\n",
              "      <td>-0.048336</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>18000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.290686</td>\n",
              "      <td>0.011524</td>\n",
              "      <td>-0.038132</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>19000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.302210</td>\n",
              "      <td>-0.285892</td>\n",
              "      <td>0.016318</td>\n",
              "      <td>-0.053994</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>4000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.676898</td>\n",
              "      <td>0.021004</td>\n",
              "      <td>-0.030096</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>5000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.682049</td>\n",
              "      <td>0.015853</td>\n",
              "      <td>-0.022715</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>6000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.657747</td>\n",
              "      <td>0.040155</td>\n",
              "      <td>-0.057536</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>7000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.672880</td>\n",
              "      <td>0.025021</td>\n",
              "      <td>-0.035852</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>8000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.655866</td>\n",
              "      <td>0.042035</td>\n",
              "      <td>-0.060231</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>9000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.701781</td>\n",
              "      <td>-0.003879</td>\n",
              "      <td>0.005558</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>10000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.734431</td>\n",
              "      <td>-0.036529</td>\n",
              "      <td>0.052341</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>11000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.795557</td>\n",
              "      <td>-0.097655</td>\n",
              "      <td>0.139927</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>12000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.883217</td>\n",
              "      <td>-0.185316</td>\n",
              "      <td>0.265533</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>13000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.858482</td>\n",
              "      <td>-0.160581</td>\n",
              "      <td>0.230091</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>14000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.882921</td>\n",
              "      <td>-0.185019</td>\n",
              "      <td>0.265108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>15000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.934700</td>\n",
              "      <td>-0.236799</td>\n",
              "      <td>0.339301</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>16000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.923487</td>\n",
              "      <td>-0.225586</td>\n",
              "      <td>0.323234</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>17000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.950743</td>\n",
              "      <td>-0.252842</td>\n",
              "      <td>0.362289</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>18000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-1.001919</td>\n",
              "      <td>-0.304017</td>\n",
              "      <td>0.435616</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>19000</td>\n",
              "      <td>2000</td>\n",
              "      <td>-0.697901</td>\n",
              "      <td>-0.964767</td>\n",
              "      <td>-0.266866</td>\n",
              "      <td>0.382383</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>4000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.178955</td>\n",
              "      <td>0.131673</td>\n",
              "      <td>2.784835</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>5000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.268404</td>\n",
              "      <td>0.221122</td>\n",
              "      <td>4.676648</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>6000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.299044</td>\n",
              "      <td>0.251762</td>\n",
              "      <td>5.324676</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>7000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.250925</td>\n",
              "      <td>0.203643</td>\n",
              "      <td>4.306972</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>8000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.239457</td>\n",
              "      <td>0.192175</td>\n",
              "      <td>4.064430</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>9000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.174952</td>\n",
              "      <td>0.127670</td>\n",
              "      <td>2.700167</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>10000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.117557</td>\n",
              "      <td>0.070275</td>\n",
              "      <td>1.486290</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>11000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.126258</td>\n",
              "      <td>0.078975</td>\n",
              "      <td>1.670302</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>12000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.011817</td>\n",
              "      <td>-0.059099</td>\n",
              "      <td>-1.249919</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>13000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>0.008462</td>\n",
              "      <td>-0.038821</td>\n",
              "      <td>-0.821040</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>14000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.009836</td>\n",
              "      <td>-0.057118</td>\n",
              "      <td>-1.208021</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>15000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.061643</td>\n",
              "      <td>-0.108925</td>\n",
              "      <td>-2.303733</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>16000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.037571</td>\n",
              "      <td>-0.084853</td>\n",
              "      <td>-1.794607</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>17000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.024498</td>\n",
              "      <td>-0.071780</td>\n",
              "      <td>-1.518119</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>18000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.092577</td>\n",
              "      <td>-0.139859</td>\n",
              "      <td>-2.957971</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>19000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.047282</td>\n",
              "      <td>-0.150977</td>\n",
              "      <td>-0.198259</td>\n",
              "      <td>-4.193102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>5000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.514669</td>\n",
              "      <td>0.106718</td>\n",
              "      <td>0.261594</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>6000</td>\n",
              "      <td>4000</td>\n",
              "      <td>0.407952</td>\n",
              "      <td>0.547358</td>\n",
              "      <td>0.139406</td>\n",
              "      <td>0.341722</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    source  target  baseline_val  swapped_val  difference  pct_change\n",
              "0     4000    1000     -0.302210    -0.315800   -0.013590    0.044967\n",
              "1     5000    1000     -0.302210    -0.316972   -0.014762    0.048848\n",
              "2     6000    1000     -0.302210    -0.316227   -0.014017    0.046382\n",
              "3     7000    1000     -0.302210    -0.310158   -0.007948    0.026300\n",
              "4     8000    1000     -0.302210    -0.308751   -0.006541    0.021645\n",
              "5     9000    1000     -0.302210    -0.306486   -0.004276    0.014150\n",
              "6    10000    1000     -0.302210    -0.310836   -0.008626    0.028543\n",
              "7    11000    1000     -0.302210    -0.309256   -0.007046    0.023314\n",
              "8    12000    1000     -0.302210    -0.313383   -0.011173    0.036972\n",
              "9    13000    1000     -0.302210    -0.307827   -0.005617    0.018588\n",
              "10   14000    1000     -0.302210    -0.301453    0.000757   -0.002504\n",
              "11   15000    1000     -0.302210    -0.303023   -0.000813    0.002689\n",
              "12   16000    1000     -0.302210    -0.296638    0.005572   -0.018436\n",
              "13   17000    1000     -0.302210    -0.287602    0.014608   -0.048336\n",
              "14   18000    1000     -0.302210    -0.290686    0.011524   -0.038132\n",
              "15   19000    1000     -0.302210    -0.285892    0.016318   -0.053994\n",
              "16    4000    2000     -0.697901    -0.676898    0.021004   -0.030096\n",
              "17    5000    2000     -0.697901    -0.682049    0.015853   -0.022715\n",
              "18    6000    2000     -0.697901    -0.657747    0.040155   -0.057536\n",
              "19    7000    2000     -0.697901    -0.672880    0.025021   -0.035852\n",
              "20    8000    2000     -0.697901    -0.655866    0.042035   -0.060231\n",
              "21    9000    2000     -0.697901    -0.701781   -0.003879    0.005558\n",
              "22   10000    2000     -0.697901    -0.734431   -0.036529    0.052341\n",
              "23   11000    2000     -0.697901    -0.795557   -0.097655    0.139927\n",
              "24   12000    2000     -0.697901    -0.883217   -0.185316    0.265533\n",
              "25   13000    2000     -0.697901    -0.858482   -0.160581    0.230091\n",
              "26   14000    2000     -0.697901    -0.882921   -0.185019    0.265108\n",
              "27   15000    2000     -0.697901    -0.934700   -0.236799    0.339301\n",
              "28   16000    2000     -0.697901    -0.923487   -0.225586    0.323234\n",
              "29   17000    2000     -0.697901    -0.950743   -0.252842    0.362289\n",
              "30   18000    2000     -0.697901    -1.001919   -0.304017    0.435616\n",
              "31   19000    2000     -0.697901    -0.964767   -0.266866    0.382383\n",
              "32    4000    3000      0.047282     0.178955    0.131673    2.784835\n",
              "33    5000    3000      0.047282     0.268404    0.221122    4.676648\n",
              "34    6000    3000      0.047282     0.299044    0.251762    5.324676\n",
              "35    7000    3000      0.047282     0.250925    0.203643    4.306972\n",
              "36    8000    3000      0.047282     0.239457    0.192175    4.064430\n",
              "37    9000    3000      0.047282     0.174952    0.127670    2.700167\n",
              "38   10000    3000      0.047282     0.117557    0.070275    1.486290\n",
              "39   11000    3000      0.047282     0.126258    0.078975    1.670302\n",
              "40   12000    3000      0.047282    -0.011817   -0.059099   -1.249919\n",
              "41   13000    3000      0.047282     0.008462   -0.038821   -0.821040\n",
              "42   14000    3000      0.047282    -0.009836   -0.057118   -1.208021\n",
              "43   15000    3000      0.047282    -0.061643   -0.108925   -2.303733\n",
              "44   16000    3000      0.047282    -0.037571   -0.084853   -1.794607\n",
              "45   17000    3000      0.047282    -0.024498   -0.071780   -1.518119\n",
              "46   18000    3000      0.047282    -0.092577   -0.139859   -2.957971\n",
              "47   19000    3000      0.047282    -0.150977   -0.198259   -4.193102\n",
              "48    5000    4000      0.407952     0.514669    0.106718    0.261594\n",
              "49    6000    4000      0.407952     0.547358    0.139406    0.341722"
            ]
          },
          "execution_count": 45,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results.head(50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "    source  target  baseline_val  swapped_val  difference  pct_change\n",
            "34    6000    3000      0.047282     0.299044    0.251762    5.324676\n",
            "33    5000    3000      0.047282     0.268404    0.221122    4.676648\n",
            "35    7000    3000      0.047282     0.250925    0.203643    4.306972\n",
            "36    8000    3000      0.047282     0.239457    0.192175    4.064430\n",
            "49    6000    4000      0.407952     0.547358    0.139406    0.341722\n",
            "32    4000    3000      0.047282     0.178955    0.131673    2.784835\n",
            "37    9000    3000      0.047282     0.174952    0.127670    2.700167\n",
            "48    5000    4000      0.407952     0.514669    0.106718    0.261594\n",
            "90    8000    7000      1.843825     1.939461    0.095636    0.051868\n",
            "39   11000    3000      0.047282     0.126258    0.078975    1.670302\n"
          ]
        }
      ],
      "source": [
        "sorted_df = results.sort_values(by='difference', ascending=False)\n",
        "top_n = sorted_df.head(10)\n",
        "print(top_n)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 47,
      "metadata": {},
      "outputs": [
        {
          "ename": "OSError",
          "evalue": "[Errno 122] Disk quota exceeded",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "\u001b[0;31mOSError\u001b[0m: [Errno 122] Disk quota exceeded",
            "\nDuring handling of the above exception, another exception occurred:\n",
            "\u001b[0;31mOSError\u001b[0m                                   Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[47], line 5\u001b[0m\n\u001b[1;32m      3\u001b[0m os\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/mnt/hdd-0/circuits-over-time/results/comp_swap\u001b[39m\u001b[38;5;124m\"\u001b[39m, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[1;32m      4\u001b[0m os\u001b[38;5;241m.\u001b[39mmakedirs(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m/mnt/hdd-0/circuits-over-time/results/comp_swap/\u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmodel_shortname\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m, exist_ok\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mTrue\u001b[39;00m)\n\u001b[0;32m----> 5\u001b[0m \u001b[43mresults\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_csv\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43mf\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m/mnt/hdd-0/circuits-over-time/results/comp_swap/\u001b[39;49m\u001b[38;5;132;43;01m{\u001b[39;49;00m\u001b[43mmodel_shortname\u001b[49m\u001b[38;5;132;43;01m}\u001b[39;49;00m\u001b[38;5;124;43m/ioi_whole_circuit_logit_diff_results.csv\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/pandas/core/generic.py:3772\u001b[0m, in \u001b[0;36mNDFrame.to_csv\u001b[0;34m(self, path_or_buf, sep, na_rep, float_format, columns, header, index, index_label, mode, encoding, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, decimal, errors, storage_options)\u001b[0m\n\u001b[1;32m   3761\u001b[0m df \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(\u001b[38;5;28mself\u001b[39m, ABCDataFrame) \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mto_frame()\n\u001b[1;32m   3763\u001b[0m formatter \u001b[38;5;241m=\u001b[39m DataFrameFormatter(\n\u001b[1;32m   3764\u001b[0m     frame\u001b[38;5;241m=\u001b[39mdf,\n\u001b[1;32m   3765\u001b[0m     header\u001b[38;5;241m=\u001b[39mheader,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   3769\u001b[0m     decimal\u001b[38;5;241m=\u001b[39mdecimal,\n\u001b[1;32m   3770\u001b[0m )\n\u001b[0;32m-> 3772\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mDataFrameRenderer\u001b[49m\u001b[43m(\u001b[49m\u001b[43mformatter\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mto_csv\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   3773\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpath_or_buf\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3774\u001b[0m \u001b[43m    \u001b[49m\u001b[43mlineterminator\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mlineterminator\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3775\u001b[0m \u001b[43m    \u001b[49m\u001b[43msep\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43msep\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3776\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoding\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoding\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3777\u001b[0m \u001b[43m    \u001b[49m\u001b[43merrors\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43merrors\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3778\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcompression\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcompression\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3779\u001b[0m \u001b[43m    \u001b[49m\u001b[43mquoting\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquoting\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3780\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3781\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindex_label\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_label\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3782\u001b[0m \u001b[43m    \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3783\u001b[0m \u001b[43m    \u001b[49m\u001b[43mchunksize\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mchunksize\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3784\u001b[0m \u001b[43m    \u001b[49m\u001b[43mquotechar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mquotechar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3785\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdate_format\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdate_format\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3786\u001b[0m \u001b[43m    \u001b[49m\u001b[43mdoublequote\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdoublequote\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3787\u001b[0m \u001b[43m    \u001b[49m\u001b[43mescapechar\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mescapechar\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3788\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstorage_options\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_options\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   3789\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/pandas/io/formats/format.py:1186\u001b[0m, in \u001b[0;36mDataFrameRenderer.to_csv\u001b[0;34m(self, path_or_buf, encoding, sep, columns, index_label, mode, compression, quoting, quotechar, lineterminator, chunksize, date_format, doublequote, escapechar, errors, storage_options)\u001b[0m\n\u001b[1;32m   1165\u001b[0m     created_buffer \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n\u001b[1;32m   1167\u001b[0m csv_formatter \u001b[38;5;241m=\u001b[39m CSVFormatter(\n\u001b[1;32m   1168\u001b[0m     path_or_buf\u001b[38;5;241m=\u001b[39mpath_or_buf,\n\u001b[1;32m   1169\u001b[0m     lineterminator\u001b[38;5;241m=\u001b[39mlineterminator,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1184\u001b[0m     formatter\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfmt,\n\u001b[1;32m   1185\u001b[0m )\n\u001b[0;32m-> 1186\u001b[0m \u001b[43mcsv_formatter\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43msave\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1188\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m created_buffer:\n\u001b[1;32m   1189\u001b[0m     \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(path_or_buf, StringIO)\n",
            "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/pandas/io/formats/csvs.py:259\u001b[0m, in \u001b[0;36mCSVFormatter.save\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    240\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m get_handle(\n\u001b[1;32m    241\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mfilepath_or_buffer,\n\u001b[1;32m    242\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmode,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    247\u001b[0m ) \u001b[38;5;28;01mas\u001b[39;00m handles:\n\u001b[1;32m    248\u001b[0m     \u001b[38;5;66;03m# Note: self.encoding is irrelevant here\u001b[39;00m\n\u001b[1;32m    249\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mwriter \u001b[38;5;241m=\u001b[39m csvlib\u001b[38;5;241m.\u001b[39mwriter(\n\u001b[1;32m    250\u001b[0m         handles\u001b[38;5;241m.\u001b[39mhandle,\n\u001b[1;32m    251\u001b[0m         lineterminator\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlineterminator,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    256\u001b[0m         quotechar\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mquotechar,\n\u001b[1;32m    257\u001b[0m     )\n\u001b[0;32m--> 259\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_save()\n",
            "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/pandas/io/common.py:138\u001b[0m, in \u001b[0;36mIOHandles.__exit__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m    137\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__exit__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs: Any) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[0;32m--> 138\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclose\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
            "File \u001b[0;32m/mnt/ssd-1/ctigges/miniconda/envs/cot/lib/python3.8/site-packages/pandas/io/common.py:130\u001b[0m, in \u001b[0;36mIOHandles.close\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    128\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcreated_handles\u001b[38;5;241m.\u001b[39mremove(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandle)\n\u001b[1;32m    129\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m handle \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcreated_handles:\n\u001b[0;32m--> 130\u001b[0m     \u001b[43mhandle\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclose\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    131\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcreated_handles \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    132\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mis_wrapped \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
            "\u001b[0;31mOSError\u001b[0m: [Errno 122] Disk quota exceeded"
          ]
        }
      ],
      "source": [
        "# save results\n",
        "model_shortname = model_name[11:]\n",
        "os.makedirs(\"/mnt/hdd-0/circuits-over-time/results/comp_swap\", exist_ok=True)\n",
        "os.makedirs(f\"/mnt/hdd-0/circuits-over-time/results/comp_swap/{model_shortname}\", exist_ok=True)\n",
        "results.to_csv(f\"/mnt/hdd-0/circuits-over-time/results/comp_swap/{model_shortname}/ioi_whole_circuit_logit_diff_results.csv\", index=False)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Greater Than"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 63,
      "metadata": {},
      "outputs": [],
      "source": [
        "model_name = \"EleutherAI/pythia-160m\"\n",
        "successor_heads = [(11, 9), (4, 10), (4, 6), (5, 0), (5, 9), (9, 0), (10, 4)]\n",
        "induction_heads = [(4, 11), (5, 0)]\n",
        "all = successor_heads + induction_heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 64,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 2000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 3000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 1000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 3000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 2000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 4000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 7000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 19000 -> Target: 3000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 5000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 10000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 4000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 6000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 5000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 7000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 19000 -> Target: 6000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 8000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 7000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 9000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 16000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n",
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 8000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 10000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 9000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 11000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 10000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 12000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 11000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 13000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 12000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 14000 -> Target: 13000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 13000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 13000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 13000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 13000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 13000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 15000 -> Target: 14000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 14000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 14000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 14000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 14000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 16000 -> Target: 15000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 15000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 15000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 15000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 17000 -> Target: 16000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 16000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 16000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 18000 -> Target: 17000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "WARNING:root:Pythia models on HF were updated on 4/3/23! add '-v0' to model name to access the old models.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Source: 19000 -> Target: 17000\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Loaded pretrained model EleutherAI/pythia-160m into HookedTransformer\n",
            "Source: 19000 -> Target: 18000\n"
          ]
        }
      ],
      "source": [
        "results = run_swap_grid_search(model_name, get_gt_prob_diff_for_model, source_range=(2000, 20000), target_range=(1000, 19000), heads=all)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 65,
      "metadata": {},
      "outputs": [],
      "source": [
        "results['pct_change'] = results['difference'] / results['baseline_ld']\n",
        "results = results.rename(columns={'baseline_ld': 'baseline_val', 'swapped_ld': 'swapped_val'})"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {},
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'results' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "Cell \u001b[0;32mIn[1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m sorted_df \u001b[38;5;241m=\u001b[39m \u001b[43mresults\u001b[49m\u001b[38;5;241m.\u001b[39msort_values(by\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mdifference\u001b[39m\u001b[38;5;124m'\u001b[39m, ascending\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)\n\u001b[1;32m      2\u001b[0m top_n \u001b[38;5;241m=\u001b[39m sorted_df\u001b[38;5;241m.\u001b[39mhead(\u001b[38;5;241m10\u001b[39m)\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28mprint\u001b[39m(top_n)\n",
            "\u001b[0;31mNameError\u001b[0m: name 'results' is not defined"
          ]
        }
      ],
      "source": [
        "sorted_df = results.sort_values(by='difference', ascending=False)\n",
        "top_n = sorted_df.head(10)\n",
        "print(top_n)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 59,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>target</th>\n",
              "      <th>baseline_ld</th>\n",
              "      <th>swapped_ld</th>\n",
              "      <th>difference</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>2000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.172767</td>\n",
              "      <td>-0.159733</td>\n",
              "      <td>0.013034</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.174749</td>\n",
              "      <td>-0.146568</td>\n",
              "      <td>0.028181</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>4000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.177742</td>\n",
              "      <td>-0.130802</td>\n",
              "      <td>0.046940</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>5000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.170297</td>\n",
              "      <td>-0.123153</td>\n",
              "      <td>0.047143</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>6000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.175155</td>\n",
              "      <td>-0.130347</td>\n",
              "      <td>0.044809</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>7000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.176924</td>\n",
              "      <td>-0.135718</td>\n",
              "      <td>0.041205</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>8000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.176415</td>\n",
              "      <td>-0.167677</td>\n",
              "      <td>0.008738</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>9000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.172683</td>\n",
              "      <td>-0.160158</td>\n",
              "      <td>0.012525</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>10000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.172487</td>\n",
              "      <td>-0.170555</td>\n",
              "      <td>0.001933</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>11000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.172088</td>\n",
              "      <td>-0.152882</td>\n",
              "      <td>0.019206</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>12000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.167130</td>\n",
              "      <td>-0.152598</td>\n",
              "      <td>0.014532</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>13000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.170507</td>\n",
              "      <td>-0.141444</td>\n",
              "      <td>0.029063</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>14000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.178303</td>\n",
              "      <td>-0.151707</td>\n",
              "      <td>0.026596</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>15000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.169360</td>\n",
              "      <td>-0.170429</td>\n",
              "      <td>-0.001069</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>16000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.176903</td>\n",
              "      <td>-0.179137</td>\n",
              "      <td>-0.002234</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>17000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.161105</td>\n",
              "      <td>-0.168279</td>\n",
              "      <td>-0.007174</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>18000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.175716</td>\n",
              "      <td>-0.182102</td>\n",
              "      <td>-0.006386</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>19000</td>\n",
              "      <td>1000</td>\n",
              "      <td>-0.170573</td>\n",
              "      <td>-0.181263</td>\n",
              "      <td>-0.010691</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>3000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.162610</td>\n",
              "      <td>0.177491</td>\n",
              "      <td>0.014881</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>4000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.163447</td>\n",
              "      <td>0.187370</td>\n",
              "      <td>0.023923</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>5000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.163780</td>\n",
              "      <td>0.185162</td>\n",
              "      <td>0.021381</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>6000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.161343</td>\n",
              "      <td>0.196313</td>\n",
              "      <td>0.034970</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>7000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.165384</td>\n",
              "      <td>0.210136</td>\n",
              "      <td>0.044753</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>8000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.159557</td>\n",
              "      <td>0.207562</td>\n",
              "      <td>0.048005</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>9000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.161969</td>\n",
              "      <td>0.214034</td>\n",
              "      <td>0.052065</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>10000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.165133</td>\n",
              "      <td>0.222236</td>\n",
              "      <td>0.057102</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>11000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.160101</td>\n",
              "      <td>0.221266</td>\n",
              "      <td>0.061165</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>12000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.157978</td>\n",
              "      <td>0.230110</td>\n",
              "      <td>0.072133</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>13000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.164190</td>\n",
              "      <td>0.224335</td>\n",
              "      <td>0.060146</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>14000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.168134</td>\n",
              "      <td>0.214409</td>\n",
              "      <td>0.046275</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>30</th>\n",
              "      <td>15000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.153287</td>\n",
              "      <td>0.224723</td>\n",
              "      <td>0.071436</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>16000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.168325</td>\n",
              "      <td>0.214902</td>\n",
              "      <td>0.046578</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>32</th>\n",
              "      <td>17000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.159002</td>\n",
              "      <td>0.211266</td>\n",
              "      <td>0.052264</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>33</th>\n",
              "      <td>18000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.162272</td>\n",
              "      <td>0.220176</td>\n",
              "      <td>0.057904</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>34</th>\n",
              "      <td>19000</td>\n",
              "      <td>2000</td>\n",
              "      <td>0.164862</td>\n",
              "      <td>0.221061</td>\n",
              "      <td>0.056200</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>35</th>\n",
              "      <td>4000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.322633</td>\n",
              "      <td>0.298588</td>\n",
              "      <td>-0.024045</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36</th>\n",
              "      <td>5000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.322927</td>\n",
              "      <td>0.275056</td>\n",
              "      <td>-0.047871</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37</th>\n",
              "      <td>6000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.323110</td>\n",
              "      <td>0.283278</td>\n",
              "      <td>-0.039832</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>38</th>\n",
              "      <td>7000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.321349</td>\n",
              "      <td>0.273165</td>\n",
              "      <td>-0.048184</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>39</th>\n",
              "      <td>8000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.313481</td>\n",
              "      <td>0.275050</td>\n",
              "      <td>-0.038431</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>40</th>\n",
              "      <td>9000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.321656</td>\n",
              "      <td>0.274293</td>\n",
              "      <td>-0.047363</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>41</th>\n",
              "      <td>10000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.320621</td>\n",
              "      <td>0.281028</td>\n",
              "      <td>-0.039593</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>42</th>\n",
              "      <td>11000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.317992</td>\n",
              "      <td>0.292169</td>\n",
              "      <td>-0.025823</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>43</th>\n",
              "      <td>12000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.324661</td>\n",
              "      <td>0.291815</td>\n",
              "      <td>-0.032846</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>44</th>\n",
              "      <td>13000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.326418</td>\n",
              "      <td>0.281349</td>\n",
              "      <td>-0.045069</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>45</th>\n",
              "      <td>14000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.318826</td>\n",
              "      <td>0.287190</td>\n",
              "      <td>-0.031635</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>46</th>\n",
              "      <td>15000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.321523</td>\n",
              "      <td>0.286486</td>\n",
              "      <td>-0.035038</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>47</th>\n",
              "      <td>16000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.320415</td>\n",
              "      <td>0.277470</td>\n",
              "      <td>-0.042945</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>48</th>\n",
              "      <td>17000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.318288</td>\n",
              "      <td>0.272786</td>\n",
              "      <td>-0.045502</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>49</th>\n",
              "      <td>18000</td>\n",
              "      <td>3000</td>\n",
              "      <td>0.316213</td>\n",
              "      <td>0.283350</td>\n",
              "      <td>-0.032863</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "    source  target  baseline_ld  swapped_ld  difference\n",
              "0     2000    1000    -0.172767   -0.159733    0.013034\n",
              "1     3000    1000    -0.174749   -0.146568    0.028181\n",
              "2     4000    1000    -0.177742   -0.130802    0.046940\n",
              "3     5000    1000    -0.170297   -0.123153    0.047143\n",
              "4     6000    1000    -0.175155   -0.130347    0.044809\n",
              "5     7000    1000    -0.176924   -0.135718    0.041205\n",
              "6     8000    1000    -0.176415   -0.167677    0.008738\n",
              "7     9000    1000    -0.172683   -0.160158    0.012525\n",
              "8    10000    1000    -0.172487   -0.170555    0.001933\n",
              "9    11000    1000    -0.172088   -0.152882    0.019206\n",
              "10   12000    1000    -0.167130   -0.152598    0.014532\n",
              "11   13000    1000    -0.170507   -0.141444    0.029063\n",
              "12   14000    1000    -0.178303   -0.151707    0.026596\n",
              "13   15000    1000    -0.169360   -0.170429   -0.001069\n",
              "14   16000    1000    -0.176903   -0.179137   -0.002234\n",
              "15   17000    1000    -0.161105   -0.168279   -0.007174\n",
              "16   18000    1000    -0.175716   -0.182102   -0.006386\n",
              "17   19000    1000    -0.170573   -0.181263   -0.010691\n",
              "18    3000    2000     0.162610    0.177491    0.014881\n",
              "19    4000    2000     0.163447    0.187370    0.023923\n",
              "20    5000    2000     0.163780    0.185162    0.021381\n",
              "21    6000    2000     0.161343    0.196313    0.034970\n",
              "22    7000    2000     0.165384    0.210136    0.044753\n",
              "23    8000    2000     0.159557    0.207562    0.048005\n",
              "24    9000    2000     0.161969    0.214034    0.052065\n",
              "25   10000    2000     0.165133    0.222236    0.057102\n",
              "26   11000    2000     0.160101    0.221266    0.061165\n",
              "27   12000    2000     0.157978    0.230110    0.072133\n",
              "28   13000    2000     0.164190    0.224335    0.060146\n",
              "29   14000    2000     0.168134    0.214409    0.046275\n",
              "30   15000    2000     0.153287    0.224723    0.071436\n",
              "31   16000    2000     0.168325    0.214902    0.046578\n",
              "32   17000    2000     0.159002    0.211266    0.052264\n",
              "33   18000    2000     0.162272    0.220176    0.057904\n",
              "34   19000    2000     0.164862    0.221061    0.056200\n",
              "35    4000    3000     0.322633    0.298588   -0.024045\n",
              "36    5000    3000     0.322927    0.275056   -0.047871\n",
              "37    6000    3000     0.323110    0.283278   -0.039832\n",
              "38    7000    3000     0.321349    0.273165   -0.048184\n",
              "39    8000    3000     0.313481    0.275050   -0.038431\n",
              "40    9000    3000     0.321656    0.274293   -0.047363\n",
              "41   10000    3000     0.320621    0.281028   -0.039593\n",
              "42   11000    3000     0.317992    0.292169   -0.025823\n",
              "43   12000    3000     0.324661    0.291815   -0.032846\n",
              "44   13000    3000     0.326418    0.281349   -0.045069\n",
              "45   14000    3000     0.318826    0.287190   -0.031635\n",
              "46   15000    3000     0.321523    0.286486   -0.035038\n",
              "47   16000    3000     0.320415    0.277470   -0.042945\n",
              "48   17000    3000     0.318288    0.272786   -0.045502\n",
              "49   18000    3000     0.316213    0.283350   -0.032863"
            ]
          },
          "execution_count": 59,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "results.head(50)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 69,
      "metadata": {},
      "outputs": [],
      "source": [
        "# save results\n",
        "model_shortname = model_name[11:]\n",
        "os.makedirs(\"/mnt/hdd-0/circuits-over-time/results/comp_swap\", exist_ok=True)\n",
        "os.makedirs(f\"/mnt/hdd-0/circuits-over-time/results/comp_swap/{model_shortname}\", exist_ok=True)\n",
        "results.to_csv(f\"/mnt/hdd-0/circuits-over-time/results/comp_swap/{model_shortname}/gt_induction_successor_head_prob_diff_results.csv\", index=False)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "x4diyMzfPCBD"
      },
      "source": [
        "## Data Setup"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {},
      "outputs": [
        {
          "data": {
            "text/plain": [
              "'pythia-160m'"
            ]
          },
          "execution_count": 54,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "model_shortname = model_name[11:]\n",
        "model_shortname"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "\n",
        "N = 70\n",
        "model.tokenizer.add_bos_token = False\n",
        "ioi_dataset, abc_dataset = generate_data_and_caches(model, N, verbose=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lrxnGyi7E19v",
        "outputId": "ef4ce3a5-ea1b-456a-c898-05188e08ffdc"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Clean logit diff: 1.1939\n",
            "Corrupted logit diff: -0.7280\n"
          ]
        }
      ],
      "source": [
        "clean_logits, clean_cache = model.run_with_cache(ioi_dataset.toks)\n",
        "corrupted_logits, corrupted_cache = model.run_with_cache(abc_dataset.toks)\n",
        "\n",
        "clean_logit_diff = _logits_to_mean_logit_diff(clean_logits, ioi_dataset)\n",
        "print(f\"Clean logit diff: {clean_logit_diff:.4f}\")\n",
        "\n",
        "corrupted_logit_diff = _logits_to_mean_logit_diff(corrupted_logits, ioi_dataset)\n",
        "print(f\"Corrupted logit diff: {corrupted_logit_diff:.4f}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "DnRTCaAoFK7n",
        "outputId": "a9be2bdb-b8c1-459c-a98d-3bce5a746d6d"
      },
      "outputs": [],
      "source": [
        "CLEAN_BASELINE = clean_logit_diff\n",
        "CORRUPTED_BASELINE = corrupted_logit_diff"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {},
      "outputs": [],
      "source": [
        "def logit_diff_denoising(\n",
        "    logits: Float[Tensor, \"batch seq d_vocab\"],\n",
        "    dataset: IOIDataset,\n",
        "    flipped_logit_diff: float = corrupted_logit_diff,\n",
        "    clean_logit_diff: float = clean_logit_diff,\n",
        "    return_tensor: bool = False,\n",
        ") -> Float[Tensor, \"\"]:\n",
        "    '''\n",
        "    Linear function of logit diff, calibrated so that it equals 0 when performance is\n",
        "    same as on flipped input, and 1 when performance is same as on clean input.\n",
        "    '''\n",
        "    patched_logit_diff = _logits_to_ave_logit_diff(logits, dataset)\n",
        "    ld = ((patched_logit_diff - flipped_logit_diff) / (clean_logit_diff  - flipped_logit_diff))\n",
        "    if return_tensor:\n",
        "        return ld\n",
        "    else:\n",
        "        return ld.item()\n",
        "\n",
        "\n",
        "def logit_diff_noising(\n",
        "        logits: Float[Tensor, \"batch seq d_vocab\"],\n",
        "        dataset: IOIDataset = ioi_dataset,\n",
        "        clean_logit_diff: float = clean_logit_diff,\n",
        "        corrupted_logit_diff: float = corrupted_logit_diff,\n",
        "        return_tensor: bool = False,\n",
        "    ) -> float:\n",
        "        '''\n",
        "        We calibrate this so that the value is 0 when performance isn't harmed (i.e. same as IOI dataset),\n",
        "        and -1 when performance has been destroyed (i.e. is same as ABC dataset).\n",
        "        '''\n",
        "        patched_logit_diff = _logits_to_ave_logit_diff(logits, dataset)\n",
        "        ld = ((patched_logit_diff - clean_logit_diff) / (clean_logit_diff - corrupted_logit_diff))\n",
        "\n",
        "        if return_tensor:\n",
        "            return ld\n",
        "        else:\n",
        "            return ld.item()\n",
        "\n",
        "logit_diff_denoising_ioi = partial(logit_diff_denoising, dataset=ioi_dataset)\n",
        "logit_diff_noising_ioi = partial(logit_diff_noising, dataset=ioi_dataset)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "oHCkQu4YSOX2"
      },
      "source": [
        "## Tool Setup"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "hZerXmikSYIN"
      },
      "source": [
        "### Activation Patching"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Direct Logit Attribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "sio_answer_tensor = torch.cat(\n",
        "    (torch.tensor(ioi_dataset.io_tokenIDs).unsqueeze(dim=1), torch.tensor(ioi_dataset.s_tokenIDs).unsqueeze(dim=1)), \n",
        "    dim=1\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "answer_residual_directions: Float[Tensor, \"batch 2 d_model\"] = model.tokens_to_residual_directions(sio_answer_tensor)\n",
        "print(\"Answer residual directions shape:\", answer_residual_directions.shape)\n",
        "\n",
        "correct_residual_directions, incorrect_residual_directions = answer_residual_directions.unbind(dim=1)\n",
        "logit_diff_directions: Float[Tensor, \"batch d_model\"] = correct_residual_directions - incorrect_residual_directions\n",
        "print(f\"Logit difference directions shape:\", logit_diff_directions.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# cache syntax - resid_post is the residual stream at the end of the layer, -1 gets the final layer. The general syntax is [activation_name, layer_index, sub_layer_type].\n",
        "final_residual_stream: Float[Tensor, \"batch seq d_model\"] = clean_cache[\"resid_post\", -1]\n",
        "print(f\"Final residual stream shape: {final_residual_stream.shape}\")\n",
        "scaled_residual_stream = clean_cache.apply_ln_to_stack(final_residual_stream, layer=-1)\n",
        "print(f\"Scaled residual stream shape: {scaled_residual_stream.shape}\")\n",
        "scaled_final_token_residual_stream: Float[Tensor, \"batch d_model\"] = scaled_residual_stream[torch.arange(final_residual_stream.size(0)), ioi_dataset.word_idx[\"end\"]]\n",
        "print(f\"Final token residual stream shape: {scaled_final_token_residual_stream.shape}\")\n",
        "\n",
        "average_logit_diff = einops.einsum(\n",
        "    scaled_final_token_residual_stream, logit_diff_directions,\n",
        "    \"batch d_model, batch d_model ->\"\n",
        ") / 70\n",
        "\n",
        "print(f\"Calculated average logit diff: {average_logit_diff:.10f}\")\n",
        "print(f\"Original logit difference:     {clean_logit_diff:.10f}\")\n",
        "\n",
        "#torch.testing.assert_close(average_logit_diff, clean_logit_diff)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "from transformer_lens import ActivationCache\n",
        "\n",
        "def residual_stack_to_logit_diff(\n",
        "    residual_stack: Float[Tensor, \"... batch d_model\"],\n",
        "    cache: ActivationCache,\n",
        "    logit_diff_directions: Float[Tensor, \"batch d_model\"] = logit_diff_directions,\n",
        ") -> Float[Tensor, \"...\"]:\n",
        "    '''\n",
        "    Gets the avg logit difference between the correct and incorrect answer for a given\n",
        "    stack of components in the residual stream.\n",
        "    '''\n",
        "    # SOLUTION\n",
        "    batch_size = residual_stack.size(-2)\n",
        "    scaled_residual_stack = cache.apply_ln_to_stack(residual_stack, layer=-1, pos_slice=-1)\n",
        "    return einops.einsum(\n",
        "        scaled_residual_stack, logit_diff_directions,\n",
        "        \"... batch d_model, batch d_model -> ...\"\n",
        "    ) / batch_size\n",
        "\n",
        "\n",
        "# Test function by checking that it gives the same result as the original logit difference\n",
        "# t.testing.assert_close(\n",
        "#     residual_stack_to_logit_diff(final_token_residual_stream, cache),\n",
        "#     original_average_logit_diff\n",
        "# )"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Logit Lens"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "accumulated_residual, labels = clean_cache.accumulated_resid(layer=-1, incl_mid=False, return_labels=True)\n",
        "accumulated_residual_final_token = accumulated_residual[:, torch.arange(accumulated_residual.size(1)), ioi_dataset.word_idx[\"end\"]]\n",
        "print(f\"Shape of accumulated residual: {accumulated_residual_final_token.shape}\")\n",
        "# accumulated_residual has shape (component, batch, d_model)\n",
        "\n",
        "logit_lens_logit_diffs: Float[Tensor, \"component\"] = residual_stack_to_logit_diff(accumulated_residual_final_token, clean_cache)\n",
        "line(logit_lens_logit_diffs, x=np.arange(model.cfg.n_layers+1), hover_name=labels, title=\"Logit Difference From Accumulated Residual Stream\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Layer Attribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_layer_residual, labels = clean_cache.decompose_resid(layer=-1, return_labels=True)\n",
        "per_layer_residual_final_token = per_layer_residual[:, torch.arange(per_layer_residual.size(1)), ioi_dataset.word_idx[\"end\"]]\n",
        "per_layer_logit_diffs = residual_stack_to_logit_diff(per_layer_residual_final_token, clean_cache)\n",
        "\n",
        "line(per_layer_logit_diffs, hover_name=labels, title=\"Logit Difference From Each Layer\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Head Attribution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_head_residual, labels = clean_cache.stack_head_results(layer=-1, return_labels=True)\n",
        "print(f\"Shape of per head residual: {per_head_residual.shape}\")\n",
        "per_head_residual_final_token = per_head_residual[:, torch.arange(per_head_residual.size(1)), ioi_dataset.word_idx[\"end\"]]\n",
        "print(f\"Shape of per head residual: {per_head_residual_final_token.shape}\")\n",
        "per_head_residual_final_token = einops.rearrange(\n",
        "    per_head_residual_final_token,\n",
        "    \"(layer head) ... -> layer head ...\",\n",
        "    layer=model.cfg.n_layers\n",
        ")\n",
        "print(f\"Shape of per head residual: {per_head_residual_final_token.shape}\")\n",
        "per_head_logit_diffs = residual_stack_to_logit_diff(per_head_residual_final_token, clean_cache)\n",
        "\n",
        "imshow(per_head_logit_diffs, xaxis=\"Head\", yaxis=\"Layer\", title=\"Logit Difference From Each Head\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(per_head_logit_diffs/clean_logit_diff, top_n=15, range_x=[0, 1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "nmh_candidates = [(8, 10), (10, 7)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "def scatter_attention_and_contribution(\n",
        "    model,\n",
        "    head,\n",
        "    prompts,\n",
        "    end_positions,\n",
        "    io_positions,\n",
        "    s_positions,\n",
        "    answer_residual_directions,\n",
        "    return_vals=False,\n",
        "    return_fig=False,\n",
        "):\n",
        "\n",
        "    df = []\n",
        "\n",
        "    layer, head_idx = head\n",
        "    # Get the attention output to the residual stream for the head\n",
        "    _, cache = model.run_with_cache(prompts)\n",
        "    per_head_residual, _ = cache.stack_head_results(\n",
        "        layer=-1, return_labels=True\n",
        "    )\n",
        "    scaled_residual_stack = cache.apply_ln_to_stack(\n",
        "        per_head_residual, layer=-1\n",
        "    )\n",
        "    print(scaled_residual_stack.shape)\n",
        "    scaled_residual_stack_final_token = scaled_residual_stack[:, torch.arange(scaled_residual_stack.size(1)), end_positions]\n",
        "    head_resid = scaled_residual_stack_final_token[layer * model.cfg.n_heads + head_idx]\n",
        "\n",
        "    # Loop over each prompt\n",
        "    for i in range(len(answer_residual_directions)):\n",
        "        # Get attention values\n",
        "        tokens, attn, names = get_attn_head_patterns(model, prompts[i], [head])\n",
        "\n",
        "        # For IO\n",
        "        # Get the attention contribution in the residual directions\n",
        "        dot = einsum(\n",
        "            \"d_model, d_model -> \", head_resid[i], answer_residual_directions[i][0]\n",
        "        )\n",
        "\n",
        "        # Get the attention probability to the IO answer\n",
        "        prob = attn[0, 14, io_positions[i]]\n",
        "        df.append([prob, dot, \"IO\", prompts[i]])\n",
        "\n",
        "        # For S\n",
        "        # Get the attention contribution in the residual directions\n",
        "        dot = einsum(\n",
        "            \"d_model, d_model -> \", head_resid[i], answer_residual_directions[i][1]\n",
        "        )\n",
        "        # Get the attention probability to the S answer\n",
        "        prob = attn[0, 14, s_positions[i]]\n",
        "        df.append([prob, dot, \"S\", prompts[i]])\n",
        "\n",
        "    # Plot the results\n",
        "    viz_df = pd.DataFrame(\n",
        "        df, columns=[f\"Attn Prob on Name\", f\"Dot w Name Embed\", \"Name Type\", \"text\"]\n",
        "    )\n",
        "    fig = px.scatter(\n",
        "        viz_df,\n",
        "        x=f\"Attn Prob on Name\",\n",
        "        y=f\"Dot w Name Embed\",\n",
        "        color=\"Name Type\",\n",
        "        hover_data=[\"text\"],\n",
        "        color_discrete_sequence=[\"rgb(114,255,100)\", \"rgb(201,165,247)\"],\n",
        "        title=f\"How Strong {layer}.{head_idx} Writes in the Name Embed Direction Relative to Attn Prob\",\n",
        "    )\n",
        "\n",
        "    if return_vals:\n",
        "        return viz_df\n",
        "    if return_fig:\n",
        "        return fig\n",
        "    else:\n",
        "        fig.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "scatter_attention_and_contribution(model, (8, 9), ioi_dataset.toks, ioi_dataset.word_idx['end'], ioi_dataset.word_idx['IO'], ioi_dataset.word_idx['S2'], answer_residual_directions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 2\n",
        "top_heads = torch.topk(-per_head_logit_diffs.flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, prompts[0], heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "scatter_attention_and_contribution(model, (8, 9), prompts, io_positions, s_positions, answer_residual_directions)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "mIE1CDqQyfRP"
      },
      "source": [
        "## Activation Patching for Model Component Importance"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "### Attention Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "results = act_patch(\n",
        "    model=model,\n",
        "    orig_input=abc_dataset.toks,\n",
        "    new_cache=clean_cache,\n",
        "    patching_nodes=IterNode(\"z\"), # iterating over all heads' output in all layers\n",
        "    patching_metric=logit_diff_denoising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "    results['z'] * 100,\n",
        "    title=\"Patching output of attention heads (corrupted -> clean)\",\n",
        "    labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=600,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "xTIMVadvpzU0"
      },
      "source": [
        "### Head Output by Component"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "78715c8355a84e7a950ee5e0541d598f",
            "ed976a6e6c9f40b2810d7869459711d4",
            "21ec4ae4727f4308868d2922abae7dc7",
            "4bf78c63b25f45dda35014e3b5f6fbce",
            "84bbf960ddee46e39a41007929a386f1",
            "6e8904a58f3a4ce3856699fb7000d725",
            "480fb7bee6a341c4bf0234aa175bd5e8",
            "1af9fa979cae41d4b95b3607e4969f80",
            "da24d7c9b66f42d7a5b0a2abff104d68",
            "325651b3a4a3456396c773dacb8b693f",
            "7fbcaecd08c24b0c895863ca04bb823e"
          ]
        },
        "id": "SDb7z0oLp7bt",
        "outputId": "f8a9b84b-8e6b-4da4-e423-1d40334af8b1"
      },
      "outputs": [],
      "source": [
        "# iterating over all heads' output in all layers\n",
        "results = act_patch(\n",
        "    model=model,\n",
        "    orig_input=abc_dataset.toks,\n",
        "    new_cache=clean_cache,\n",
        "    patching_nodes=IterNode([\"z\", \"q\", \"k\", \"v\", \"pattern\"]),\n",
        "    patching_metric=logit_diff_denoising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "assert results.keys() == {\"z\", \"q\", \"k\", \"v\", \"pattern\"}\n",
        "#assert all([r.shape == (12, 12) for r in results.values()])\n",
        "\n",
        "imshow_p(\n",
        "    torch.stack(tuple(results.values())) * 100,\n",
        "    facet_col=0,\n",
        "    facet_labels=[\"Output\", \"Query\", \"Key\", \"Value\", \"Pattern\"],\n",
        "    title=\"Patching output of attention heads (corrupted -> clean)\",\n",
        "    labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=1500,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "Qghcp74yJYWa"
      },
      "source": [
        "### Residual Stream & Layer Outputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "46e0b6d1197741ee812720e6b599b6c8",
            "c8310b424eb941f6ae389d7de87b7287",
            "8fb1bff2394a4aa492a717a0e7db3d2b",
            "74ee415f994d4acf8d31af2e81a93123",
            "044171c855c240ff923b69b4e27e1538",
            "65917c03b217485bac9bdd53e962def8",
            "60ec18de81af433880b6167a01fba30f",
            "66ff90fae10642eca0fa5e631447975b",
            "88861a3614834f7fb3bdc99de0d2baeb",
            "7d3c726ca3fb48de98432b35c8550c0b",
            "5af261b7e4754241abd86e41c98f2750"
          ]
        },
        "id": "csi9QlLwIGR8",
        "outputId": "f3ebde51-e599-434d-8a75-e641c5f3c869"
      },
      "outputs": [],
      "source": [
        "results = act_patch(\n",
        "    model=model,\n",
        "    orig_input=abc_dataset.toks,\n",
        "    new_cache=clean_cache,\n",
        "    patching_nodes=IterNode([\"resid_pre\", \"attn_out\", \"mlp_out\"], seq_pos=\"each\"),\n",
        "    patching_metric=logit_diff_denoising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "assert results.keys() == {\"resid_pre\", \"attn_out\", \"mlp_out\"}\n",
        "labels = [f\"{tok} {i}\" for i, tok in enumerate(model.to_str_tokens(ioi_dataset.toks[0]))]\n",
        "imshow_p(\n",
        "    torch.stack([r.T for r in results.values()]) * 100, # we transpose so layer is on the y-axis\n",
        "    facet_col=0,\n",
        "    facet_labels=[\"resid_pre\", \"attn_out\", \"mlp_out\"],\n",
        "    title=\"Patching at resid stream & layer outputs (corrupted -> clean)\",\n",
        "    labels={\"x\": \"Sequence position\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    x=labels,\n",
        "    xaxis_tickangle=45,\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=1400,\n",
        "    height=600,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "jTR8Yuu-glCM"
      },
      "source": [
        "## Circuit Sketching"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "fmGNftUvUlOC"
      },
      "source": [
        "### First Level"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VirxokHeDQ28"
      },
      "source": [
        "#### Heads Influencing Logit Diff Directly"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.cfg.use_attn_in = True"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 574,
          "referenced_widgets": [
            "bb40ed2d47ed4e3cbe89e68e816677e4",
            "0c1eb5b72748416ba1a78a06543a7bb8",
            "b08c648d67474d9eb9986ea24f28311b",
            "1ded46589d814836a88da906225cda46",
            "2f43dfcb6a6b46a0a36406767d040503",
            "d4f67196e74e4ff9b5df7045ed98b17a",
            "f5947be879d44715846b197614974298",
            "d7754c6b0a7a4ba6b40b3c1ffc458b55",
            "f0f946dd323d47f88ed986c4c0a2537f",
            "64387ba7a3b04f5f8d844ee43146212d",
            "3c84b4575d8d45c1866f771bb081aeb4"
          ]
        },
        "id": "FkJzhFdnCmF8",
        "outputId": "067a5e08-3c47-47c4-c08c-c0fbe7c88cab"
      },
      "outputs": [],
      "source": [
        "path_patch_resid_post = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode('z'), # This means iterate over all heads in all layers\n",
        "    receiver_nodes=Node('resid_post', 11), # This is resid_post at layer 11\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "    path_patch_resid_post['z'] * 100,\n",
        "    title=\"Direct effect on logit diff (patch from head output -> final resid)\",\n",
        "    labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "    coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "    border=True,\n",
        "    width=600,\n",
        "    margin={\"r\": 100, \"l\": 100}\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(-path_patch_resid_post['z'].cuda(), top_n=10, range_x=[0, 1.0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 1\n",
        "DISPLAY_IDX = 0\n",
        "top_heads = torch.topk(-path_patch_resid_post['z'].flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "#DE_NMH = [(8, 10), (8, 2)]\n",
        "DE_S2I = [(8, 9)]\n",
        "#DE_PUNC_CONJ = [(10, 1)]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "logit_diff_denoising_ioi_t = partial(logit_diff_denoising, dataset=ioi_dataset, return_tensor=True)\n",
        "logit_diff_noising_ioi_t = partial(logit_diff_noising, dataset=ioi_dataset, return_tensor=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "ioi_metric = logit_diff_denoising_ioi\n",
        "attn_head_pattern_all_pos_act_patch_results = patching.get_act_patch_attn_head_pattern_all_pos(model, abc_dataset.toks, ioi_cache, logit_diff_denoising_ioi_t)\n",
        "# imshow(attn_head_pattern_all_pos_act_patch_results, \n",
        "#        yaxis=\"Layer\", \n",
        "#        xaxis=\"Head\", \n",
        "#        title=\"IOI Metric for 'attn_head_pattern' Activation Patching (All Pos)\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "attn_head_out_all_pos_act_patch_results = patching.get_act_patch_attn_head_out_all_pos(model, abc_dataset.toks, ioi_cache, logit_diff_denoising_ioi_t)\n",
        "# imshow(attn_head_out_all_pos_act_patch_results, \n",
        "#        yaxis=\"Layer\", \n",
        "#        xaxis=\"Head\", \n",
        "#        title=\"IOI Metric for 'attn_head_out' Activation Patching (All Pos)\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 236
        },
        "id": "4_5KSLiq-lfT",
        "outputId": "207fb013-2ba3-4c70-b24a-8b9cee0af2a8"
      },
      "outputs": [],
      "source": [
        "from utils.visualization import l_scatter\n",
        "head_labels = [f\"L{l}H{h}\" for l in range(model.cfg.n_layers) for h in range(model.cfg.n_heads)]\n",
        "l_scatter(\n",
        "    x=utils.to_numpy(attn_head_pattern_all_pos_act_patch_results.flatten()), \n",
        "    y=utils.to_numpy(attn_head_out_all_pos_act_patch_results.flatten()), \n",
        "    hover_name = head_labels,\n",
        "    xaxis=\"Attention Patch\",\n",
        "    yaxis=\"Output Patch\",\n",
        "    title=\"Scatter plot of output patching vs attention patching\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### NMH Knockout"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### All Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "heads_to_ablate = DE_NMH\n",
        "\n",
        "print(f\"Heads to ablate: {heads_to_ablate}\")\n",
        "def ablate_top_head_hook(z: TT[\"batch\", \"pos\", \"head_index\", \"d_head\"], hook, head_idx=0):\n",
        "    z[:, :, head_idx, :] = 0\n",
        "    return z\n",
        "# Adds a hook into global model state\n",
        "for layer, head in heads_to_ablate:\n",
        "    ablate_head_hook = partial(ablate_top_head_hook, head_idx=head)\n",
        "    model.blocks[layer].attn.hook_z.add_hook(ablate_head_hook)\n",
        "# Runs the model, temporarily adds caching hooks and then removes *all* hooks after running, including the ablation hook.\n",
        "ablated_logits, ablated_cache = model.run_with_cache(ioi_dataset.toks)\n",
        "print(f\"Original IOI Metric: {_logits_to_ave_logit_diff(clean_logits, ioi_dataset).item():.4f}\")\n",
        "print(f\"Post ablation IOI Metric: {_logits_to_ave_logit_diff(ablated_logits, ioi_dataset).item()}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_head_ablated_residual, labels = ablated_cache.stack_head_results(layer=-1, pos_slice=-1, return_labels=True)\n",
        "per_head_ablated_logit_diffs = residual_stack_to_logit_diff(per_head_ablated_residual, logit_diff_directions, prompts, ablated_cache)\n",
        "per_head_ablated_logit_diffs = per_head_ablated_logit_diffs.reshape(model.cfg.n_layers, model.cfg.n_heads)\n",
        "imshow_n(per_head_ablated_logit_diffs, labels={\"x\":\"Head\", \"y\":\"Layer\"}, zmin=-1.5, zmax=1.5, title=\"Post-Ablation Direct Logit Attribution of Heads\")\n",
        "l_scatter(y=per_head_logit_diffs.flatten(), x=per_head_ablated_logit_diffs.flatten(), hover_name=head_labels, range_x=(-3, 3), range_y=(-3, 3), xaxis=\"Ablated\", yaxis=\"Original\", title=\"Original vs Post-Ablation Direct Logit Attribution of Heads\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "exclusions = [(6, 6), (7, 9), (8, 9)] + [(9, 1), (9, 5)]\n",
        "delta = per_head_ablated_logit_diffs - per_head_logit_diffs\n",
        "for layer, head in exclusions:\n",
        "    per_head_ablated_logit_diffs[layer, head] = 0\n",
        "\n",
        "plot_attention_heads(\n",
        "    per_head_ablated_logit_diffs/clean_logit_diff, \n",
        "    title=\"Logit Diff Contribution From Backup Heads\", \n",
        "    top_n=15, \n",
        "    range_x=[0, 0.5]\n",
        ")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### Individual Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# Get indices of all heads where the ablation had a positive effect\n",
        "delta = per_head_ablated_logit_diffs - per_head_logit_diffs\n",
        "backup_nmh_candidates = np.argwhere(delta.cpu().detach().numpy() > 0.05)\n",
        "backup_nmh_candidates = [tuple(h) for h in backup_nmh_candidates]\n",
        "backup_nmh_candidates = [h for h in backup_nmh_candidates if h not in exclusions]\n",
        "print(f\"Backup NMH Candidates: {backup_nmh_candidates}\")\n",
        "for l, h in backup_nmh_candidates:\n",
        "    for layer, head in heads_to_ablate:\n",
        "        ablate_head_hook = partial(ablate_top_head_hook, head_idx=head)\n",
        "        model.blocks[layer].attn.hook_z.add_hook(ablate_head_hook)\n",
        "    scatter_attention_and_contribution(model, (l, h), prompts, io_positions, s_positions, answer_residual_directions)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "attn_head_pattern_all_pos_act_patch_results['z'].shape"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 5\n",
        "top_heads = torch.topk(-attn_head_pattern_all_pos_act_patch_results['z'].flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, prompts, heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# V-weighted version\n",
        "plot_attention(\n",
        "    model, \n",
        "    prompts[0],\n",
        "    nmh_candidates,\n",
        "    clean_cache,\n",
        "    weighted=True)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "V7scfz6oqIWU"
      },
      "source": [
        "### Contributors to DE Heads"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Attention Out by Position"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "results = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode(node_names=[\"attn_out\"], seq_pos=\"each\"),\n",
        "    receiver_nodes=[Node(\"v\", layer, head=head) for layer, head in DE_S2I],\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True,\n",
        ")\n",
        "results = einops.rearrange(results['attn_out'], \"seq layer -> layer seq\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "imshow_n(\n",
        "        results * 100,\n",
        "        title=f\"Direct effect on DE Heads' values\",\n",
        "        xaxis=\"Pos\", \n",
        "        x=[f\"{tok} {i}\" for i, tok in enumerate(model.to_str_tokens(ioi_dataset.toks[0]))],\n",
        "        y=[f\"Layer {layer}\" for layer in range(model.cfg.n_layers)],\n",
        "        width=1500,\n",
        "        height=600,\n",
        "    )"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#####"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### Attention Out by Head"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "I91M2_BOe98X"
      },
      "outputs": [],
      "source": [
        "results = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode(\"z\"),\n",
        "    receiver_nodes=[Node(\"v\", layer, head=head) for layer, head in DE_S2I],\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "0RDS-RRP8mx6",
        "outputId": "433a4d26-ac34-4a03-9d88-97b2121b459c"
      },
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "        results[\"z\"][:10] * 100,\n",
        "        title=f\"Direct effect on NMH' queries\",\n",
        "        labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "        coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "        border=True,\n",
        "        width=700,\n",
        "        margin={\"r\": 100, \"l\": 100}\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(-results['z'].cuda(), top_n=10, range_x=[0, 1.0])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "top_k = 5\n",
        "DISPLAY_IDX = 0\n",
        "top_heads = torch.topk(-results['z'].flatten(), k=top_k).indices.cpu().numpy()\n",
        "heads = [(head // model.cfg.n_heads, head % model.cfg.n_heads) for head in top_heads]\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "qz8WYbfCUq6W"
      },
      "source": [
        "### Second Level"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "aB0fKK8OVy34"
      },
      "source": [
        "#### Attention Pattern for Second-Level Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 844
        },
        "id": "zXTbbs53V3bB",
        "outputId": "a3f1f366-2ae4-4f98-c885-8a08d0a8484d"
      },
      "outputs": [],
      "source": [
        "second_level_positive_heads = [(6, 6), (7, 2), (7, 9)]\n",
        "IE_S2I = second_level_positive_heads\n",
        "\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), second_level_positive_heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)\n",
        "\n",
        "#second_level_negative_heads = [(7, 8), (8, 10)]\n",
        "#visualize_attention_patterns(torch.tensor([l*12+h for l, h in second_level_negative_heads]), title=f\"Top Negative Second Level IOI Metric Heads\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "Qpk9IlyEOqwy",
        "outputId": "44e8a816-f1ff-47b8-e841-52854211c13e"
      },
      "outputs": [],
      "source": [
        "head_labels = [f\"L{l}H{h}\" for l in range(model.cfg.n_layers) for h in range(model.cfg.n_heads)]\n",
        "l_scatter(\n",
        "    x=utils.to_numpy(attn_head_v_all_pos_act_patch_results.flatten()), \n",
        "    y=utils.to_numpy(attn_head_out_all_pos_act_patch_results.flatten()), \n",
        "    xaxis=\"Value Patch\",\n",
        "    yaxis=\"Output Patch\",\n",
        "    #caxis=\"Layer\",\n",
        "    hover_name = head_labels,\n",
        "    color=einops.repeat(np.arange(model.cfg.n_layers), \"layer -> (layer head)\", head=model.cfg.n_heads),\n",
        "    range_x=(-1.5, 1.5),\n",
        "    range_y=(-1.5, 1.5),\n",
        "    title=\"Scatter plot of output patching vs value patching\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "s2i_candidates = [(6, 6), (7, 2), (7, 9), (8, 9)]\n",
        "#s2i_candidates = [(8, 9)]"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### S2I Knockout"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "##### All Heads"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "heads_to_ablate = s2i_candidates\n",
        "\n",
        "print(f\"Heads to ablate: {heads_to_ablate}\")\n",
        "def ablate_top_head_hook(z: TT[\"batch\", \"pos\", \"head_index\", \"d_head\"], hook, head_idx=0):\n",
        "    z[:, -1, head_idx, :] = 0\n",
        "    return z\n",
        "# Adds a hook into global model state\n",
        "for layer, head in heads_to_ablate:\n",
        "    ablate_head_hook = partial(ablate_top_head_hook, head_idx=head)\n",
        "    model.blocks[layer].attn.hook_z.add_hook(ablate_head_hook)\n",
        "# Runs the model, temporarily adds caching hooks and then removes *all* hooks after running, including the ablation hook.\n",
        "ablated_logits, ablated_cache = model.run_with_cache(clean_tokens)\n",
        "print(f\"Original IOI Metric: {ioi_metric(clean_logits).item():.4f}\")\n",
        "print(f\"Post ablation IOI Metric: {ioi_metric(ablated_logits).item()}\")\n",
        "#print(f\"Direct Logit Attribution of top name mover head: {per_head_logit_diffs.flatten()[top_name_mover].item()}\")\n",
        "#print(f\"Naive prediction of post ablation logit diff: {original_average_logit_diff - per_head_logit_diffs.flatten()[top_name_mover].item()}\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "per_head_ablated_residual, labels = ablated_cache.stack_head_results(layer=-1, pos_slice=-1, return_labels=True)\n",
        "per_head_ablated_logit_diffs = residual_stack_to_logit_diff(per_head_ablated_residual, ablated_cache)\n",
        "per_head_ablated_logit_diffs = per_head_ablated_logit_diffs.reshape(model.cfg.n_layers, model.cfg.n_heads)\n",
        "imshow(per_head_ablated_logit_diffs, labels={\"x\":\"Head\", \"y\":\"Layer\"})\n",
        "l_scatter(y=per_head_logit_diffs.flatten(), x=per_head_ablated_logit_diffs.flatten(), hover_name=head_labels, range_x=(-3, 3), range_y=(-3, 3), xaxis=\"Ablated\", yaxis=\"Original\", title=\"Original vs Post-Ablation Direct Logit Attribution of Heads\")"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "VTFd8gN-qUXf"
      },
      "source": [
        "#### Path Patching for S2-Inhibition Candidates"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Dy44bI-8qYky"
      },
      "outputs": [],
      "source": [
        "receiver_heads = second_level_positive_heads\n",
        "\n",
        "results = path_patch(\n",
        "    model,\n",
        "    orig_input=ioi_dataset.toks,\n",
        "    new_input=abc_dataset.toks,\n",
        "    sender_nodes=IterNode(\"z\"),\n",
        "    receiver_nodes=[Node(\"v\", layer, head=head) for layer, head in s2i_candidates],\n",
        "    patching_metric=logit_diff_noising_ioi,\n",
        "    verbose=True,\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 542
        },
        "id": "iXd5UX5mqbg2",
        "outputId": "775cb8b5-6d8b-4998-cc8b-4d180ad8f29b"
      },
      "outputs": [],
      "source": [
        "imshow_p(\n",
        "        results[\"z\"][:10] * 100,\n",
        "        title=f\"Direct effect on S2Is' values\",\n",
        "        labels={\"x\": \"Head\", \"y\": \"Layer\", \"color\": \"Logit diff variation\"},\n",
        "        coloraxis=dict(colorbar_ticksuffix = \"%\"),\n",
        "        border=True,\n",
        "        width=700,\n",
        "        margin={\"r\": 100, \"l\": 100}\n",
        "    )"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "plot_attention_heads(-results['z'].cuda(), top_n=10, range_x=[0, 1.0])"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "5rEXsaPhdF32"
      },
      "source": [
        "### Third Level"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "jGgbWtVEdKKO"
      },
      "source": [
        "#### Attention Patterns for Third-Level Heads"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "OQOHwopPgmTV"
      },
      "source": [
        "We have a mix of induction heads and duplicate token heads here, as well as two heads that focus on S2 at S2."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 780
        },
        "id": "d4dciSHKdPh-",
        "outputId": "633c04a1-4f18-4055-f196-d3d26ce96543"
      },
      "outputs": [],
      "source": [
        "third_level_positive_heads = [(4, 6), (4, 11)]\n",
        "DISPLAY_IDX = 0\n",
        "tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), third_level_positive_heads)\n",
        "#tokens, attn, names = get_attn_head_patterns(model, model.to_string((ioi_dataset.toks[DISPLAY_IDX][:ioi_dataset.word_idx[\"end\"][DISPLAY_IDX]+1])), second_level_positive_heads)\n",
        "cv.attention.attention_heads(tokens=tokens, attention=attn, attention_head_names=names)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Save the Circuit"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# define circuit\n",
        "from collections import namedtuple\n",
        "import pickle\n",
        "\n",
        "CircuitComponent = namedtuple(\n",
        "    \"CircuitComponent\", [\"heads\", \"position\", \"receiver_type\"]\n",
        ")\n",
        "\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        DE_NMH, -1, \"hook_q\"\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent(s2i_candidates, ioi_dataset.word_idx[\"S2\"], \"hook_v\"),\n",
        "    # \"duplicate-name\": CircuitComponent([(7, 15), (9, 1)], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}\n",
        "\n",
        "# Specify a filename for saving the circuit dictionary\n",
        "circuit_filename = 'results/circuits/pythia_160m_circuit.pkl'\n",
        "\n",
        "# Save the circuit dictionary using pickle\n",
        "with open(circuit_filename, 'wb') as f:\n",
        "    pickle.dump(circuit, f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "\n",
        "# save 410m circuit\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        [(17, 10), (17, 6), (17, 11), (18, 0), (18, 8), (18, 13), (18, 14)],\n",
        "        -1,\n",
        "        \"hook_q\",\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent(\n",
        "        [(11, 4), (13, 1), (13, 5), (16, 0)], 10, \"hook_v\"\n",
        "    ),\n",
        "    # \"duplicate-name\": CircuitComponent([], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# save 1.4b circuit\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        [(12, 15), (13, 1), (13, 6), (15, 15), (16, 13), (17, 7)], -1, \"hook_q\"\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent([(10, 7)], 10, \"hook_v\"),\n",
        "    # \"duplicate-name\": CircuitComponent([(7, 15), (9, 1)], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "# save 2.8b circuit\n",
        "circuit = {\n",
        "    \"name-movers\": CircuitComponent(\n",
        "        [(12, 15), (13, 1), (13, 6), (15, 15), (16, 13), (17, 7)], -1, \"hook_q\"\n",
        "    ),\n",
        "    \"s2-inhibition\": CircuitComponent([(10, 7)], 10, \"hook_v\"),\n",
        "    # \"duplicate-name\": CircuitComponent([(7, 15), (9, 1)], 10, 'head_v'),\n",
        "    # \"induction\": CircuitComponent([], 10, 'head_v')\n",
        "}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "circuit_file = \"pythia_1-4b_circuit.pkl\"\n",
        "circuit_root = \"results/circuits/\"\n",
        "with open(circuit_root + circuit_file, 'rb') as f:\n",
        "    circuit = pickle.load(f)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "res = torch.load(f\"results/pythia-1.4b-no-dropout/value_perf.pt\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "res[18, 12]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model = HookedTransformer.from_pretrained(\n",
        "    \"EleutherAI/pythia-1.4b\",\n",
        "    center_unembed=True,\n",
        "    center_writing_weights=True,\n",
        "    fold_ln=True,\n",
        "    refactor_factored_attn_matrices=False,\n",
        ")\n",
        "model.set_use_hook_mlp_in(True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "model.cfg"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "provenance": [],
      "toc_visible": true
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.19"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "00cb5a0118bb438fb38ca3bd304a3016": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b7021c7fad1643368f77f69290c1b5ae",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_65070d65c6bd4e47ae25410c379b0e2b",
            "value": 144
          }
        },
        "03d73123d0644c8c81d7a0b3f810b8da": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "044171c855c240ff923b69b4e27e1538": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "05e002ce5986414c816e4435c631404b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06227bde4fac46bbb093003e74953cad": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "063c166bc4b44f0292ca6ff9133cd0b7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "077b84575f3c4ca5955980a81949216b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "07cbbe5fb260457987b2464bb84268fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0953726feb674665894bddd714143d65": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_70f4de7a068b43dc80053d7508ae9cf4",
            "placeholder": "​",
            "style": "IPY_MODEL_f80833a8b8f442409dec918c4aba25c1",
            "value": "100%"
          }
        },
        "0abe281fc6e5443e87a9a7d28419783a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0bf040751e6841a79713aa8862ca5224": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b9c8ade472944666a4cc4f6793ec0f35",
            "placeholder": "​",
            "style": "IPY_MODEL_3ba2350417e54744b9ae1c5567362202",
            "value": " 144/144 [00:05&lt;00:00, 19.52it/s]"
          }
        },
        "0c1eb5b72748416ba1a78a06543a7bb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d4f67196e74e4ff9b5df7045ed98b17a",
            "placeholder": "​",
            "style": "IPY_MODEL_f5947be879d44715846b197614974298",
            "value": "100%"
          }
        },
        "0d12e07a6cd747588e0d8360a4d9a8d5": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5530412cffde446ba35ae281cbebf6c2",
            "placeholder": "​",
            "style": "IPY_MODEL_4821854623904c838635611c3725f278",
            "value": " 180/180 [00:06&lt;00:00, 28.90it/s]"
          }
        },
        "0ebe50c7f7404ef0926e461dc673b1c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8188934c142d4862aaf6559a5e1f60e2",
            "placeholder": "​",
            "style": "IPY_MODEL_25fbd835426d4ebab9eef02ed5cebc69",
            "value": "100%"
          }
        },
        "0ff7c562d0fe412595ffa3badb275e71": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "14d3b49b8dd542d298fef799a774ba91": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "15878b0ec9a247aca534f81378ff5fca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_184ffa21537a459789826d1348552f76",
            "placeholder": "​",
            "style": "IPY_MODEL_54171321fd0c470f84f1fe58eaccfa6a",
            "value": " 144/144 [00:06&lt;00:00, 20.08it/s]"
          }
        },
        "1620fe07e0c14088bc03ec51f6383846": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "184ffa21537a459789826d1348552f76": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1af9fa979cae41d4b95b3607e4969f80": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1b57fa43b71a4aba9a1c48b32e090a21": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d7716c911ef47caa5b1875234fb9ed8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d9762ab63cd4bdf8a40c92c0a3d17a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_71944864009f4728a831eb00192bcc24",
              "IPY_MODEL_9cdfb071b7924b1b92c45bbc3b77fdab",
              "IPY_MODEL_f031bcd482304cf9bbe669a1a1a92ff6"
            ],
            "layout": "IPY_MODEL_b151ec6b8b9543ee9fda04de3da221b5"
          }
        },
        "1ded46589d814836a88da906225cda46": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_64387ba7a3b04f5f8d844ee43146212d",
            "placeholder": "​",
            "style": "IPY_MODEL_3c84b4575d8d45c1866f771bb081aeb4",
            "value": " 144/144 [00:05&lt;00:00, 30.63it/s]"
          }
        },
        "1e837c99ea954afea4ad0fb78656c437": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1e85072a9aaa4fea99a0f6ea760901dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "21ec4ae4727f4308868d2922abae7dc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1af9fa979cae41d4b95b3607e4969f80",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_da24d7c9b66f42d7a5b0a2abff104d68",
            "value": 180
          }
        },
        "22a664b1fd2a4f9fbfd24f1081746fc0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "234abcfad26c4c04a5b66f76a7700401": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": "center",
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": "flex",
            "flex": null,
            "flex_flow": "column",
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": "50%"
          }
        },
        "25fbd835426d4ebab9eef02ed5cebc69": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "28e8be6b8d7245c395afa2b32f21470c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2979a00ad822456d80251bbc65086580": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0ebe50c7f7404ef0926e461dc673b1c0",
              "IPY_MODEL_d11dbd9d6497460580184f1b0050645e",
              "IPY_MODEL_15878b0ec9a247aca534f81378ff5fca"
            ],
            "layout": "IPY_MODEL_7ac670a352d44e51b9a3446b219edb21"
          }
        },
        "2b4ed578eab0412cab5191a1cff83364": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_96ad2b63efe349f2ab068599a0d8c74e",
              "IPY_MODEL_90ca171bbc754e089e502b53713e1188",
              "IPY_MODEL_50c94f552e0f4d90886bc24be7b1e198"
            ],
            "layout": "IPY_MODEL_df9ccf7fb6b149b6b13a81116f0623e9"
          }
        },
        "2c3112527ebd4790b5c9bc7fb84ff494": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6cf92f0162f946049b8d67f7a89dfc27",
            "placeholder": "​",
            "style": "IPY_MODEL_03d73123d0644c8c81d7a0b3f810b8da",
            "value": " 144/144 [00:05&lt;00:00, 34.23it/s]"
          }
        },
        "2f43dfcb6a6b46a0a36406767d040503": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2fbb8019e37846dab7d6445f91645506": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "30bd8772a8ca42a59c6db93865c61581": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "325651b3a4a3456396c773dacb8b693f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33519b6e3e8b497ebb22ca138ad08c0a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3407bb38c9c34c08a93eb3f95733b525": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3674a319ee694b939cd4ddda552948d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7fefca14bcba4b5092ab98577a088b9f",
            "placeholder": "​",
            "style": "IPY_MODEL_d466487165f84d89bf112aefb990e880",
            "value": "100%"
          }
        },
        "375136b0c3754f79825f46860f010a55": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "388d5aa11c7448ab82effc14c083dace": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "391dfa794c3941c0a608ac7d7353b494": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3ba2350417e54744b9ae1c5567362202": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3c84b4575d8d45c1866f771bb081aeb4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3dcc933a7d134c39ace2c20beee3028c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec4883abcba84051ae9d6c41ddefbcb6",
            "placeholder": "​",
            "style": "IPY_MODEL_a0e05b2d05ba4674983ec9349f57b8ca",
            "value": " 180/180 [00:07&lt;00:00, 28.93it/s]"
          }
        },
        "3e92516b195d413987ec326233626054": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46e0b6d1197741ee812720e6b599b6c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c8310b424eb941f6ae389d7de87b7287",
              "IPY_MODEL_8fb1bff2394a4aa492a717a0e7db3d2b",
              "IPY_MODEL_74ee415f994d4acf8d31af2e81a93123"
            ],
            "layout": "IPY_MODEL_044171c855c240ff923b69b4e27e1538"
          }
        },
        "480fb7bee6a341c4bf0234aa175bd5e8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4821854623904c838635611c3725f278": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4bf78c63b25f45dda35014e3b5f6fbce": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_325651b3a4a3456396c773dacb8b693f",
            "placeholder": "​",
            "style": "IPY_MODEL_7fbcaecd08c24b0c895863ca04bb823e",
            "value": " 180/180 [00:06&lt;00:00, 21.78it/s]"
          }
        },
        "4d2ede20c4484505a9d67a9d9300ef15": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4ecd7f1f2b634facbb9b2e48c31295ed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "508c777cf4ba400388cabd026ed1a08c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "50c94f552e0f4d90886bc24be7b1e198": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_82a9834fb9bc44199e454e8b92bbcdee",
            "placeholder": "​",
            "style": "IPY_MODEL_2fbb8019e37846dab7d6445f91645506",
            "value": " 2160/2160 [01:26&lt;00:00, 22.69it/s]"
          }
        },
        "51efc4c80b20442ea75c68a7f2afbaed": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "52ef2d85092f4e38aa2af174f41c233a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54171321fd0c470f84f1fe58eaccfa6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5492c6b666a84d59bd64c4b6cba9108f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5530412cffde446ba35ae281cbebf6c2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "576a2c6d603d4c0486684a47c115f6c3": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "577743f69ee944c6b8efdad9fdde6627": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0953726feb674665894bddd714143d65",
              "IPY_MODEL_7f122c7354974bb9a6eea53582c0f12d",
              "IPY_MODEL_bd1c647c971a474291feac1b2d87a778"
            ],
            "layout": "IPY_MODEL_06227bde4fac46bbb093003e74953cad"
          }
        },
        "5a6db6d2bd9c4d5697041d3b6cfc2ed6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5b4e510574df4e6dae68e7cbeda49029",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c0047268b94745a8b8010d06f9315199",
            "value": 144
          }
        },
        "5af261b7e4754241abd86e41c98f2750": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5b29a1117851485e9333114fabb4e072": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b4e510574df4e6dae68e7cbeda49029": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ec1577140e64c34be0d2860ed137766": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "CheckboxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "CheckboxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "CheckboxView",
            "description": "Add token as git credential?",
            "description_tooltip": null,
            "disabled": false,
            "indent": true,
            "layout": "IPY_MODEL_1d7716c911ef47caa5b1875234fb9ed8",
            "style": "IPY_MODEL_c2593312b66d49bd9119824ba09aa2be",
            "value": true
          }
        },
        "60ec18de81af433880b6167a01fba30f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "621c519f5bbe477ab1d6ba8f740cc382": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6355e7df26a141b183bdfb7f8be79778": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8501c54f82c84882a8705833798ecfb4",
            "placeholder": "​",
            "style": "IPY_MODEL_e39919bc2f864f57abc72184365887ec",
            "value": "100%"
          }
        },
        "63bd8043677442e696ccc0d8a3fe5839": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_508c777cf4ba400388cabd026ed1a08c",
            "placeholder": "​",
            "style": "IPY_MODEL_30bd8772a8ca42a59c6db93865c61581",
            "value": " 180/180 [00:07&lt;00:00, 29.38it/s]"
          }
        },
        "64387ba7a3b04f5f8d844ee43146212d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64e799df39eb44e59b36212d393ba581": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7a4e3ed388964a4da909c7210ead5b9a",
              "IPY_MODEL_00cb5a0118bb438fb38ca3bd304a3016",
              "IPY_MODEL_6ce55cdccad141e5a380f4eb07e96033"
            ],
            "layout": "IPY_MODEL_077b84575f3c4ca5955980a81949216b"
          }
        },
        "65070d65c6bd4e47ae25410c379b0e2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6541cdb41ba34ae1a8fdf6d42bbe61d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "654bc4b795c94ef280027a0207c5e14f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65917c03b217485bac9bdd53e962def8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65ca015c87154fdb86e63727041b946c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66ff90fae10642eca0fa5e631447975b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6802a79e197b4396b127eb6fb9ea29e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8c389ea18e7c435a801c8f11fa508cf0",
              "IPY_MODEL_b5643a5bb9c54a0983f5b0aefb19d050",
              "IPY_MODEL_bd9df13e01aa4795bb3a7e644dde2bd1"
            ],
            "layout": "IPY_MODEL_4ecd7f1f2b634facbb9b2e48c31295ed"
          }
        },
        "6ce55cdccad141e5a380f4eb07e96033": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ec8361b504df49e7a1ffb60ece4355fd",
            "placeholder": "​",
            "style": "IPY_MODEL_391dfa794c3941c0a608ac7d7353b494",
            "value": " 144/144 [00:05&lt;00:00, 30.34it/s]"
          }
        },
        "6cf92f0162f946049b8d67f7a89dfc27": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e8904a58f3a4ce3856699fb7000d725": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "70259059c97e4238b778cb1fb53f29e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f5d17117789a44bd8e16584a2f939115",
              "IPY_MODEL_78024e1dc6434eb58caccc64d36d82c3",
              "IPY_MODEL_0bf040751e6841a79713aa8862ca5224"
            ],
            "layout": "IPY_MODEL_654bc4b795c94ef280027a0207c5e14f"
          }
        },
        "70f4de7a068b43dc80053d7508ae9cf4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "71944864009f4728a831eb00192bcc24": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_28e8be6b8d7245c395afa2b32f21470c",
            "placeholder": "​",
            "style": "IPY_MODEL_5492c6b666a84d59bd64c4b6cba9108f",
            "value": "100%"
          }
        },
        "74ee415f994d4acf8d31af2e81a93123": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7d3c726ca3fb48de98432b35c8550c0b",
            "placeholder": "​",
            "style": "IPY_MODEL_5af261b7e4754241abd86e41c98f2750",
            "value": " 180/180 [00:06&lt;00:00, 29.15it/s]"
          }
        },
        "75995faef432410299c34717b1da96e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_9b1f31b34c9342fc8a1b2bfef0cf1f29",
              "IPY_MODEL_81e526b016b7440abf420cc44f64b236",
              "IPY_MODEL_3dcc933a7d134c39ace2c20beee3028c"
            ],
            "layout": "IPY_MODEL_9c78081eb3fe4146a1d79df360cd66eb"
          }
        },
        "7793574382e94f4fb1eb22c287bcfb02": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78024e1dc6434eb58caccc64d36d82c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_05e002ce5986414c816e4435c631404b",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bfcf9873f5f14d559f3f0906d6d4b62d",
            "value": 144
          }
        },
        "78715c8355a84e7a950ee5e0541d598f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ed976a6e6c9f40b2810d7869459711d4",
              "IPY_MODEL_21ec4ae4727f4308868d2922abae7dc7",
              "IPY_MODEL_4bf78c63b25f45dda35014e3b5f6fbce"
            ],
            "layout": "IPY_MODEL_84bbf960ddee46e39a41007929a386f1"
          }
        },
        "7a4e3ed388964a4da909c7210ead5b9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52ef2d85092f4e38aa2af174f41c233a",
            "placeholder": "​",
            "style": "IPY_MODEL_a6229b8c298947498453f64d0e90d004",
            "value": "100%"
          }
        },
        "7ac670a352d44e51b9a3446b219edb21": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7d3c726ca3fb48de98432b35c8550c0b": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f122c7354974bb9a6eea53582c0f12d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9497f8b2ca644fa981a0acf97fcb8ef8",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_aae2ea570bca4227952e0cf9e9b39243",
            "value": 144
          }
        },
        "7f924ae9b05f451b9a6c8eb4920f2c2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_addabcbe6ef94a49b1ab76027ce7bd31",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1e837c99ea954afea4ad0fb78656c437",
            "value": 180
          }
        },
        "7fbcaecd08c24b0c895863ca04bb823e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7fefca14bcba4b5092ab98577a088b9f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8109a631fa3f4443885ecd6953c1137e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "811ebb000e664308b9a9dc519a70deab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "8188934c142d4862aaf6559a5e1f60e2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81e526b016b7440abf420cc44f64b236": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1b57fa43b71a4aba9a1c48b32e090a21",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_375136b0c3754f79825f46860f010a55",
            "value": 180
          }
        },
        "825fb3ad8d7c408f97c2a2939ca0640d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b012ced6282544ffa601fcdf819930c7",
              "IPY_MODEL_91d87964d9a74543b9c95c80f3371187",
              "IPY_MODEL_e98f8f6751114942bbd8bf8d89366e41"
            ],
            "layout": "IPY_MODEL_22a664b1fd2a4f9fbfd24f1081746fc0"
          }
        },
        "82a9834fb9bc44199e454e8b92bbcdee": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "830fda49b567472b91d0c2c210db34cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8d3b5565501247f7954ae0b18dba7c58",
              "IPY_MODEL_7f924ae9b05f451b9a6c8eb4920f2c2b",
              "IPY_MODEL_63bd8043677442e696ccc0d8a3fe5839"
            ],
            "layout": "IPY_MODEL_91970688d85e42eb90aa86ccd0a91be7"
          }
        },
        "84bbf960ddee46e39a41007929a386f1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8501c54f82c84882a8705833798ecfb4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "886acd0422994886888136cc71f23519": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88861a3614834f7fb3bdc99de0d2baeb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8c04505825f449bf8669c1ad29ee706c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8c21b9e91a28441f8297d3a04f5b4d9a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8c389ea18e7c435a801c8f11fa508cf0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ff227884d1644848b0185b93c0fa6868",
            "placeholder": "​",
            "style": "IPY_MODEL_07cbbe5fb260457987b2464bb84268fe",
            "value": "100%"
          }
        },
        "8ca69d3bc7d5458288f648a3764b3a84": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8d0ec28c53b042f78568e8d2323e4d29": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3407bb38c9c34c08a93eb3f95733b525",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ec0581dd1df844639b90b53a455454f9",
            "value": 180
          }
        },
        "8d3b5565501247f7954ae0b18dba7c58": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdffa3c04b2a4318adac0c9070e40200",
            "placeholder": "​",
            "style": "IPY_MODEL_8109a631fa3f4443885ecd6953c1137e",
            "value": "100%"
          }
        },
        "8e7c46f6e6624ea8926d067813711c89": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fb1bff2394a4aa492a717a0e7db3d2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66ff90fae10642eca0fa5e631447975b",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_88861a3614834f7fb3bdc99de0d2baeb",
            "value": 180
          }
        },
        "9025a4ebdb5f4881aaf62e5f84e6414d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3674a319ee694b939cd4ddda552948d4",
              "IPY_MODEL_8d0ec28c53b042f78568e8d2323e4d29",
              "IPY_MODEL_0d12e07a6cd747588e0d8360a4d9a8d5"
            ],
            "layout": "IPY_MODEL_8e7c46f6e6624ea8926d067813711c89"
          }
        },
        "90ca171bbc754e089e502b53713e1188": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_576a2c6d603d4c0486684a47c115f6c3",
            "max": 2160,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eaa96b03fde14c8298310984e35dfda6",
            "value": 2160
          }
        },
        "91970688d85e42eb90aa86ccd0a91be7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91d87964d9a74543b9c95c80f3371187": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_51efc4c80b20442ea75c68a7f2afbaed",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8ca69d3bc7d5458288f648a3764b3a84",
            "value": 144
          }
        },
        "9408900617b942e394ced6a0d8b4d799": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ButtonModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ButtonView",
            "button_style": "",
            "description": "Login",
            "disabled": false,
            "icon": "",
            "layout": "IPY_MODEL_ce5debeb4ec54287a2cc248cdb637651",
            "style": "IPY_MODEL_b21edc97e6b04b3ea5eee2aae3f4eb86",
            "tooltip": ""
          }
        },
        "9497f8b2ca644fa981a0acf97fcb8ef8": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96ad2b63efe349f2ab068599a0d8c74e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_063c166bc4b44f0292ca6ff9133cd0b7",
            "placeholder": "​",
            "style": "IPY_MODEL_fffaff2fb61446a5b900e35345b2b5eb",
            "value": "100%"
          }
        },
        "97e98f74d3b74e38be50fb205d6f1efd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b1f31b34c9342fc8a1b2bfef0cf1f29": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0abe281fc6e5443e87a9a7d28419783a",
            "placeholder": "​",
            "style": "IPY_MODEL_aeb95b87161f46f6bf173943b77a4e52",
            "value": "100%"
          }
        },
        "9c78081eb3fe4146a1d79df360cd66eb": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9cdfb071b7924b1b92c45bbc3b77fdab": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8c04505825f449bf8669c1ad29ee706c",
            "max": 180,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8c21b9e91a28441f8297d3a04f5b4d9a",
            "value": 180
          }
        },
        "a0e05b2d05ba4674983ec9349f57b8ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a6229b8c298947498453f64d0e90d004": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aae2ea570bca4227952e0cf9e9b39243": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "addabcbe6ef94a49b1ab76027ce7bd31": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "aeb95b87161f46f6bf173943b77a4e52": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b012ced6282544ffa601fcdf819930c7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65ca015c87154fdb86e63727041b946c",
            "placeholder": "​",
            "style": "IPY_MODEL_811ebb000e664308b9a9dc519a70deab",
            "value": "100%"
          }
        },
        "b08c648d67474d9eb9986ea24f28311b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d7754c6b0a7a4ba6b40b3c1ffc458b55",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f0f946dd323d47f88ed986c4c0a2537f",
            "value": 144
          }
        },
        "b151ec6b8b9543ee9fda04de3da221b5": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b21edc97e6b04b3ea5eee2aae3f4eb86": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ButtonStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ButtonStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "button_color": null,
            "font_weight": ""
          }
        },
        "b2b7a69c70d7489b8dd9cd4475fb22f9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b5643a5bb9c54a0983f5b0aefb19d050": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4d2ede20c4484505a9d67a9d9300ef15",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1620fe07e0c14088bc03ec51f6383846",
            "value": 144
          }
        },
        "b7021c7fad1643368f77f69290c1b5ae": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b76c6442906a46f0b164449244f518eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "VBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "VBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "VBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b95e718ee1a0448aa9150505e439fad3",
              "IPY_MODEL_bb7569ae4d9e4996a7c74e6d6b0e172e",
              "IPY_MODEL_5ec1577140e64c34be0d2860ed137766",
              "IPY_MODEL_9408900617b942e394ced6a0d8b4d799",
              "IPY_MODEL_d3aeb6e3ae164e33812fec42f2205e08"
            ],
            "layout": "IPY_MODEL_234abcfad26c4c04a5b66f76a7700401"
          }
        },
        "b95e718ee1a0448aa9150505e439fad3": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7793574382e94f4fb1eb22c287bcfb02",
            "placeholder": "​",
            "style": "IPY_MODEL_6541cdb41ba34ae1a8fdf6d42bbe61d8",
            "value": "<center> <img\nsrc=https://huggingface.co/front/assets/huggingface_logo-noborder.svg\nalt='Hugging Face'> <br> Copy a token from <a\nhref=\"https://huggingface.co/settings/tokens\" target=\"_blank\">your Hugging Face\ntokens page</a> and paste it below. <br> Immediately click login after copying\nyour token or it might be stored in plain text in this notebook file. </center>"
          }
        },
        "b9c8ade472944666a4cc4f6793ec0f35": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb40ed2d47ed4e3cbe89e68e816677e4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0c1eb5b72748416ba1a78a06543a7bb8",
              "IPY_MODEL_b08c648d67474d9eb9986ea24f28311b",
              "IPY_MODEL_1ded46589d814836a88da906225cda46"
            ],
            "layout": "IPY_MODEL_2f43dfcb6a6b46a0a36406767d040503"
          }
        },
        "bb7569ae4d9e4996a7c74e6d6b0e172e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "PasswordModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "PasswordModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "PasswordView",
            "continuous_update": true,
            "description": "Token:",
            "description_tooltip": null,
            "disabled": false,
            "layout": "IPY_MODEL_5b29a1117851485e9333114fabb4e072",
            "placeholder": "​",
            "style": "IPY_MODEL_bc8a2cda603a424299c53080b6050273",
            "value": ""
          }
        },
        "bc8a2cda603a424299c53080b6050273": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bd1c647c971a474291feac1b2d87a778": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2b7a69c70d7489b8dd9cd4475fb22f9",
            "placeholder": "​",
            "style": "IPY_MODEL_621c519f5bbe477ab1d6ba8f740cc382",
            "value": " 144/144 [00:04&lt;00:00, 28.77it/s]"
          }
        },
        "bd9df13e01aa4795bb3a7e644dde2bd1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_886acd0422994886888136cc71f23519",
            "placeholder": "​",
            "style": "IPY_MODEL_14d3b49b8dd542d298fef799a774ba91",
            "value": " 144/144 [00:05&lt;00:00, 30.99it/s]"
          }
        },
        "bfcf9873f5f14d559f3f0906d6d4b62d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c0047268b94745a8b8010d06f9315199": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c0d8e052762f4742a4589527fced5fd2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6355e7df26a141b183bdfb7f8be79778",
              "IPY_MODEL_5a6db6d2bd9c4d5697041d3b6cfc2ed6",
              "IPY_MODEL_2c3112527ebd4790b5c9bc7fb84ff494"
            ],
            "layout": "IPY_MODEL_ce028da02fad43608ca2d68936a66129"
          }
        },
        "c2593312b66d49bd9119824ba09aa2be": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c8310b424eb941f6ae389d7de87b7287": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_65917c03b217485bac9bdd53e962def8",
            "placeholder": "​",
            "style": "IPY_MODEL_60ec18de81af433880b6167a01fba30f",
            "value": "100%"
          }
        },
        "cda06f33eb604e25aea0c0b09d47827e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "cdffa3c04b2a4318adac0c9070e40200": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce028da02fad43608ca2d68936a66129": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce5debeb4ec54287a2cc248cdb637651": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d11dbd9d6497460580184f1b0050645e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e92516b195d413987ec326233626054",
            "max": 144,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_cda06f33eb604e25aea0c0b09d47827e",
            "value": 144
          }
        },
        "d3aeb6e3ae164e33812fec42f2205e08": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb65972194544c41b8977d99eaa97718",
            "placeholder": "​",
            "style": "IPY_MODEL_97e98f74d3b74e38be50fb205d6f1efd",
            "value": "\n<b>Pro Tip:</b> If you don't already have one, you can create a dedicated\n'notebooks' token with 'write' access, that you can then easily reuse for all\nnotebooks. </center>"
          }
        },
        "d466487165f84d89bf112aefb990e880": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d4f67196e74e4ff9b5df7045ed98b17a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d7754c6b0a7a4ba6b40b3c1ffc458b55": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "da24d7c9b66f42d7a5b0a2abff104d68": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "db81ac65ab5847819ccf4c083d049302": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df27d8e8836146c7a5ff13850032924f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "df9ccf7fb6b149b6b13a81116f0623e9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e39919bc2f864f57abc72184365887ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e98f8f6751114942bbd8bf8d89366e41": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_df27d8e8836146c7a5ff13850032924f",
            "placeholder": "​",
            "style": "IPY_MODEL_388d5aa11c7448ab82effc14c083dace",
            "value": " 144/144 [00:04&lt;00:00, 29.19it/s]"
          }
        },
        "eaa96b03fde14c8298310984e35dfda6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ec0581dd1df844639b90b53a455454f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ec4883abcba84051ae9d6c41ddefbcb6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ec8361b504df49e7a1ffb60ece4355fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ed976a6e6c9f40b2810d7869459711d4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e8904a58f3a4ce3856699fb7000d725",
            "placeholder": "​",
            "style": "IPY_MODEL_480fb7bee6a341c4bf0234aa175bd5e8",
            "value": "100%"
          }
        },
        "f031bcd482304cf9bbe669a1a1a92ff6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db81ac65ab5847819ccf4c083d049302",
            "placeholder": "​",
            "style": "IPY_MODEL_1e85072a9aaa4fea99a0f6ea760901dd",
            "value": " 180/180 [00:06&lt;00:00, 27.93it/s]"
          }
        },
        "f0f946dd323d47f88ed986c4c0a2537f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f5947be879d44715846b197614974298": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5d17117789a44bd8e16584a2f939115": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_33519b6e3e8b497ebb22ca138ad08c0a",
            "placeholder": "​",
            "style": "IPY_MODEL_0ff7c562d0fe412595ffa3badb275e71",
            "value": "100%"
          }
        },
        "f80833a8b8f442409dec918c4aba25c1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "fb65972194544c41b8977d99eaa97718": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ff227884d1644848b0185b93c0fa6868": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fffaff2fb61446a5b900e35345b2b5eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}