notebooks/Ecco_Output_Token_Scores.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "02 - Output Token Scores.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNORIPvKnAcckSeyL7zIJjU",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/jalammar/ecco/blob/main/notebooks/Ecco_Output_Token_Scores.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jVUTEvBOJMuu"
      },
      "source": [
        "!pip install ecco"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wv3l86vBJNpY"
      },
      "source": [
        "import ecco\n",
        "lm = ecco.from_pretrained('distilgpt2')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "J9LbZ76mWhUs"
      },
      "source": [
        "## Fill in the blank: \"1, 1, 1, ____\""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        },
        "id": "CYbyc2uMWXcE",
        "outputId": "f79c337a-55bc-4863-d190-40935adba75e"
      },
      "source": [
        "# Generate one token to fill in the blank\r\n",
        "output_0 = lm.generate(\" 1, 1, 1,\", generate=1, do_sample=False)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "\n",
              "         requirejs( ['basic', 'ecco'], function(basic, ecco){\n",
              "            basic.init('viz_127577')\n",
              "\n",
              "            window.ecco['viz_127577'] = ecco.renderOutputSequence('viz_127577', {'tokens': [{'token': ' 1', 'position': 0, 'token_id': 352, 'type': 'input'}, {'token': ',', 'position': 1, 'token_id': 11, 'type': 'input'}, {'token': ' 1', 'position': 2, 'token_id': 352, 'type': 'input'}, {'token': ',', 'position': 3, 'token_id': 11, 'type': 'input'}, {'token': ' 1', 'position': 4, 'token_id': 352, 'type': 'input'}, {'token': ',', 'position': 5, 'token_id': 11, 'type': 'input'}]})\n",
              "         }, function (err) {\n",
              "            console.log(err);\n",
              "        })\n"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        // We don't really need these require scripts. But this is to avert\n",
              "        //this code from running before display_input_sequence which DOES require external files\n",
              "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "                console.log('addToken viz_id', 'viz_127577');\n",
              "                window.ecco['viz_127577'].addToken({\"token\": \" 1\", \"token_id\": 352, \"position\": 6, \"type\": \"output\"})\n",
              "                window.ecco['viz_127577'].redraw()\n",
              "        })\n",
              "        "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "id": "gqBY9Y0HWcgD",
        "outputId": "24dd1ec6-d84c-477d-c52b-515c95e7e59f"
      },
      "source": [
        "# Visualize\r\n",
        "output_0.layer_predictions(position=6, layer=5)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions for position 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "            const viz_id = basic.init()\n",
              "\n",
              "\n",
              "            let pred = new ecco.LayerPredictions({\n",
              "                parentDiv: viz_id,\n",
              "                data:[[{\"token\": \" 1\", \"prob\": \"0.59793174\", \"ranking\": 1, \"layer\": 5}, {\"token\": \" 2\", \"prob\": \"0.1766438\", \"ranking\": 2, \"layer\": 5}, {\"token\": \" 0\", \"prob\": \"0.047203474\", \"ranking\": 3, \"layer\": 5}, {\"token\": \" 3\", \"prob\": \"0.04305835\", \"ranking\": 4, \"layer\": 5}, {\"token\": \" 4\", \"prob\": \"0.014155654\", \"ranking\": 5, \"layer\": 5}, {\"token\": \" 5\", \"prob\": \"0.012131831\", \"ranking\": 6, \"layer\": 5}, {\"token\": \" and\", \"prob\": \"0.010471825\", \"ranking\": 7, \"layer\": 5}, {\"token\": \"1\", \"prob\": \"0.008584005\", \"ranking\": 8, \"layer\": 5}, {\"token\": \" 6\", \"prob\": \"0.006193402\", \"ranking\": 9, \"layer\": 5}, {\"token\": \" 10\", \"prob\": \"0.0041250973\", \"ranking\": 10, \"layer\": 5}]]\n",
              "            })\n",
              "            pred.init()\n",
              "         }, function (err) {\n",
              "            console.log(viz_id, err);\n",
              "        })"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t_-xqYY7Wqab"
      },
      "source": [
        "## Fill in the blank: \"Heathrow airport is located in the city of ____\""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tQp2xgISJPMu",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49
        },
        "outputId": "a67c6e21-648c-41d0-e850-50e72fbe0080"
      },
      "source": [
        "text= \" Heathrow airport is in the city of\"\n",
        "\n",
        "output = lm.generate(text, generate=1, do_sample=False)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "\n",
              "         requirejs( ['basic', 'ecco'], function(basic, ecco){\n",
              "            basic.init('viz_209792')\n",
              "\n",
              "            window.ecco['viz_209792'] = ecco.renderOutputSequence('viz_209792', {'tokens': [{'token': ' Heath', 'position': 0, 'token_id': 25846, 'type': 'input'}, {'token': 'row', 'position': 1, 'token_id': 808, 'type': 'input'}, {'token': ' airport', 'position': 2, 'token_id': 9003, 'type': 'input'}, {'token': ' is', 'position': 3, 'token_id': 318, 'type': 'input'}, {'token': ' in', 'position': 4, 'token_id': 287, 'type': 'input'}, {'token': ' the', 'position': 5, 'token_id': 262, 'type': 'input'}, {'token': ' city', 'position': 6, 'token_id': 1748, 'type': 'input'}, {'token': ' of', 'position': 7, 'token_id': 286, 'type': 'input'}]})\n",
              "         }, function (err) {\n",
              "            console.log(err);\n",
              "        })\n"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "        // We don't really need these require scripts. But this is to avert\n",
              "        //this code from running before display_input_sequence which DOES require external files\n",
              "        requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "                console.log('addToken viz_id', 'viz_209792');\n",
              "                window.ecco['viz_209792'].addToken({\"token\": \" London\", \"token_id\": 3576, \"position\": 8, \"type\": \"output\"})\n",
              "                window.ecco['viz_209792'].redraw()\n",
              "        })\n",
              "        "
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "czxFXc8kJb-d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 175
        },
        "outputId": "42aa023c-3b7d-4f5d-d75f-f826c880643f"
      },
      "source": [
        "# Visualize the candidate tokens at the last layer of the model (layer 5)\r\n",
        "output.layer_predictions(position=8, layer=5) \r\n"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions for position 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "            const viz_id = basic.init()\n",
              "\n",
              "\n",
              "            let pred = new ecco.LayerPredictions({\n",
              "                parentDiv: viz_id,\n",
              "                data:[[{\"token\": \" London\", \"prob\": \"0.12731227\", \"ranking\": 1, \"layer\": 5}, {\"token\": \" Birmingham\", \"prob\": \"0.080451205\", \"ranking\": 2, \"layer\": 5}, {\"token\": \" Manchester\", \"prob\": \"0.04099104\", \"ranking\": 3, \"layer\": 5}, {\"token\": \" Belfast\", \"prob\": \"0.018735664\", \"ranking\": 4, \"layer\": 5}, {\"token\": \" Glasgow\", \"prob\": \"0.018269148\", \"ranking\": 5, \"layer\": 5}, {\"token\": \" Leeds\", \"prob\": \"0.015156696\", \"ranking\": 6, \"layer\": 5}, {\"token\": \" Liverpool\", \"prob\": \"0.014440072\", \"ranking\": 7, \"layer\": 5}, {\"token\": \" Sheffield\", \"prob\": \"0.012512773\", \"ranking\": 8, \"layer\": 5}, {\"token\": \" Edinburgh\", \"prob\": \"0.012484929\", \"ranking\": 9, \"layer\": 5}, {\"token\": \" Brighton\", \"prob\": \"0.0116892895\", \"ranking\": 10, \"layer\": 5}]]\n",
              "            })\n",
              "            pred.init()\n",
              "         }, function (err) {\n",
              "            console.log(viz_id, err);\n",
              "        })"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 257
        },
        "id": "zQzva4T_XXWG",
        "outputId": "ea3d3dfb-ad4a-4b78-e2ff-a063639ba8dd"
      },
      "source": [
        "# We can see more tokens using the topk parameter\r\n",
        "output.layer_predictions(position=8, layer=5, topk=20)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions for position 8\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "            const viz_id = basic.init()\n",
              "\n",
              "\n",
              "            let pred = new ecco.LayerPredictions({\n",
              "                parentDiv: viz_id,\n",
              "                data:[[{\"token\": \" London\", \"prob\": \"0.12731227\", \"ranking\": 1, \"layer\": 5}, {\"token\": \" Birmingham\", \"prob\": \"0.080451205\", \"ranking\": 2, \"layer\": 5}, {\"token\": \" Manchester\", \"prob\": \"0.04099104\", \"ranking\": 3, \"layer\": 5}, {\"token\": \" Belfast\", \"prob\": \"0.018735664\", \"ranking\": 4, \"layer\": 5}, {\"token\": \" Glasgow\", \"prob\": \"0.018269148\", \"ranking\": 5, \"layer\": 5}, {\"token\": \" Leeds\", \"prob\": \"0.015156696\", \"ranking\": 6, \"layer\": 5}, {\"token\": \" Liverpool\", \"prob\": \"0.014440072\", \"ranking\": 7, \"layer\": 5}, {\"token\": \" Sheffield\", \"prob\": \"0.012512773\", \"ranking\": 8, \"layer\": 5}, {\"token\": \" Edinburgh\", \"prob\": \"0.012484929\", \"ranking\": 9, \"layer\": 5}, {\"token\": \" Brighton\", \"prob\": \"0.0116892895\", \"ranking\": 10, \"layer\": 5}, {\"token\": \" Bristol\", \"prob\": \"0.010995406\", \"ranking\": 11, \"layer\": 5}, {\"token\": \" Cov\", \"prob\": \"0.009609883\", \"ranking\": 12, \"layer\": 5}, {\"token\": \" Dund\", \"prob\": \"0.008984293\", \"ranking\": 13, \"layer\": 5}, {\"token\": \" Dublin\", \"prob\": \"0.008965806\", \"ranking\": 14, \"layer\": 5}, {\"token\": \" Newcastle\", \"prob\": \"0.008799739\", \"ranking\": 15, \"layer\": 5}, {\"token\": \" St\", \"prob\": \"0.0084905\", \"ranking\": 16, \"layer\": 5}, {\"token\": \" D\", \"prob\": \"0.00789767\", \"ranking\": 17, \"layer\": 5}, {\"token\": \" Nottingham\", \"prob\": \"0.0071086255\", \"ranking\": 18, \"layer\": 5}, {\"token\": \" Toronto\", \"prob\": \"0.0069225314\", \"ranking\": 19, \"layer\": 5}, {\"token\": \" Chelsea\", \"prob\": \"0.0068343165\", \"ranking\": 20, \"layer\": 5}]]\n",
              "            })\n",
              "            pred.init()\n",
              "         }, function (err) {\n",
              "            console.log(viz_id, err);\n",
              "        })"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GIS5Kw_GJiAv",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 857
        },
        "outputId": "fc285a51-78f4-4bfd-92f6-a213fb22eaac"
      },
      "source": [
        "# Visualize the candidate tokens at every layer\r\n",
        "output.layer_predictions(position=6)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Predictions for position 6\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<html lang=\"en\">\n",
              "<script src=\"https://requirejs.org/docs/release/2.3.6/minified/require.js\"></script>\n",
              "<script>\n",
              "    var ecco_url = 'https://storage.googleapis.com/ml-intro/ecco/'\n",
              "    // var ecco_url = 'http://localhost:8000/ecco/'\n",
              "\n",
              "    if (window.ecco === undefined) window.ecco = {}\n",
              "\n",
              "    requirejs.config({\n",
              "        urlArgs: \"bust=\" + (new Date()).getTime(),\n",
              "        nodeRequire: require,\n",
              "        paths: {\n",
              "            d3: \"https://d3js.org/d3.v5.min\",\n",
              "            \"d3-array\": \"https://d3js.org/d3-array.v2.min\",\n",
              "            jquery: \"https://code.jquery.com/jquery-3.5.1.min\",\n",
              "            // oldEcco: ecco_url + 'ecco',\n",
              "            ecco: ecco_url + 'js/0.0.4/ecco-bundle.min',\n",
              "            xregexp: 'https://cdnjs.cloudflare.com/ajax/libs/xregexp/3.2.0/xregexp-all.min'\n",
              "        }\n",
              "    });\n",
              "    requirejs(['d3'],\n",
              "        function (d3) {\n",
              "            d3.select('#css').attr('href', ecco_url + 'html/styles.css')\n",
              "        })\n",
              "\n",
              "    console.log('Ecco initialize.')\n",
              "</script>\n",
              "\n",
              "<head>\n",
              "    <link id='css' rel=\"stylesheet\" type=\"text/css\">\n",
              "</head>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<script>\n",
              "    // console.log('defining basic')\n",
              "    define('basic', ['d3'],\n",
              "        function (d3) {\n",
              "            return {\n",
              "                init: function (viz_id = null) {\n",
              "\n",
              "                    // console.log('basic init')\n",
              "                    if (viz_id == null){\n",
              "                        viz_id = \"viz_\" + Math.round(Math.random() * 10000000)\n",
              "                    }\n",
              "                    const div = d3.select('#basic').attr('id', viz_id),\n",
              "                        div_parent = d3.select('#' + viz_id).node().parentNode\n",
              "\n",
              "                    // Link to CSS file\n",
              "                    d3.select(div_parent).insert('link')\n",
              "                        .attr('rel', 'stylesheet')\n",
              "                        .attr('type', 'text/css')\n",
              "                        .attr('href', ecco_url + 'html/styles.css')\n",
              "\n",
              "                    return viz_id\n",
              "                }\n",
              "            }\n",
              "        }, function (err) {console.log(err);}\n",
              "    )\n",
              "    // console.log('after defining basic')\n",
              "\n",
              "</script>\n",
              "<div id=\"basic\"></div>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "application/javascript": [
              "\n",
              "         requirejs(['basic', 'ecco'], function(basic, ecco){\n",
              "            const viz_id = basic.init()\n",
              "\n",
              "\n",
              "            let pred = new ecco.LayerPredictions({\n",
              "                parentDiv: viz_id,\n",
              "                data:[[{\"token\": \" the\", \"prob\": \"0.38011166\", \"ranking\": 1, \"layer\": 0}, {\"token\": \" first\", \"prob\": \"0.26564986\", \"ranking\": 2, \"layer\": 0}, {\"token\": \" same\", \"prob\": \"0.1181903\", \"ranking\": 3, \"layer\": 0}, {\"token\": \" \\\"\", \"prob\": \"0.10605994\", \"ranking\": 4, \"layer\": 0}, {\"token\": \" right\", \"prob\": \"0.041444484\", \"ranking\": 5, \"layer\": 0}, {\"token\": \" last\", \"prob\": \"0.01262477\", \"ranking\": 6, \"layer\": 0}, {\"token\": \" most\", \"prob\": \"0.011340259\", \"ranking\": 7, \"layer\": 0}, {\"token\": \" a\", \"prob\": \"0.011186912\", \"ranking\": 8, \"layer\": 0}, {\"token\": \" other\", \"prob\": \"0.010309837\", \"ranking\": 9, \"layer\": 0}, {\"token\": \" new\", \"prob\": \"0.007988729\", \"ranking\": 10, \"layer\": 0}], [{\"token\": \" same\", \"prob\": \"0.9150151\", \"ranking\": 1, \"layer\": 1}, {\"token\": \" first\", \"prob\": \"0.07509311\", \"ranking\": 2, \"layer\": 1}, {\"token\": \" second\", \"prob\": \"0.001469628\", \"ranking\": 3, \"layer\": 1}, {\"token\": \" new\", \"prob\": \"0.001143247\", \"ranking\": 4, \"layer\": 1}, {\"token\": \" last\", \"prob\": \"0.0011226581\", \"ranking\": 5, \"layer\": 1}, {\"token\": \" right\", \"prob\": \"0.0010052796\", \"ranking\": 6, \"layer\": 1}, {\"token\": \" following\", \"prob\": \"0.00074669986\", \"ranking\": 7, \"layer\": 1}, {\"token\": \" most\", \"prob\": \"0.0006345953\", \"ranking\": 8, \"layer\": 1}, {\"token\": \" the\", \"prob\": \"0.00060761295\", \"ranking\": 9, \"layer\": 1}, {\"token\": \" very\", \"prob\": \"0.0005755255\", \"ranking\": 10, \"layer\": 1}], [{\"token\": \" same\", \"prob\": \"0.98376435\", \"ranking\": 1, \"layer\": 2}, {\"token\": \" first\", \"prob\": \"0.010908621\", \"ranking\": 2, \"layer\": 2}, {\"token\": \" right\", \"prob\": \"0.0026073102\", \"ranking\": 3, \"layer\": 2}, {\"token\": \" very\", \"prob\": \"0.0009207938\", \"ranking\": 4, \"layer\": 2}, {\"token\": \" second\", \"prob\": \"0.00070992653\", \"ranking\": 5, \"layer\": 2}, {\"token\": \" near\", \"prob\": \"0.00037718014\", \"ranking\": 6, \"layer\": 2}, {\"token\": \" most\", \"prob\": \"0.00016937983\", \"ranking\": 7, \"layer\": 2}, {\"token\": \" way\", \"prob\": \"0.0001388696\", \"ranking\": 8, \"layer\": 2}, {\"token\": \" final\", \"prob\": \"5.557743e-05\", \"ranking\": 9, \"layer\": 2}, {\"token\": \" last\", \"prob\": \"5.159698e-05\", \"ranking\": 10, \"layer\": 2}], [{\"token\": \" same\", \"prob\": \"0.8391301\", \"ranking\": 1, \"layer\": 3}, {\"token\": \" middle\", \"prob\": \"0.121873476\", \"ranking\": 2, \"layer\": 3}, {\"token\": \" first\", \"prob\": \"0.018782783\", \"ranking\": 3, \"layer\": 3}, {\"token\": \" heart\", \"prob\": \"0.011213957\", \"ranking\": 4, \"layer\": 3}, {\"token\": \" near\", \"prob\": \"0.0038564822\", \"ranking\": 5, \"layer\": 3}, {\"token\": \" right\", \"prob\": \"0.003102224\", \"ranking\": 6, \"layer\": 3}, {\"token\": \" midst\", \"prob\": \"0.00089015934\", \"ranking\": 7, \"layer\": 3}, {\"token\": \" very\", \"prob\": \"0.0005122589\", \"ranking\": 8, \"layer\": 3}, {\"token\": \" process\", \"prob\": \"0.0001211585\", \"ranking\": 9, \"layer\": 3}, {\"token\": \" world\", \"prob\": \"0.00011663796\", \"ranking\": 10, \"layer\": 3}], [{\"token\": \" \\\"\", \"prob\": \"0.99696344\", \"ranking\": 1, \"layer\": 4}, {\"token\": \" the\", \"prob\": \"0.002465162\", \"ranking\": 2, \"layer\": 4}, {\"token\": \",\", \"prob\": \"0.00056704076\", \"ranking\": 3, \"layer\": 4}, {\"token\": \" a\", \"prob\": \"1.5210879e-06\", \"ranking\": 4, \"layer\": 4}, {\"token\": \" in\", \"prob\": \"1.2727845e-06\", \"ranking\": 5, \"layer\": 4}, {\"token\": \"\\n\", \"prob\": \"1.0272428e-06\", \"ranking\": 6, \"layer\": 4}, {\"token\": \" first\", \"prob\": \"2.4070596e-07\", \"ranking\": 7, \"layer\": 4}, {\"token\": \"-\", \"prob\": \"1.3172854e-07\", \"ranking\": 8, \"layer\": 4}, {\"token\": \" and\", \"prob\": \"2.4276682e-08\", \"ranking\": 9, \"layer\": 4}, {\"token\": \" to\", \"prob\": \"1.7757136e-08\", \"ranking\": 10, \"layer\": 4}], [{\"token\": \" process\", \"prob\": \"0.21568331\", \"ranking\": 1, \"layer\": 5}, {\"token\": \" midst\", \"prob\": \"0.14830583\", \"ranking\": 2, \"layer\": 5}, {\"token\": \" middle\", \"prob\": \"0.062179733\", \"ranking\": 3, \"layer\": 5}, {\"token\": \" running\", \"prob\": \"0.04196696\", \"ranking\": 4, \"layer\": 5}, {\"token\": \" heart\", \"prob\": \"0.021339823\", \"ranking\": 5, \"layer\": 5}, {\"token\": \" same\", \"prob\": \"0.018006714\", \"ranking\": 6, \"layer\": 5}, {\"token\": \" works\", \"prob\": \"0.014822714\", \"ranking\": 7, \"layer\": 5}, {\"token\": \" hands\", \"prob\": \"0.01314227\", \"ranking\": 8, \"layer\": 5}, {\"token\": \" city\", \"prob\": \"0.012230565\", \"ranking\": 9, \"layer\": 5}, {\"token\": \" centre\", \"prob\": \"0.011854541\", \"ranking\": 10, \"layer\": 5}]]\n",
              "            })\n",
              "            pred.init()\n",
              "         }, function (err) {\n",
              "            console.log(viz_id, err);\n",
              "        })"
            ],
            "text/plain": [
              "<IPython.core.display.Javascript object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    }
  ]
}