Using_🤗_Transformers.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tM38xPIjzXAh"
      },
      "source": [
        "# Behind the pipeline (PyTorch)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "nBllKJihzXAj"
      },
      "source": [
        "Install the Transformers, Datasets, and Evaluate libraries to run this notebook. "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "H6AfHuhSzXAj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "22594984-76e2-49b2-ba8a-bcf6b2593f81"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting datasets\n",
            "  Downloading datasets-2.15.0-py3-none-any.whl (521 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m521.2/521.2 kB\u001b[0m \u001b[31m4.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting evaluate\n",
            "  Downloading evaluate-0.4.1-py3-none-any.whl (84 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m84.1/84.1 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: transformers[sentencepiece] in /usr/local/lib/python3.10/dist-packages (4.35.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
            "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (10.0.1)\n",
            "Collecting pyarrow-hotfix (from datasets)\n",
            "  Downloading pyarrow_hotfix-0.6-py3-none-any.whl (7.9 kB)\n",
            "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
            "  Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m11.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from datasets) (3.4.1)\n",
            "Collecting multiprocess (from datasets)\n",
            "  Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m9.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: fsspec[http]<=2023.10.0,>=2023.1.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.9.1)\n",
            "Requirement already satisfied: huggingface-hub>=0.18.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (0.19.4)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
            "Collecting responses<0.19 (from evaluate)\n",
            "  Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (3.13.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (2023.6.3)\n",
            "Requirement already satisfied: tokenizers<0.19,>=0.14 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.15.0)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (0.4.1)\n",
            "Collecting sentencepiece!=0.1.92,>=0.1.91 (from transformers[sentencepiece])\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m16.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers[sentencepiece]) (3.20.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.4)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.18.0->datasets) (4.5.0)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.3.2)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.6)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2.0.7)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.11.17)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
            "Installing collected packages: sentencepiece, pyarrow-hotfix, dill, responses, multiprocess, datasets, evaluate\n",
            "Successfully installed datasets-2.15.0 dill-0.3.7 evaluate-0.4.1 multiprocess-0.70.15 pyarrow-hotfix-0.6 responses-0.18.0 sentencepiece-0.1.99\n"
          ]
        }
      ],
      "source": [
        "!pip install datasets evaluate transformers[sentencepiece]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "g7g50ZuCzXAj",
        "outputId": "4e192f77-3433-4842-e919-7954b5b137b5",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 234,
          "referenced_widgets": [
            "571e63ec6edb4ddbb71f8a1cabfac887",
            "b24a4aa9d6f94e3ba9ad996d5192cfba",
            "0ced62ff865f4e5397ae9fdc69171efe",
            "477023d24b0c4abd98c26618b750fe9b",
            "24a919b6f4e945f0adbb985c8740aa31",
            "9646f36da355470d9de945e5acf50c40",
            "be40c1d552394dc29394e943c86ce4c2",
            "b73f5737bbd24477aa0dd9701071f52b",
            "857613b1fe644ebc9ed9a89dbc4ec82e",
            "c2189cdb1881446daa064db7d5d25646",
            "07a2ab84a0e648079b42d0caf46d21f2",
            "89d079303c0f47559675abe3b7e6455e",
            "014f434ac08440a9ad7e813377f34c3b",
            "f01baec0dbc04853a1559f8fda3ae2fe",
            "743051740f1948b7a74eac90e13787fe",
            "12b7ec239ac0466d91fddb3b62be9d81",
            "21cff44c890b4447a8822f32f3112420",
            "88169a31c37f4d69b9b7dc02201ea954",
            "2f20117bfab04d1796a59e865c8685e4",
            "5d8dbed5cafa4c9582f5191032b92d23",
            "9afe106aa7f3489ab3060070fb75650b",
            "e8581583a14d42f492d1b49dbe74f000",
            "255b6d535f864ae68f625ffd32e5a1a7",
            "bac5ff3f45584e62965af68326ac5d24",
            "ceb8356c57804fb9b1d1a2ba72728c7e",
            "7320ada6c1364180a771484a071a60c0",
            "3a64c9792f9c4aaf97d3830930934ee3",
            "f16ee8c592684eba8ada32aa5d2128e6",
            "bcd9a622a84446f185c579244aecb7cf",
            "d4651d14129242458e87f5882104a84b",
            "bb3b2c67015047d784edf8f9f58f1131",
            "66cec7b55b5c4997b627b6d3e69556a8",
            "646a39447b7742c0a499855ba9372596",
            "a2f93e9e509e46ac9ff7453788d9444f",
            "32f76ac706ea451eac1782ca030b26e2",
            "c81d667434ff41aa8df709eef1a728a2",
            "a1f63f693b2944c09bbf1718e6432f11",
            "ae765fb243f14841a0cbc6750656c4db",
            "b201d521c34a4e0ba3df6930280a004f",
            "c16bdcd544d846bb8e9eef928c8a9506",
            "d81311218dbb433ab745dd3ad5ff9f9f",
            "3f1a1e84520e4625832df6fc845dc7cf",
            "1d379424dac043bf9ab703a530492720",
            "fc59489776574abdbe27d629c4440331"
          ]
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "No model was supplied, defaulted to distilbert-base-uncased-finetuned-sst-2-english and revision af0f99b (https://huggingface.co/distilbert-base-uncased-finetuned-sst-2-english).\n",
            "Using a pipeline without specifying a model name and revision in production is not recommended.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/629 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "571e63ec6edb4ddbb71f8a1cabfac887"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/268M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "89d079303c0f47559675abe3b7e6455e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/48.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "255b6d535f864ae68f625ffd32e5a1a7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/232k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a2f93e9e509e46ac9ff7453788d9444f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'label': 'POSITIVE', 'score': 0.9598048329353333},\n",
              " {'label': 'NEGATIVE', 'score': 0.9994558691978455}]"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "classifier = pipeline(\"sentiment-analysis\")\n",
        "classifier(\n",
        "    [\n",
        "        \"I've been waiting for a HuggingFace course my whole life.\",\n",
        "        \"I hate this so much!\",\n",
        "    ]\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Preprocessing with a Tokenizer\n",
        "\n",
        "Transformer models, like other neural networks, cannot directly process raw text. Hence, the initial step in our pipeline involves converting text inputs into numerical representations that the model can comprehend. This process is handled by a tokenizer, responsible for:\n",
        "\n",
        "1. Breaking the input into tokens (words, subwords, or symbols like punctuation)\n",
        "2. Mapping each token to an integer\n",
        "3. Adding supplementary inputs beneficial to the model\n",
        "\n",
        "This preprocessing must mirror the same process applied during the model's pretraining. Therefore, we need to retrieve this information from the Model Hub. The `AutoTokenizer` class and its `from_pretrained()` method help in this task. By using the checkpoint name of our model, it automatically fetches and caches the associated data.\n"
      ],
      "metadata": {
        "id": "BBsFLE-p0CSp"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "6AVPNY-FzXAk"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenizing and Preparing Input\n",
        "\n",
        "Once we have the tokenizer, we can directly pass our sentences to it and receive a dictionary ready to feed into our model! The final step is converting the list of input IDs to tensors.\n",
        "\n",
        "While 🤗 Transformers abstracts the underlying ML framework (PyTorch, TensorFlow, or Flax) from the user, Transformer models accept tensors as input. Tensors are akin to NumPy arrays—a scalar (0D), a vector (1D), a matrix (2D), or can have higher dimensions, effectively representing tensors. Other ML framework tensors exhibit similar behavior to NumPy arrays and are usually as simple to create.\n",
        "\n",
        "To specify the type of tensors we wish to obtain (PyTorch, TensorFlow, or plain NumPy), we utilize the `return_tensors` argument:\n"
      ],
      "metadata": {
        "id": "6T26L7nE0RWu"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "wCPtmYxIzXAk",
        "outputId": "8342eba5-b353-49ab-e1e4-f5715bb0fa3f",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'input_ids': tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
            "          2607,  2026,  2878,  2166,  1012,   102],\n",
            "        [  101,  1045,  5223,  2023,  2061,  2172,   999,   102,     0,     0,\n",
            "             0,     0,     0,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],\n",
            "        [1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0]])}\n"
          ]
        }
      ],
      "source": [
        "raw_inputs = [\n",
        "    \"I've been waiting for a HuggingFace course my whole life.\",\n",
        "    \"I hate this so much!\",\n",
        "]\n",
        "inputs = tokenizer(raw_inputs, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "print(inputs)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding the Base Transformer Module\n",
        "\n",
        "This architecture specifically comprises the base Transformer module. When given inputs, it generates what we refer to as hidden states, also recognized as features. Each model input corresponds to a high-dimensional vector obtained from the Transformer model, representing the contextual comprehension of that specific input.\n"
      ],
      "metadata": {
        "id": "gkuV9DQS00st"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "JoaSZeIazXAl"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModel\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModel.from_pretrained(checkpoint)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding High-Dimensional Vectors from the Transformer Module\n",
        "\n",
        "The vector output from the Transformer module tends to be large and encompasses three primary dimensions:\n",
        "\n",
        "1. **Batch size:** Represents the number of sequences processed simultaneously (e.g., 2 in our example).\n",
        "2. **Sequence length:** Denotes the length of the numerical representation of the sequence (e.g., 16 in our example).\n",
        "3. **Hidden size:** Signifies the vector dimension of each model input, contributing to the high-dimensional aspect of the vector.\n",
        "\n",
        "The \"high dimensional\" attribute arises primarily from the hidden size, which can be notably large. For instance, smaller models commonly have a hidden size of 768, while larger models might extend to 3072 or even higher.\n",
        "\n",
        "This structure becomes evident when we input our preprocessed inputs into the model. The outputs from 🤗 Transformers models are akin to namedtuples or dictionaries. Accessing these elements can be achieved through:\n",
        "\n",
        "1. **Attributes:** Elements can be accessed using attributes.\n",
        "   - Example: `outputs.last_hidden_state`\n",
        "\n",
        "2. **Keys:** Access elements using keys as in dictionaries.\n",
        "   - Example: `outputs[\"last_hidden_state\"]`\n",
        "\n",
        "3. **Indices:** If aware of the specific position of the required element, it can be accessed using index.\n",
        "   - Example: `outputs[0]`\n"
      ],
      "metadata": {
        "id": "EE49qwsy09eY"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "id": "ntaSAaZRzXAl",
        "outputId": "1555b6b2-3b2e-43fa-ae98-8439f6fa006a",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([2, 16, 768])\n"
          ]
        }
      ],
      "source": [
        "outputs = model(**inputs)\n",
        "print(outputs.last_hidden_state.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding Model Heads: Transforming Hidden States\n",
        "\n",
        "Model heads receive the high-dimensional vector of hidden states as input and project them onto a different dimension. Typically, these heads consist of one or a few linear layers, and they process the output of the Transformer model directly.\n",
        "\n",
        "The embeddings layer converts each tokenized input ID into a corresponding vector, while subsequent layers manipulate these vectors using the attention mechanism to generate the final sentence representations.\n",
        "\n",
        "🤗 Transformers offers various architectures tailored for specific tasks, including but not limited to:\n",
        "\n",
        "- `Model` (retrieves hidden states)\n",
        "- `ForCausalLM`\n",
        "- `ForMaskedLM`\n",
        "- `ForMultipleChoice`\n",
        "- `ForQuestionAnswering`\n",
        "- `ForSequenceClassification`\n",
        "- `ForTokenClassification`\n",
        "- and others 🤗\n",
        "\n",
        "For our example, we require a model equipped with a sequence classification head (for classifying sentences as positive or negative). Thus, we'll utilize not the `AutoModel` class, but `AutoModelForSequenceClassification`.\n"
      ],
      "metadata": {
        "id": "C4I0hBi11t-_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "id": "WTcvE3eQzXAl"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoModelForSequenceClassification\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "outputs = model(**inputs)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "When examining the shape of our outputs, we'll notice a significant reduction in dimensionality. The model head, operating on the previously mentioned high-dimensional vectors, outputs vectors containing only two values, representing each label. As we have two sentences and two labels, the resulting shape from our model is 2 x 2.\n"
      ],
      "metadata": {
        "id": "-qoTcfdq2BT1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "s4P1W7R1zXAl",
        "outputId": "9462f073-0cc2-40c0-c1ab-3e9d4abee9a8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([2, 2])\n"
          ]
        }
      ],
      "source": [
        "print(outputs.logits.shape)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Postprocessing Model Output\n",
        "\n",
        "The values obtained from our model are not probabilities but logits, representing raw, unnormalized scores from the last layer. To convert these logits into probabilities, they must pass through a SoftMax layer. All 🤗 Transformers models output logits as the loss function for training typically integrates the last activation function (like SoftMax) with the actual loss function (such as cross-entropy)."
      ],
      "metadata": {
        "id": "KGRnw7IO2WDv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "id": "Ff5UR0Z0zXAm",
        "outputId": "8bd47ad6-b95e-4eb4-f626-03c51c143029",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[-1.5607,  1.6123],\n",
            "        [ 4.1692, -3.3464]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ],
      "source": [
        "print(outputs.logits)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "id": "GeTrsJbQzXAm",
        "outputId": "fce334b3-4fc8-4364-f26e-f5584250e2a5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[4.0195e-02, 9.5980e-01],\n",
            "        [9.9946e-01, 5.4418e-04]], grad_fn=<SoftmaxBackward0>)\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "\n",
        "predictions = torch.nn.functional.softmax(outputs.logits, dim=-1)\n",
        "print(predictions)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "- Model prediction before SoftMax:\n",
        "  - First sentence: [-1.5607, 1.6123]\n",
        "  - Second sentence: [4.1692, -3.3464]\n",
        "\n",
        "After applying SoftMax, the model predicted:\n",
        "- First sentence: [0.0402, 0.9598]\n",
        "- Second sentence: [0.9995, 0.0005]\n",
        "  - These scores represent recognizable probability distributions.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "JSzuQWIR2bcV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "id": "-KN-fr2yzXAm",
        "outputId": "3746b0ca-2108-4905-9c05-374725fa7723",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{0: 'NEGATIVE', 1: 'POSITIVE'}"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "model.config.id2label"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "To determine the labels for each position, we can inspect the `id2label` attribute of the model config (detailed in the next section). Conclusively, the model predicted:\n",
        "\n",
        "- First sentence: NEGATIVE: 0.0402, POSITIVE: 0.9598\n",
        "- Second sentence: NEGATIVE: 0.9995, POSITIVE: 0.0005\n",
        "\n",
        "This reproduces the three primary steps of the pipeline: preprocessing using tokenizers, model input processing, and postprocessing. Let's delve deeper into each of these steps."
      ],
      "metadata": {
        "id": "DvQQt9ySYwFy"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "j6imyaV92f0J"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Models (PyTorch)"
      ],
      "metadata": {
        "id": "qk72awdxY38a"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding Model Creation and Usage\n",
        "\n",
        "To initialize a BERT model, the initial step is loading a configuration object, which includes various attributes used in building the model:\n",
        "\n",
        "- `hidden_size`: Defines the size of the hidden_states vector.\n",
        "- `num_hidden_layers`: Specifies the number of layers within the Transformer model.\n",
        "\n",
        "Different loading methods are available for initializing Transformer models. Creating a model from the default configuration initializes it with random values, requiring training before meaningful inference.\n",
        "\n"
      ],
      "metadata": {
        "id": "8zhK8t35ZlEm"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertConfig, BertModel\n",
        "\n",
        "# Building the config\n",
        "config = BertConfig()\n",
        "\n",
        "# Building the model from the config\n",
        "model = BertModel(config)"
      ],
      "metadata": {
        "id": "dzxhIYbbY58R"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "print(config)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "p5Jte2KWY7hW",
        "outputId": "52cad162-3ad2-4dcc-ad62-b28edc105f83"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "BertConfig {\n",
            "  \"attention_probs_dropout_prob\": 0.1,\n",
            "  \"classifier_dropout\": null,\n",
            "  \"hidden_act\": \"gelu\",\n",
            "  \"hidden_dropout_prob\": 0.1,\n",
            "  \"hidden_size\": 768,\n",
            "  \"initializer_range\": 0.02,\n",
            "  \"intermediate_size\": 3072,\n",
            "  \"layer_norm_eps\": 1e-12,\n",
            "  \"max_position_embeddings\": 512,\n",
            "  \"model_type\": \"bert\",\n",
            "  \"num_attention_heads\": 12,\n",
            "  \"num_hidden_layers\": 12,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"position_embedding_type\": \"absolute\",\n",
            "  \"transformers_version\": \"4.35.2\",\n",
            "  \"type_vocab_size\": 2,\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 30522\n",
            "}\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertConfig, BertModel\n",
        "\n",
        "config = BertConfig()\n",
        "model = BertModel(config)\n",
        "\n",
        "# Model is randomly initialized!"
      ],
      "metadata": {
        "id": "nAOq6WHQY-5v"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Loading a pre-trained Transformer model is achieved through the `from_pretrained()` method, simplifying the process of sharing and reusing trained models. Utilizing the `AutoModel` class ensures checkpoint-agnostic code, facilitating compatibility across different architectures trained for similar tasks.\n",
        "\n",
        "The `bert-base-cased` identifier, for instance, represents a BERT checkpoint trained by the authors of BERT, initializing the model with pre-existing weights for inference or fine-tuning on new tasks.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "XgU0dc0wZqvx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertModel\n",
        "\n",
        "model = BertModel.from_pretrained(\"bert-base-cased\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "ca25cf3d11f944babdd0b9a961c1592a",
            "0ea5db5350874ec089adc09229ae5937",
            "410a2d13a72b4e1e8da604a32ab1ff57",
            "bb40880c682047109cdbeb361ab3178f",
            "c9a8c9bb8e0b4c2fbf0e9d72f654483f",
            "e9b09849a6c14ad4840d2f45afc6e907",
            "0c0df20e42484a3fafd9e1953a509862",
            "574903aadd444978b8d5f5e01f3ab659",
            "6f7969b2b07b463b9f23071933088cfe",
            "f249d1da6e1e471ea267195bb14fb09b",
            "79e6dd3760cb43958497c258e7b99ead",
            "87a14b99ba9d40c6a69ee7fc0e8ed95e",
            "79be3a4cee6d45368d49ed5a07be1153",
            "683b2a52f93d4ed3816a03f725c64f55",
            "6dc2cc649d4c421aaf293279e85729f9",
            "47825908a90248538b510444895cfe13",
            "8d1dc7988e04429c8d60113c038362b1",
            "6652dabd747b4cdabfde969f614d7420",
            "99013e28d4b94b64a3d917b0e69d0f82",
            "ccecaaa0fdb040369a749926cb947e11",
            "ccc3976ab50240f3b70caf611bfc665c",
            "f37c5d1c783442bbabb7c156992c5165"
          ]
        },
        "id": "VSoFPbYFZBE3",
        "outputId": "9f3ea272-1e27-415e-8115-5c06c19aa2f4"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "config.json:   0%|          | 0.00/570 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ca25cf3d11f944babdd0b9a961c1592a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "model.safetensors:   0%|          | 0.00/436M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "87a14b99ba9d40c6a69ee7fc0e8ed95e"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Saving a model can be done using the `save_pretrained()` method, which generates two files: `config.json` (detailing model architecture and metadata) and `pytorch_model.bin` (containing model weights).\n",
        "\n"
      ],
      "metadata": {
        "id": "Gn3HGHn-Zya7"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model.save_pretrained(\"directory_on_my_computer\")"
      ],
      "metadata": {
        "id": "5FtWPfS-ZtgK"
      },
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Using a Transformer model for inference\n",
        "Using a Transformer model for inference involves tokenizing inputs into vocabulary indices, termed as input IDs, before processing them through the model. Tokenizers handle input conversion to the appropriate tensors, crucial for the model's understanding.\n",
        "\n",
        "For instance, sequences are converted to input IDs by tokenizers, resulting in lists of encoded sequences, which can be converted to tensors, enabling compatibility with the model's input requirements.\n",
        "\n"
      ],
      "metadata": {
        "id": "3dtUo8OeZ4wf"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "sequences = [\"Hello!\", \"Cool.\", \"Nice!\"]"
      ],
      "metadata": {
        "id": "506VF5NHZ0rp"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_sequences = [\n",
        "    [101, 7592, 999, 102],\n",
        "    [101, 4658, 1012, 102],\n",
        "    [101, 3835, 999, 102],\n",
        "]"
      ],
      "metadata": {
        "id": "5s-0WsPDZ9s8"
      },
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "\n",
        "model_inputs = torch.tensor(encoded_sequences)"
      ],
      "metadata": {
        "id": "C1lG28mcZ-pQ"
      },
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "### Using the tensors as inputs to the model\n",
        "The process of using tensors as inputs to the model is straightforward — call the model with the inputs, typically passing the input IDs. While the model accepts various arguments, only the input IDs are necessary for inference. Understanding the functionalities of additional arguments and their necessity is explained later in this section, following an exploration of tokenizers that facilitate Transformer model inputs."
      ],
      "metadata": {
        "id": "_HVyp8zzaIoy"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "output = model(model_inputs)"
      ],
      "metadata": {
        "id": "jmsPlslVZ_ob"
      },
      "execution_count": 22,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "0j969vLJaCTN"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Tokenizers (PyTorch)"
      ],
      "metadata": {
        "id": "poAMM7aQaZp0"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding Tokenization in NLP\n",
        "\n",
        "Tokenizers play a crucial role in the NLP pipeline by translating text into data that can be processed by models. Since models exclusively process numerical data, tokenizers are pivotal in converting text inputs into numerical formats. In typical NLP tasks, the data processed is raw text.\n",
        "\n",
        "However, as models solely handle numbers, text-to-number conversion becomes imperative. Tokenizers step in to achieve this conversion, employing various algorithms to transform raw text into numerical representations. The primary aim is to identify the most meaningful and compact representation that aligns with the model's comprehension.\n",
        "\n"
      ],
      "metadata": {
        "id": "OOwdO2XCa1GJ"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Word-based Tokenization\n",
        "\n",
        "In NLP, word-based tokenization is a straightforward approach, usually simple to implement and effective in yielding reasonable results. The objective is to segment raw text into words and assign a numerical representation to each word. Here's a basic breakdown:\n",
        "\n",
        "### Tokenization Process:\n",
        "- **Using Whitespace:** We can tokenize text into words by applying Python's `split()` function, using whitespace as a delimiter.\n",
        "  \n",
        "### Vocabulary and Word IDs:\n",
        "- Each word is assigned an ID, typically starting from 0 and spanning the size of the vocabulary. These IDs are utilized by the model to differentiate words.\n",
        "- **Vocabulary Size:** Covering an entire language with a word-based tokenizer demands a vast number of unique tokens. For instance, English has over 500,000 words, necessitating the management of a substantial number of IDs.\n",
        "- **Word Variations:** Words like \"dog\" and \"dogs\" might be represented differently, leading the model to initially treat them as distinct. Similar variations such as \"run\" and \"running\" also pose similar challenges initially.\n",
        "  \n",
        "### Handling Unknown Tokens:\n",
        "- **Unknown Token ([UNK]):** Words not in the vocabulary are represented using a custom token like \"[UNK]\" or \"\". An abundance of these tokens during tokenization suggests the tokenizer struggles to represent certain words adequately, resulting in information loss.\n",
        "- **Optimizing Vocabulary:** The goal is to construct the vocabulary in a manner that minimizes the usage of the unknown token, thereby retaining as much information as possible.\n",
        "\n"
      ],
      "metadata": {
        "id": "xbZY2KnQa_pc"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_text = \"Jim Henson was a puppeteer\".split()\n",
        "print(tokenized_text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rqYZmra5aaNH",
        "outputId": "994d69c2-4aa4-4d87-ae0f-6a0cbb31a4e5"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Jim', 'Henson', 'was', 'a', 'puppeteer']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Character-based Tokenization\n",
        "\n",
        "Character-based tokenizers segment text into individual characters rather than words. This method presents several advantages:\n",
        "\n",
        "### Advantages:\n",
        "1. **Smaller Vocabulary:** The vocabulary size is significantly reduced.\n",
        "2. **Minimized Unknown Tokens:** As every word can be constructed from characters, there are fewer out-of-vocabulary tokens.\n",
        "\n",
        "However, some considerations arise regarding spaces and punctuation:\n",
        "\n",
        "### Challenges:\n",
        "- **Representation:** Character-based tokenization may seem less meaningful as each character lacks the context and semantic value that words possess. However, this varies across languages; for instance, Chinese characters often carry more inherent meaning.\n",
        "- **Token Count:** The number of tokens increases considerably compared to word-based tokenization. A single word could result in multiple tokens when represented by characters.\n",
        "\n",
        "### Balancing Approaches: Subword Tokenization\n",
        "To address these limitations and capitalize on the benefits of both word-based and character-based approaches, a hybrid technique called subword tokenization comes into play.\n"
      ],
      "metadata": {
        "id": "blRqqJ7sbTTV"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Subword Tokenization\n",
        "\n",
        "Subword tokenization algorithms function based on the principle that commonly used words should remain intact while infrequently used words are divided into meaningful subwords.\n",
        "\n",
        "For example, \"annoyingly\" might be considered rare and split into \"annoying\" and \"ly\", as these subwords appear more frequently independently. Yet, the composite meaning of \"annoyingly\" remains through the combination of \"annoying\" and \"ly\".\n",
        "\n",
        "Subwords contribute rich semantic meaning; for instance, \"tokenization\" is segmented into \"token\" and \"ization\", providing meaningful representation with a reduced number of tokens. This allows for comprehensive coverage with smaller vocabularies and minimizes unknown tokens.\n",
        "\n",
        "Subword tokenization is particularly beneficial in agglutinative languages like Turkish, where complex words can be formed by stringing together subwords almost limitlessly.\n",
        "\n",
        "### Additional Techniques\n",
        "Other notable techniques include:\n",
        "\n",
        "- **Byte-level BPE:** Employed in models like GPT-2\n",
        "- **WordPiece:** Utilized in BERT\n",
        "- **SentencePiece or Unigram:** Found in various multilingual models\n",
        "\n",
        "This diverse array of techniques provides a comprehensive understanding of tokenization methods, enabling us to effectively leverage the API.\n"
      ],
      "metadata": {
        "id": "R2vCJnbhbn42"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Loading and Saving Tokenizers\n",
        "\n",
        "The process of loading and saving tokenizers mirrors that of models. In essence, it's reliant on two familiar methods: `from_pretrained()` and `save_pretrained()`. These methods handle loading or saving the tokenizer's underlying algorithm (similar to the model's architecture) along with its vocabulary (akin to the model's weights).\n",
        "\n",
        "For instance, loading the BERT tokenizer, which is trained with the same checkpoint as BERT, follows a similar process to loading the model. However, this time we employ the `BertTokenizer` class. The `from_pretrained()` method enables the instantiation of the tokenizer using the specified pretrained tokenizer checkpoint, such as 'bert-base-uncased'. This straightforward approach allows seamless loading of tokenizers pre-trained with specific models.\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "MaTsKUwXb4ln"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import BertTokenizer\n",
        "\n",
        "tokenizer = BertTokenizer.from_pretrained(\"bert-base-cased\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "3212ee6425f34dcea759e488d1072e41",
            "1e538fdc010f4ae6b35d027bffd6f7db",
            "c6fff925af90400eb6f88cbccbf999ec",
            "9439a66e3fd04214ba3b6aad91c1de60",
            "9f958874a2d54602b83989162b666f3a",
            "8fcc2079e09345349cb04813a4e57025",
            "59b92687aaa8428fa7454bc0540126ce",
            "62664275c5e34a22b2ad5dfc0ae705b6",
            "3ce03d5d40914b84bceddbf98843df38",
            "b70a68e58b0c45949bbcc818c5dba425",
            "f323302316a04536bd131d691024d0aa",
            "931e29c8e4eb412ea4435631d7f6ecd7",
            "168ebc56a4064e0d9d0f75f0e91f7466",
            "b52758fea3384ebaa8b4ec5cec8246ce",
            "a05c1cc497774c7795edd9e639ffc371",
            "2323b8e4a5df4a09b0ac6f4571eb6edf",
            "6e9d75846f5c49819a9e95a60578741b",
            "a9173759a6fc40a4a96b62083c103818",
            "427106c41dde4dfb8be90ad0c333da2c",
            "25d3dff6ed1947b8acf1e2e4e487e2ec",
            "3e1ffb81f98b4405b069c9e0ddc3e6af",
            "46b02352b66842eca83fed3e7c8a06fc",
            "f05ba1bc80944ab387dbe2fb3fd0bbc6",
            "cfb0ae744f5e460ebdbe00438ef10547",
            "609df9531b604174a07d964dca34da7f",
            "1225fceccd8b4e81a7b40f6983cb02bc",
            "c2384fa4cac04630832cc2fd2011b98c",
            "2311567c05d74c52ae4deb358b9fcc56",
            "fe899c4870c4471abccf39e5088c711d",
            "51a1d9ac155d49f594b0ff6e53d81e89",
            "3a0f124e233d485db89851819ed9e3dd",
            "2226fade94a445c5b8ee2e871d51bcc4",
            "64ed462019d645b285cb27c4fb50ad56"
          ]
        },
        "id": "-n11cebhacB4",
        "outputId": "073b5421-6d42-4e00-8331-daf7bcbe18bf"
      },
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer_config.json:   0%|          | 0.00/29.0 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "3212ee6425f34dcea759e488d1072e41"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "931e29c8e4eb412ea4435631d7f6ecd7"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "tokenizer.json:   0%|          | 0.00/436k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f05ba1bc80944ab387dbe2fb3fd0bbc6"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## AutoTokenizer for Efficient Tokenization\n",
        "\n",
        "Similar to `AutoModel`, the `AutoTokenizer` class efficiently selects the appropriate tokenizer class from the library based on the checkpoint name. It seamlessly pairs with any checkpoint:\n"
      ],
      "metadata": {
        "id": "An80FwsdcPEr"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")"
      ],
      "metadata": {
        "id": "pKn-DxcUad1v"
      },
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer(\"Using a Transformer network is simple\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2CHtV7yoafG8",
        "outputId": "87e1ab6a-4135-49b5-cd96-44b4ca1d920f"
      },
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [101, 7993, 170, 13809, 23763, 2443, 1110, 3014, 102], 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {},
          "execution_count": 27
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The AutoTokenizer offers a flexible approach to work with various checkpoints directly. Saving a tokenizer follows the same pattern as saving a model:"
      ],
      "metadata": {
        "id": "IIO8KOxBcXHx"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.save_pretrained(\"directory_on_my_computer\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UQF6-hDdagYT",
        "outputId": "0a1bc80b-87bf-4c41-8355-9f8e7e724514"
      },
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('directory_on_my_computer/tokenizer_config.json',\n",
              " 'directory_on_my_computer/special_tokens_map.json',\n",
              " 'directory_on_my_computer/vocab.txt',\n",
              " 'directory_on_my_computer/added_tokens.json',\n",
              " 'directory_on_my_computer/tokenizer.json')"
            ]
          },
          "metadata": {},
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Encoding\n",
        "\n",
        "Translating text into numerical representations involves a two-step process: tokenization followed by conversion to input IDs.\n",
        "\n",
        "Tokenization breaks text into tokens—words, parts of words, or punctuation symbols—governed by various rules. It is essential to instantiate the tokenizer using the model's name to ensure consistency with the rules used during the model's pretraining.\n",
        "\n",
        "The second step involves converting these tokens into numbers, allowing the creation of a tensor to be fed into the model. This process relies on the tokenizer's vocabulary, which is obtained during instantiation with the `from_pretrained()` method. Consistency in using the same vocabulary as the one used during model pretraining is crucial.\n",
        "\n",
        "Note that for practical purposes, we should directly invoke the tokenizer on your inputs, as demonstrated earlier.\n"
      ],
      "metadata": {
        "id": "dIFDmQfBch-g"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "Tokenization is performed using the `tokenize()` method of the tokenizer. The output from this method is a list of strings or tokens."
      ],
      "metadata": {
        "id": "mSvMSWxHcvjO"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"bert-base-cased\")\n",
        "\n",
        "sequence = \"Using a Transformer network is simple\"\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "\n",
        "print(tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ht7tiylzahsc",
        "outputId": "84eda9ad-1cde-4585-bc85-f6852a91f278"
      },
      "execution_count": 29,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['Using', 'a', 'Trans', '##former', 'network', 'is', 'simple']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This specific tokenizer is a subword tokenizer, meaning it splits words until it generates tokens present in its vocabulary. For example, \"transformer\" is divided into two tokens: \"transform\" and \"##er\".\n",
        "\n",
        "**From Tokens to Input IDs**\n",
        "\n",
        "The conversion to input IDs is handled by the convert_tokens_to_ids() tokenizer method. Once these outputs are converted to the appropriate framework tensor, they can be utilized as inputs for a model, as demonstrated earlier."
      ],
      "metadata": {
        "id": "2_R1oDc8c4Ar"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "print(ids)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "COqeqcojaimw",
        "outputId": "d5970b66-d950-43d1-c51c-806d47ae239d"
      },
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[7993, 170, 13809, 23763, 2443, 1110, 3014]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Decoding\n",
        "Decoding involves the reverse operation: starting from vocabulary indices, we aim to obtain a string. This can be achieved using the `decode()` method.\n",
        "\n",
        "The decode method not only converts the indices back to tokens but also groups together tokens that were part of the same words to generate a readable sentence. This behavior becomes particularly valuable when working with models that predict new text, be it generated text from a prompt or for sequence-to-sequence tasks like translation or summarization.\n"
      ],
      "metadata": {
        "id": "hPOKVUgSdD-T"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "decoded_string = tokenizer.decode([7993, 170, 11303, 1200, 2443, 1110, 3014])\n",
        "print(decoded_string)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "uZ8VtZVkaj3Y",
        "outputId": "5d35b565-271e-4894-f4dd-40c163b77366"
      },
      "execution_count": 32,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Using a transformer network is simple\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "UmOrgiqxdlf0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EKsCLurFdeiD"
      },
      "source": [
        "# Handling multiple sequences (PyTorch)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Our earlier exploration delved into straightforward inference with a single sequence of modest length. However, this led to a cascade of questions:\n",
        "\n",
        "- How do we manage multiple sequences?\n",
        "- What about sequences of varying lengths?\n",
        "- Are vocabulary indices the exclusive inputs that yield optimal model performance?\n",
        "- Is there a limit to the sequence length that a model can handle effectively?\n",
        "\n",
        "Let's delve into these inquiries and uncover solutions utilizing the 🤗 Transformers API.\n",
        "\n",
        "### Batch Inputs for Models\n",
        "\n",
        "We witnessed the transformation of sequences into numerical lists in the previous exercise. Now, let's take these lists of numbers, convert them into a tensor, and feed them into the model:\n"
      ],
      "metadata": {
        "id": "R9jbWgJveJAD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "yhuN21dBdeiG",
        "outputId": "aa60bdb2-3498-41b6-b7e4-e25768a8d635",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 357
        }
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "IndexError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mIndexError\u001b[0m                                Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-35-2b9fc1b6cd40>\u001b[0m in \u001b[0;36m<cell line: 14>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0minput_ids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m \u001b[0;31m# This line will fail.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 14\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1517\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1518\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1520\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1525\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1526\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1529\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/distilbert/modeling_distilbert.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, head_mask, inputs_embeds, labels, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    777\u001b[0m         \u001b[0mreturn_dict\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0mreturn_dict\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m \u001b[0;32melse\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muse_return_dict\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    778\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 779\u001b[0;31m         distilbert_output = self.distilbert(\n\u001b[0m\u001b[1;32m    780\u001b[0m             \u001b[0minput_ids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    781\u001b[0m             \u001b[0mattention_mask\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattention_mask\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1516\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_compiled_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[misc]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1517\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1518\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1519\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1520\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_call_impl\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1525\u001b[0m                 \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_pre_hooks\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0m_global_backward_hooks\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1526\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1527\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1528\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1529\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/models/distilbert/modeling_distilbert.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, input_ids, attention_mask, head_mask, inputs_embeds, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    580\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"You cannot specify both input_ids and inputs_embeds at the same time\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    581\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0minput_ids\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 582\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwarn_if_padding_and_no_attention_mask\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0minput_ids\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mattention_mask\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    583\u001b[0m             \u001b[0minput_shape\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    584\u001b[0m         \u001b[0;32melif\u001b[0m \u001b[0minputs_embeds\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.10/dist-packages/transformers/modeling_utils.py\u001b[0m in \u001b[0;36mwarn_if_padding_and_no_attention_mask\u001b[0;34m(self, input_ids, attention_mask)\u001b[0m\n\u001b[1;32m   4114\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   4115\u001b[0m         \u001b[0;31m# Check only the first and last input IDs to reduce overhead.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 4116\u001b[0;31m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpad_token_id\u001b[0m \u001b[0;32min\u001b[0m \u001b[0minput_ids\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   4117\u001b[0m             warn_string = (\n\u001b[1;32m   4118\u001b[0m                 \u001b[0;34m\"We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See \"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mIndexError\u001b[0m: too many indices for tensor of dimension 1"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "\n",
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "input_ids = torch.tensor(ids)\n",
        "# This line will fail.\n",
        "model(input_ids)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Unforeseen Obstacle: Failure in Process\n",
        "\n",
        "Oops! What went wrong here? It seems we encountered a glitch while following the procedures outlined in section 2 of the pipeline.\n",
        "\n",
        "The issue stems from our attempt to pass a solitary sequence to the model, while 🤗 Transformers models inherently anticipate multiple sentences. We attempted to replicate the tokenization process the tokenizer autonomously undertakes when applied to a sequence. But upon closer inspection, we notice that the tokenizer didn't merely convert the input IDs list into a tensor. It appended an additional dimension to the data."
      ],
      "metadata": {
        "id": "IutloHkIeb4y"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {
        "id": "m9pLyjYYdeiG",
        "outputId": "16eba349-948d-443b-aeed-5ce5dafd1ad4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[  101,  1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,\n",
            "          2607,  2026,  2878,  2166,  1012,   102]])\n"
          ]
        }
      ],
      "source": [
        "tokenized_inputs = tokenizer(sequence, return_tensors=\"pt\")\n",
        "print(tokenized_inputs[\"input_ids\"])"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Second Attempt: Introducing a New Dimension\n",
        "\n",
        "Let's give this another shot and introduce an additional dimension to the data:"
      ],
      "metadata": {
        "id": "qM-sfLzsez1f"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 37,
      "metadata": {
        "id": "lJzzLqyTdeiH",
        "outputId": "4c19e410-de02-482d-ffda-e54771c236f8",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Input IDs: tensor([[ 1045,  1005,  2310,  2042,  3403,  2005,  1037, 17662, 12172,  2607,\n",
            "          2026,  2878,  2166,  1012]])\n",
            "Logits: tensor([[-2.7276,  2.8789]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "\n",
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "\n",
        "input_ids = torch.tensor([ids])\n",
        "print(\"Input IDs:\", input_ids)\n",
        "\n",
        "output = model(input_ids)\n",
        "print(\"Logits:\", output.logits)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Batching involves the process of concurrently sending multiple sentences through the model. When we have just one sentence, we can effortlessly construct a batch containing a single sequence:"
      ],
      "metadata": {
        "id": "azPZUud8e4yF"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "ndYR7keGdeiH"
      },
      "outputs": [],
      "source": [
        "batched_ids = [\n",
        "    [200, 200, 200],\n",
        "    [200, 200]\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "By batching, the model becomes equipped to handle multiple sentences fed into it. Incorporating multiple sequences is as straightforward as forming a batch with a single sequence. However, a subsequent concern arises. When attempting to batch together two or more sentences, the lengths might differ. Given tensors necessitate a rectangular shape, we can't directly convert the input IDs list into a tensor. To circumvent this obstacle, the typical approach involves padding the inputs.\n"
      ],
      "metadata": {
        "id": "LVMTw__AfA1W"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Applying Padding to Inputs\n",
        "\n",
        "The tensor representation formed by a list of lists can't be directly converted. To address this issue, padding is used to ensure that tensors possess a rectangular shape. Padding aims to standardize the length of all sentences by introducing a special token, the padding token, to sentences with fewer tokens. For instance, if there are 10 sentences with 10 words each and 1 sentence with 20 words, padding ensures uniformity by expanding all sentences to 20 words."
      ],
      "metadata": {
        "id": "bO6DxeWTfYRV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "axJumIlydeiH"
      },
      "outputs": [],
      "source": [
        "padding_id = 100\n",
        "\n",
        "batched_ids = [\n",
        "    [200, 200, 200],\n",
        "    [200, 200, padding_id],\n",
        "]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The ID for the padding token can be accessed using `tokenizer.pad_token_id`. Let's use this token and pass both individual sentences and a batched sequence through the model.\n"
      ],
      "metadata": {
        "id": "nH8p0EMpfmnD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "id": "3F9me3sFdeiI",
        "outputId": "44892b4e-c585-424b-d200-aa44e1f627d1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "We strongly recommend passing in an `attention_mask` since your input_ids may be padded. See https://huggingface.co/docs/transformers/troubleshooting#incorrect-output-when-padding-tokens-arent-masked.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 1.5694, -1.3895]], grad_fn=<AddmmBackward0>)\n",
            "tensor([[ 0.5803, -0.4125]], grad_fn=<AddmmBackward0>)\n",
            "tensor([[ 1.5694, -1.3895],\n",
            "        [ 1.3374, -1.2163]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ],
      "source": [
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "\n",
        "sequence1_ids = [[200, 200, 200]]\n",
        "sequence2_ids = [[200, 200]]\n",
        "batched_ids = [\n",
        "    [200, 200, 200],\n",
        "    [200, 200, tokenizer.pad_token_id],\n",
        "]\n",
        "\n",
        "print(model(torch.tensor(sequence1_ids)).logits)\n",
        "print(model(torch.tensor(sequence2_ids)).logits)\n",
        "print(model(torch.tensor(batched_ids)).logits)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "However, an issue arises with the logits in our batched predictions: the second row doesn't match the logits for the second sentence; instead, we have entirely different values!\n",
        "\n",
        "This discrepancy emerges because Transformer models utilize attention layers that contextualize each token. These layers consider padding tokens, as they attend to every token within a sequence. To ensure consistent results between passing individual sentences of varying lengths through the model and using a padded batch with the same sentences, we need to instruct these attention layers to disregard the padding tokens. This is accomplished using an attention mask.\n"
      ],
      "metadata": {
        "id": "y74Ctp7uftvs"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Understanding Attention Masks\n",
        "\n",
        "Attention masks are tensors shaped identically to the input IDs tensor, containing 0s and 1s. In this context, 1s denote tokens to be attended to, while 0s indicate tokens that should be ignored by the model's attention layers.\n",
        "\n",
        "To illustrate, let's enhance the prior example by including an attention mask:"
      ],
      "metadata": {
        "id": "BNtg5dxFfyIU"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "id": "zjVy-eVHdeiI",
        "outputId": "ee9d3f8f-1488-4aaf-d1e0-29d4b5dba0f1",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([[ 1.5694, -1.3895],\n",
            "        [ 0.5803, -0.4125]], grad_fn=<AddmmBackward0>)\n"
          ]
        }
      ],
      "source": [
        "batched_ids = [\n",
        "    [200, 200, 200],\n",
        "    [200, 200, tokenizer.pad_token_id],\n",
        "]\n",
        "\n",
        "attention_mask = [\n",
        "    [1, 1, 1],\n",
        "    [1, 1, 0],\n",
        "]\n",
        "\n",
        "outputs = model(torch.tensor(batched_ids), attention_mask=torch.tensor(attention_mask))\n",
        "print(outputs.logits)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "By incorporating the attention mask, we now observe consistent logits for the second sentence within the batch.\n",
        "\n",
        "It's important to note that the last value of the second sequence aligns with a padding ID, represented as a 0 value in the attention mask."
      ],
      "metadata": {
        "id": "NMIUYHiVf11P"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Dealing with Lengthy Sequences\n",
        "\n",
        "Transformer models typically come with restrictions on the lengths of sequences they can process. These limits usually range from 512 to 1024 tokens. Exceeding these lengths might cause the models to crash. To tackle this issue, consider two strategies:\n",
        "\n",
        "1. **Opt for a Model Supporting Longer Sequences:**\n",
        "   Certain models specialize in handling longer sequences, such as Longformer or LED. They are designed to manage extended inputs beyond the standard limitations. If your task necessitates processing very long sequences, exploring these models might be beneficial.\n",
        "\n",
        "2. **Sequence Truncation:**\n",
        "   An alternative approach involves truncating sequences by setting the `max_sequence_length` parameter:\n",
        "\n"
      ],
      "metadata": {
        "id": "8RxEZAKDf7xN"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "84c19-HrdeiI"
      },
      "outputs": [],
      "source": [
        "max_sequence_length = 300\n",
        "sequence = sequence[:max_sequence_length]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "By employing this parameter, we can control the maximum sequence length, ensuring it adheres to the model's accepted input size."
      ],
      "metadata": {
        "id": "llHCs-BxgKJg"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "ibGyfRRUak9h"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "810xRBrGgSdM"
      },
      "source": [
        "# Putting it all together (PyTorch)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Leveraging High-Level Functions in 🤗 Transformers API\n",
        "\n",
        "In the preceding sections, we delved into the intricacies of tokenizers, covering tokenization, input ID conversion, padding, truncation, and attention masks manually.\n",
        "\n",
        "Nevertheless, as showcased in Section 2, the 🤗 Transformers API streamlines these processes with a high-level function. When we employ our tokenizer directly on a sentence, the outputs are primed for direct transmission to our model."
      ],
      "metadata": {
        "id": "gAuWVm-sgnrc"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "id": "5scA0PiVgSdP"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "\n",
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "\n",
        "model_inputs = tokenizer(sequence)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "This high-level functionality abstracts away the complexities we previously managed manually, enabling a more straightforward and efficient workflow."
      ],
      "metadata": {
        "id": "BKwiqBgvgujK"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Tokenizer's Power in Preparing Model Inputs\n",
        "\n",
        "The `model_inputs` variable aggregates all essential elements necessary for a model's optimal functionality. For instance, in DistilBERT, this includes input IDs along with the attention mask. Other models accommodating additional inputs will have these components provided by the tokenizer object.\n",
        "\n",
        "Demonstrating its versatility, this method efficiently tokenizes a single sequence:"
      ],
      "metadata": {
        "id": "iWMi_nbpg6rv"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 48,
      "metadata": {
        "id": "JlNGADS5gSdP"
      },
      "outputs": [],
      "source": [
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "\n",
        "model_inputs = tokenizer(sequence)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Moreover, it seamlessly manages multiple sequences without altering the API:"
      ],
      "metadata": {
        "id": "KBHoSiBpg_-_"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "b2XrNql-gSdP"
      },
      "outputs": [],
      "source": [
        "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
        "\n",
        "model_inputs = tokenizer(sequences)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Offering diverse padding approaches to cater to distinct objectives:\n"
      ],
      "metadata": {
        "id": "jkytnwjchCN-"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 50,
      "metadata": {
        "id": "OLtXjINdgSdQ"
      },
      "outputs": [],
      "source": [
        "# Will pad the sequences up to the maximum sequence length\n",
        "model_inputs = tokenizer(sequences, padding=\"longest\")\n",
        "\n",
        "# Will pad the sequences up to the model max length\n",
        "# (512 for BERT or DistilBERT)\n",
        "model_inputs = tokenizer(sequences, padding=\"max_length\")\n",
        "\n",
        "# Will pad the sequences up to the specified max length\n",
        "model_inputs = tokenizer(sequences, padding=\"max_length\", max_length=8)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "Additionally, it adeptly truncates sequences:\n",
        "\n",
        "\n"
      ],
      "metadata": {
        "id": "g4n-n51NhGgG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "id": "HCrcA34AgSdQ"
      },
      "outputs": [],
      "source": [
        "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
        "\n",
        "# Will truncate the sequences that are longer than the model max length\n",
        "# (512 for BERT or DistilBERT)\n",
        "model_inputs = tokenizer(sequences, truncation=True)\n",
        "\n",
        "# Will truncate the sequences that are longer than the specified max length\n",
        "model_inputs = tokenizer(sequences, max_length=8, truncation=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "The tokenizer object effortlessly handles tensor conversions specific to various frameworks. In the snippet below, the tokenizer is directed to produce tensors compatible with different frameworks:"
      ],
      "metadata": {
        "id": "XrUpQKy9hIlD"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 52,
      "metadata": {
        "id": "sRsMoUMugSdQ"
      },
      "outputs": [],
      "source": [
        "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
        "\n",
        "# Returns PyTorch tensors\n",
        "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"pt\")\n",
        "\n",
        "# Returns TensorFlow tensors\n",
        "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"tf\")\n",
        "\n",
        "# Returns NumPy arrays\n",
        "model_inputs = tokenizer(sequences, padding=True, return_tensors=\"np\")"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Special Tokens in Tokenized Sequences\n",
        "\n",
        "Upon inspecting the input IDs produced by the tokenizer, a slight difference is evident compared to earlier representations:"
      ],
      "metadata": {
        "id": "og2SVl4bhUyl"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 53,
      "metadata": {
        "id": "Opg4ggkSgSdR",
        "outputId": "f9dc9c07-c75c-45cc-f3a7-a6c5e8983976",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[101, 1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012, 102]\n",
            "[1045, 1005, 2310, 2042, 3403, 2005, 1037, 17662, 12172, 2607, 2026, 2878, 2166, 1012]\n"
          ]
        }
      ],
      "source": [
        "sequence = \"I've been waiting for a HuggingFace course my whole life.\"\n",
        "\n",
        "model_inputs = tokenizer(sequence)\n",
        "print(model_inputs[\"input_ids\"])\n",
        "\n",
        "tokens = tokenizer.tokenize(sequence)\n",
        "ids = tokenizer.convert_tokens_to_ids(tokens)\n",
        "print(ids)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Upon decoding the two sequences of IDs mentioned above, the presence of special tokens becomes apparent:"
      ],
      "metadata": {
        "id": "d0VAtMEthZ9G"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 54,
      "metadata": {
        "id": "o2Nc2Iv1gSdR",
        "outputId": "7e167ffa-8104-4184-d3bf-a61c1c2d9b1b",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[CLS] i've been waiting for a huggingface course my whole life. [SEP]\n",
            "i've been waiting for a huggingface course my whole life.\n"
          ]
        }
      ],
      "source": [
        "print(tokenizer.decode(model_inputs[\"input_ids\"]))\n",
        "print(tokenizer.decode(ids))"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The tokenizer has inserted the special token `[CLS]` at the sequence's start and `[SEP]` at its conclusion. This addition stems from the model's pretraining requirements, ensuring consistency for inference. It is important to note that certain models might not introduce special tokens, or they might include different ones. Additionally, some models could add these special tokens solely at the start or only at the end. The tokenizer handles these distinctions automatically, understanding the requisite tokens and managing them accordingly.\n"
      ],
      "metadata": {
        "id": "lPhFvFMxheaq"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Concluding the Tokenizer's Functionality\n",
        "\n",
        "In summarizing the diverse capabilities of the tokenizer object when dealing with text inputs, we revisit its proficiency in managing various scenarios involving multiple sequences, long sequences, and diverse tensor types via its core API."
      ],
      "metadata": {
        "id": "QYXeq5cuhkBX"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "dgLyH-8ogSdR"
      },
      "outputs": [],
      "source": [
        "import torch\n",
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "checkpoint = \"distilbert-base-uncased-finetuned-sst-2-english\"\n",
        "tokenizer = AutoTokenizer.from_pretrained(checkpoint)\n",
        "model = AutoModelForSequenceClassification.from_pretrained(checkpoint)\n",
        "sequences = [\"I've been waiting for a HuggingFace course my whole life.\", \"So have I!\"]\n",
        "\n",
        "tokens = tokenizer(sequences, padding=True, truncation=True, return_tensors=\"pt\")\n",
        "output = model(**tokens)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This comprehensive representation underscores the tokenizer's adaptability in handling multiple sequences-efficiently managing padding, accommodating lengthy sequences through truncation, and seamlessly converting inputs into different tensor types. The tokenizer's extensive functionality streamlines text processing across a multitude of scenarios.\n",
        "\n"
      ],
      "metadata": {
        "id": "eQGCBJ-VhpSL"
      }
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "gvUP1mKmhprH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [],
      "metadata": {
        "id": "ra11jsAMh2Ku"
      }
    }
  ],
  "metadata": {
    "colab": {
      "provenance": [],
      "collapsed_sections": [
        "tM38xPIjzXAh"
      ]
    },
    "language_info": {
      "name": "python"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "571e63ec6edb4ddbb71f8a1cabfac887": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b24a4aa9d6f94e3ba9ad996d5192cfba",
              "IPY_MODEL_0ced62ff865f4e5397ae9fdc69171efe",
              "IPY_MODEL_477023d24b0c4abd98c26618b750fe9b"
            ],
            "layout": "IPY_MODEL_24a919b6f4e945f0adbb985c8740aa31"
          }
        },
        "b24a4aa9d6f94e3ba9ad996d5192cfba": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9646f36da355470d9de945e5acf50c40",
            "placeholder": "​",
            "style": "IPY_MODEL_be40c1d552394dc29394e943c86ce4c2",
            "value": "config.json: 100%"
          }
        },
        "0ced62ff865f4e5397ae9fdc69171efe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b73f5737bbd24477aa0dd9701071f52b",
            "max": 629,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_857613b1fe644ebc9ed9a89dbc4ec82e",
            "value": 629
          }
        },
        "477023d24b0c4abd98c26618b750fe9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c2189cdb1881446daa064db7d5d25646",
            "placeholder": "​",
            "style": "IPY_MODEL_07a2ab84a0e648079b42d0caf46d21f2",
            "value": " 629/629 [00:00&lt;00:00, 18.5kB/s]"
          }
        },
        "24a919b6f4e945f0adbb985c8740aa31": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9646f36da355470d9de945e5acf50c40": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be40c1d552394dc29394e943c86ce4c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b73f5737bbd24477aa0dd9701071f52b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "857613b1fe644ebc9ed9a89dbc4ec82e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c2189cdb1881446daa064db7d5d25646": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "07a2ab84a0e648079b42d0caf46d21f2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "89d079303c0f47559675abe3b7e6455e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_014f434ac08440a9ad7e813377f34c3b",
              "IPY_MODEL_f01baec0dbc04853a1559f8fda3ae2fe",
              "IPY_MODEL_743051740f1948b7a74eac90e13787fe"
            ],
            "layout": "IPY_MODEL_12b7ec239ac0466d91fddb3b62be9d81"
          }
        },
        "014f434ac08440a9ad7e813377f34c3b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_21cff44c890b4447a8822f32f3112420",
            "placeholder": "​",
            "style": "IPY_MODEL_88169a31c37f4d69b9b7dc02201ea954",
            "value": "model.safetensors: 100%"
          }
        },
        "f01baec0dbc04853a1559f8fda3ae2fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2f20117bfab04d1796a59e865c8685e4",
            "max": 267832558,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5d8dbed5cafa4c9582f5191032b92d23",
            "value": 267832558
          }
        },
        "743051740f1948b7a74eac90e13787fe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9afe106aa7f3489ab3060070fb75650b",
            "placeholder": "​",
            "style": "IPY_MODEL_e8581583a14d42f492d1b49dbe74f000",
            "value": " 268M/268M [00:03&lt;00:00, 77.6MB/s]"
          }
        },
        "12b7ec239ac0466d91fddb3b62be9d81": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "21cff44c890b4447a8822f32f3112420": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88169a31c37f4d69b9b7dc02201ea954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2f20117bfab04d1796a59e865c8685e4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5d8dbed5cafa4c9582f5191032b92d23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9afe106aa7f3489ab3060070fb75650b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8581583a14d42f492d1b49dbe74f000": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "255b6d535f864ae68f625ffd32e5a1a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bac5ff3f45584e62965af68326ac5d24",
              "IPY_MODEL_ceb8356c57804fb9b1d1a2ba72728c7e",
              "IPY_MODEL_7320ada6c1364180a771484a071a60c0"
            ],
            "layout": "IPY_MODEL_3a64c9792f9c4aaf97d3830930934ee3"
          }
        },
        "bac5ff3f45584e62965af68326ac5d24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f16ee8c592684eba8ada32aa5d2128e6",
            "placeholder": "​",
            "style": "IPY_MODEL_bcd9a622a84446f185c579244aecb7cf",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "ceb8356c57804fb9b1d1a2ba72728c7e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d4651d14129242458e87f5882104a84b",
            "max": 48,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_bb3b2c67015047d784edf8f9f58f1131",
            "value": 48
          }
        },
        "7320ada6c1364180a771484a071a60c0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_66cec7b55b5c4997b627b6d3e69556a8",
            "placeholder": "​",
            "style": "IPY_MODEL_646a39447b7742c0a499855ba9372596",
            "value": " 48.0/48.0 [00:00&lt;00:00, 1.99kB/s]"
          }
        },
        "3a64c9792f9c4aaf97d3830930934ee3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f16ee8c592684eba8ada32aa5d2128e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bcd9a622a84446f185c579244aecb7cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d4651d14129242458e87f5882104a84b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bb3b2c67015047d784edf8f9f58f1131": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "66cec7b55b5c4997b627b6d3e69556a8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "646a39447b7742c0a499855ba9372596": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a2f93e9e509e46ac9ff7453788d9444f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_32f76ac706ea451eac1782ca030b26e2",
              "IPY_MODEL_c81d667434ff41aa8df709eef1a728a2",
              "IPY_MODEL_a1f63f693b2944c09bbf1718e6432f11"
            ],
            "layout": "IPY_MODEL_ae765fb243f14841a0cbc6750656c4db"
          }
        },
        "32f76ac706ea451eac1782ca030b26e2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b201d521c34a4e0ba3df6930280a004f",
            "placeholder": "​",
            "style": "IPY_MODEL_c16bdcd544d846bb8e9eef928c8a9506",
            "value": "vocab.txt: 100%"
          }
        },
        "c81d667434ff41aa8df709eef1a728a2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d81311218dbb433ab745dd3ad5ff9f9f",
            "max": 231508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3f1a1e84520e4625832df6fc845dc7cf",
            "value": 231508
          }
        },
        "a1f63f693b2944c09bbf1718e6432f11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d379424dac043bf9ab703a530492720",
            "placeholder": "​",
            "style": "IPY_MODEL_fc59489776574abdbe27d629c4440331",
            "value": " 232k/232k [00:00&lt;00:00, 591kB/s]"
          }
        },
        "ae765fb243f14841a0cbc6750656c4db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b201d521c34a4e0ba3df6930280a004f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c16bdcd544d846bb8e9eef928c8a9506": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d81311218dbb433ab745dd3ad5ff9f9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3f1a1e84520e4625832df6fc845dc7cf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1d379424dac043bf9ab703a530492720": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fc59489776574abdbe27d629c4440331": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ca25cf3d11f944babdd0b9a961c1592a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_0ea5db5350874ec089adc09229ae5937",
              "IPY_MODEL_410a2d13a72b4e1e8da604a32ab1ff57",
              "IPY_MODEL_bb40880c682047109cdbeb361ab3178f"
            ],
            "layout": "IPY_MODEL_c9a8c9bb8e0b4c2fbf0e9d72f654483f"
          }
        },
        "0ea5db5350874ec089adc09229ae5937": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9b09849a6c14ad4840d2f45afc6e907",
            "placeholder": "​",
            "style": "IPY_MODEL_0c0df20e42484a3fafd9e1953a509862",
            "value": "config.json: 100%"
          }
        },
        "410a2d13a72b4e1e8da604a32ab1ff57": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_574903aadd444978b8d5f5e01f3ab659",
            "max": 570,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6f7969b2b07b463b9f23071933088cfe",
            "value": 570
          }
        },
        "bb40880c682047109cdbeb361ab3178f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f249d1da6e1e471ea267195bb14fb09b",
            "placeholder": "​",
            "style": "IPY_MODEL_79e6dd3760cb43958497c258e7b99ead",
            "value": " 570/570 [00:00&lt;00:00, 8.96kB/s]"
          }
        },
        "c9a8c9bb8e0b4c2fbf0e9d72f654483f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9b09849a6c14ad4840d2f45afc6e907": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c0df20e42484a3fafd9e1953a509862": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "574903aadd444978b8d5f5e01f3ab659": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6f7969b2b07b463b9f23071933088cfe": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f249d1da6e1e471ea267195bb14fb09b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79e6dd3760cb43958497c258e7b99ead": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "87a14b99ba9d40c6a69ee7fc0e8ed95e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_79be3a4cee6d45368d49ed5a07be1153",
              "IPY_MODEL_683b2a52f93d4ed3816a03f725c64f55",
              "IPY_MODEL_6dc2cc649d4c421aaf293279e85729f9"
            ],
            "layout": "IPY_MODEL_47825908a90248538b510444895cfe13"
          }
        },
        "79be3a4cee6d45368d49ed5a07be1153": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8d1dc7988e04429c8d60113c038362b1",
            "placeholder": "​",
            "style": "IPY_MODEL_6652dabd747b4cdabfde969f614d7420",
            "value": "model.safetensors: 100%"
          }
        },
        "683b2a52f93d4ed3816a03f725c64f55": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_99013e28d4b94b64a3d917b0e69d0f82",
            "max": 435755784,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ccecaaa0fdb040369a749926cb947e11",
            "value": 435755784
          }
        },
        "6dc2cc649d4c421aaf293279e85729f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ccc3976ab50240f3b70caf611bfc665c",
            "placeholder": "​",
            "style": "IPY_MODEL_f37c5d1c783442bbabb7c156992c5165",
            "value": " 436M/436M [00:05&lt;00:00, 81.0MB/s]"
          }
        },
        "47825908a90248538b510444895cfe13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8d1dc7988e04429c8d60113c038362b1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6652dabd747b4cdabfde969f614d7420": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "99013e28d4b94b64a3d917b0e69d0f82": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ccecaaa0fdb040369a749926cb947e11": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ccc3976ab50240f3b70caf611bfc665c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f37c5d1c783442bbabb7c156992c5165": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3212ee6425f34dcea759e488d1072e41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1e538fdc010f4ae6b35d027bffd6f7db",
              "IPY_MODEL_c6fff925af90400eb6f88cbccbf999ec",
              "IPY_MODEL_9439a66e3fd04214ba3b6aad91c1de60"
            ],
            "layout": "IPY_MODEL_9f958874a2d54602b83989162b666f3a"
          }
        },
        "1e538fdc010f4ae6b35d027bffd6f7db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8fcc2079e09345349cb04813a4e57025",
            "placeholder": "​",
            "style": "IPY_MODEL_59b92687aaa8428fa7454bc0540126ce",
            "value": "tokenizer_config.json: 100%"
          }
        },
        "c6fff925af90400eb6f88cbccbf999ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62664275c5e34a22b2ad5dfc0ae705b6",
            "max": 29,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3ce03d5d40914b84bceddbf98843df38",
            "value": 29
          }
        },
        "9439a66e3fd04214ba3b6aad91c1de60": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b70a68e58b0c45949bbcc818c5dba425",
            "placeholder": "​",
            "style": "IPY_MODEL_f323302316a04536bd131d691024d0aa",
            "value": " 29.0/29.0 [00:00&lt;00:00, 566B/s]"
          }
        },
        "9f958874a2d54602b83989162b666f3a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fcc2079e09345349cb04813a4e57025": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "59b92687aaa8428fa7454bc0540126ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "62664275c5e34a22b2ad5dfc0ae705b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3ce03d5d40914b84bceddbf98843df38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b70a68e58b0c45949bbcc818c5dba425": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f323302316a04536bd131d691024d0aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "931e29c8e4eb412ea4435631d7f6ecd7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_168ebc56a4064e0d9d0f75f0e91f7466",
              "IPY_MODEL_b52758fea3384ebaa8b4ec5cec8246ce",
              "IPY_MODEL_a05c1cc497774c7795edd9e639ffc371"
            ],
            "layout": "IPY_MODEL_2323b8e4a5df4a09b0ac6f4571eb6edf"
          }
        },
        "168ebc56a4064e0d9d0f75f0e91f7466": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6e9d75846f5c49819a9e95a60578741b",
            "placeholder": "​",
            "style": "IPY_MODEL_a9173759a6fc40a4a96b62083c103818",
            "value": "vocab.txt: 100%"
          }
        },
        "b52758fea3384ebaa8b4ec5cec8246ce": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_427106c41dde4dfb8be90ad0c333da2c",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_25d3dff6ed1947b8acf1e2e4e487e2ec",
            "value": 213450
          }
        },
        "a05c1cc497774c7795edd9e639ffc371": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3e1ffb81f98b4405b069c9e0ddc3e6af",
            "placeholder": "​",
            "style": "IPY_MODEL_46b02352b66842eca83fed3e7c8a06fc",
            "value": " 213k/213k [00:00&lt;00:00, 549kB/s]"
          }
        },
        "2323b8e4a5df4a09b0ac6f4571eb6edf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6e9d75846f5c49819a9e95a60578741b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9173759a6fc40a4a96b62083c103818": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "427106c41dde4dfb8be90ad0c333da2c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25d3dff6ed1947b8acf1e2e4e487e2ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3e1ffb81f98b4405b069c9e0ddc3e6af": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "46b02352b66842eca83fed3e7c8a06fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f05ba1bc80944ab387dbe2fb3fd0bbc6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cfb0ae744f5e460ebdbe00438ef10547",
              "IPY_MODEL_609df9531b604174a07d964dca34da7f",
              "IPY_MODEL_1225fceccd8b4e81a7b40f6983cb02bc"
            ],
            "layout": "IPY_MODEL_c2384fa4cac04630832cc2fd2011b98c"
          }
        },
        "cfb0ae744f5e460ebdbe00438ef10547": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2311567c05d74c52ae4deb358b9fcc56",
            "placeholder": "​",
            "style": "IPY_MODEL_fe899c4870c4471abccf39e5088c711d",
            "value": "tokenizer.json: 100%"
          }
        },
        "609df9531b604174a07d964dca34da7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_51a1d9ac155d49f594b0ff6e53d81e89",
            "max": 435797,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_3a0f124e233d485db89851819ed9e3dd",
            "value": 435797
          }
        },
        "1225fceccd8b4e81a7b40f6983cb02bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2226fade94a445c5b8ee2e871d51bcc4",
            "placeholder": "​",
            "style": "IPY_MODEL_64ed462019d645b285cb27c4fb50ad56",
            "value": " 436k/436k [00:00&lt;00:00, 747kB/s]"
          }
        },
        "c2384fa4cac04630832cc2fd2011b98c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2311567c05d74c52ae4deb358b9fcc56": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fe899c4870c4471abccf39e5088c711d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "51a1d9ac155d49f594b0ff6e53d81e89": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3a0f124e233d485db89851819ed9e3dd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "2226fade94a445c5b8ee2e871d51bcc4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64ed462019d645b285cb27c4fb50ad56": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}