AbstractT5-Small.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "gpuClass": "standard"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ralfferreira/generate-abstract/blob/main/Abstract.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 278,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HvJm5NncaIJM",
        "outputId": "4dd62d51-a113-4d01-87f0-96f4fa6131fa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.8/dist-packages (4.25.1)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.8/dist-packages (2.8.0)\n",
            "Requirement already satisfied: evaluate in /usr/local/lib/python3.8/dist-packages (0.4.0)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.8/dist-packages (1.13.0+cu116)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.8.2)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.11.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.13.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.8/dist-packages (from datasets) (3.1.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.8/dist-packages (from datasets) (0.70.14)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2022.11.0)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.3)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.1.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.25.11)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.6)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
          ]
        }
      ],
      "source": [
        "pip install transformers datasets evaluate torch"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "os.environ['PYTORCH_CUDA_ALLOC_CONF'] = 'max_split_size_mb:2'"
      ],
      "metadata": {
        "id": "NrdSn0ztpU8a"
      },
      "execution_count": 279,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "sci_papers = load_dataset(\"hackathon-pln-es/scientific_papers_en\", split=\"train\")\n",
        "sci_papers = sci_papers.train_test_split(test_size=0.2)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RBqvTuM-aUXg",
        "outputId": "37b2e8b6-dd05-4c3b-b377-3244f460dd26"
      },
      "execution_count": 280,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.builder:Using custom data configuration hackathon-pln-es--scientific_papers_en-70a8e377796a64cf\n",
            "WARNING:datasets.builder:Found cached dataset csv (/root/.cache/huggingface/datasets/hackathon-pln-es___csv/hackathon-pln-es--scientific_papers_en-70a8e377796a64cf/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317)\n",
            "WARNING:datasets.arrow_dataset:Loading cached split indices for dataset at /root/.cache/huggingface/datasets/hackathon-pln-es___csv/hackathon-pln-es--scientific_papers_en-70a8e377796a64cf/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-b64ccc4c8cfdda71.arrow and /root/.cache/huggingface/datasets/hackathon-pln-es___csv/hackathon-pln-es--scientific_papers_en-70a8e377796a64cf/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-12ab02ee5de92882.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n",
        "model = (\"t5-small\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SN_ENPGFaj0K",
        "outputId": "d8f49c33-56b7-4d6f-ee8a-2dd8f3b16eea"
      },
      "execution_count": 281,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/config.json\n",
            "Model config T5Config {\n",
            "  \"_name_or_path\": \"t5-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dense_act_fn\": \"relu\",\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"is_gated_act\": false,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_max_distance\": 128,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.25.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "loading file spiece.model from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/spiece.model\n",
            "loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/tokenizer.json\n",
            "loading file added_tokens.json from cache at None\n",
            "loading file special_tokens_map.json from cache at None\n",
            "loading file tokenizer_config.json from cache at None\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/config.json\n",
            "Model config T5Config {\n",
            "  \"_name_or_path\": \"t5-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dense_act_fn\": \"relu\",\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"is_gated_act\": false,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_max_distance\": 128,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.25.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "prefix = \"summarize: \"\n",
        "\n",
        "\n",
        "def preprocess_function(examples):\n",
        "    inputs = [prefix + doc for doc in examples[\"full_text\"]]\n",
        "    model_inputs = tokenizer(inputs, max_length=1024, truncation=True)\n",
        "\n",
        "    labels = tokenizer(text_target=examples[\"abstract\"], max_length=500, truncation=True)\n",
        "\n",
        "    model_inputs[\"labels\"] = labels[\"input_ids\"]\n",
        "    return model_inputs"
      ],
      "metadata": {
        "id": "ZfuX3yS4asQx"
      },
      "execution_count": 282,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenized_sci_papers = sci_papers.map(preprocess_function, batched=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QJbqRqTgRhZF",
        "outputId": "75f9c91b-2656-4024-8626-f696dce45ab2"
      },
      "execution_count": 283,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/hackathon-pln-es___csv/hackathon-pln-es--scientific_papers_en-70a8e377796a64cf/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-f626c48ff4e9b630.arrow\n",
            "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/hackathon-pln-es___csv/hackathon-pln-es--scientific_papers_en-70a8e377796a64cf/0.0.0/6b34fb8fcf56f7c8ba51dc895bfa2bfbe43546f190a60fcf74bb5e8afdcc2317/cache-85de029a69f682f8.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import DataCollatorForSeq2Seq\n",
        "\n",
        "data_collator = DataCollatorForSeq2Seq(tokenizer=tokenizer, model=model)"
      ],
      "metadata": {
        "id": "UZ6heqypa46y"
      },
      "execution_count": 284,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install rouge_score\n",
        "import evaluate\n",
        "\n",
        "rouge = evaluate.load(\"rouge\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x_gFiNWJbeVA",
        "outputId": "8ef802db-de75-421c-d26d-32378cfc2857"
      },
      "execution_count": 285,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: rouge_score in /usr/local/lib/python3.8/dist-packages (0.1.2)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.8/dist-packages (from rouge_score) (1.21.6)\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.8/dist-packages (from rouge_score) (3.7)\n",
            "Requirement already satisfied: absl-py in /usr/local/lib/python3.8/dist-packages (from rouge_score) (1.3.0)\n",
            "Requirement already satisfied: six>=1.14.0 in /usr/local/lib/python3.8/dist-packages (from rouge_score) (1.15.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.8/dist-packages (from nltk->rouge_score) (2022.6.2)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.8/dist-packages (from nltk->rouge_score) (4.64.1)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.8/dist-packages (from nltk->rouge_score) (1.2.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.8/dist-packages (from nltk->rouge_score) (7.1.2)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "\n",
        "\n",
        "def compute_metrics(eval_pred):\n",
        "    predictions, labels = eval_pred\n",
        "    decoded_preds = tokenizer.batch_decode(predictions, skip_special_tokens=True)\n",
        "    labels = np.where(labels != -100, labels, tokenizer.pad_token_id)\n",
        "    decoded_labels = tokenizer.batch_decode(labels, skip_special_tokens=True)\n",
        "\n",
        "    result = rouge.compute(predictions=decoded_preds, references=decoded_labels, use_stemmer=True)\n",
        "\n",
        "    prediction_lens = [np.count_nonzero(pred != tokenizer.pad_token_id) for pred in predictions]\n",
        "    result[\"gen_len\"] = np.mean(prediction_lens)\n",
        "\n",
        "    return {k: round(v, 4) for k, v in result.items()}"
      ],
      "metadata": {
        "id": "jZC2CAmJbp6D"
      },
      "execution_count": 286,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForSeq2SeqLM, Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
        "\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R_BCaFQNbuSI",
        "outputId": "81fd2bc3-b033-48ca-dafa-e56b50cb3bb2"
      },
      "execution_count": 287,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/config.json\n",
            "Model config T5Config {\n",
            "  \"_name_or_path\": \"t5-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dense_act_fn\": \"relu\",\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"is_gated_act\": false,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_max_distance\": 128,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.25.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/pytorch_model.bin\n",
            "All model checkpoint weights were used when initializing T5ForConditionalGeneration.\n",
            "\n",
            "All the weights of T5ForConditionalGeneration were initialized from the model checkpoint at t5-small.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use T5ForConditionalGeneration for predictions without further training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "cuda0 = torch.device('cuda:0')\n",
        "\n",
        "x = torch.tensor([1., 2.], device=cuda0)\n",
        "# x.device is device(type='cuda', index=0)\n",
        "\n",
        "import torch\n",
        "torch.cuda.empty_cache()\n",
        "torch.cuda.memory_summary(device=cuda0, abbreviated=False)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 435
        },
        "id": "59vqJjxblpvf",
        "outputId": "0eefc734-5c53-427d-caeb-e2aedb00531a"
      },
      "execution_count": 288,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'|===========================================================================|\\n|                  PyTorch CUDA memory summary, device ID 0                 |\\n|---------------------------------------------------------------------------|\\n|            CUDA OOMs: 16           |        cudaMalloc retries: 17        |\\n|===========================================================================|\\n|        Metric         | Cur Usage  | Peak Usage | Tot Alloc  | Tot Freed  |\\n|---------------------------------------------------------------------------|\\n| Allocated memory      |  236355 KB |   14273 MB |   42584 MB |   42353 MB |\\n|       from large pool |  162560 KB |   14175 MB |   42395 MB |   42237 MB |\\n|       from small pool |   73795 KB |     145 MB |     188 MB |     116 MB |\\n|---------------------------------------------------------------------------|\\n| Active memory         |  236355 KB |   14273 MB |   42584 MB |   42353 MB |\\n|       from large pool |  162560 KB |   14175 MB |   42395 MB |   42237 MB |\\n|       from small pool |   73795 KB |     145 MB |     188 MB |     116 MB |\\n|---------------------------------------------------------------------------|\\n| GPU reserved memory   |  272384 KB |   14312 MB |   28256 MB |   27990 MB |\\n|       from large pool |  194560 KB |   14212 MB |   28086 MB |   27896 MB |\\n|       from small pool |   77824 KB |     146 MB |     170 MB |      94 MB |\\n|---------------------------------------------------------------------------|\\n| Non-releasable memory |   36029 KB |    1581 MB |   18823 MB |   18788 MB |\\n|       from large pool |   32000 KB |    1580 MB |   18667 MB |   18636 MB |\\n|       from small pool |    4029 KB |       4 MB |     155 MB |     151 MB |\\n|---------------------------------------------------------------------------|\\n| Allocations           |     133    |     401    |    1135    |    1002    |\\n|       from large pool |      25    |     239    |     701    |     676    |\\n|       from small pool |     108    |     221    |     434    |     326    |\\n|---------------------------------------------------------------------------|\\n| Active allocs         |     133    |     401    |    1135    |    1002    |\\n|       from large pool |      25    |     239    |     701    |     676    |\\n|       from small pool |     108    |     221    |     434    |     326    |\\n|---------------------------------------------------------------------------|\\n| GPU reserved segments |      46    |     174    |     282    |     236    |\\n|       from large pool |       8    |     108    |     197    |     189    |\\n|       from small pool |      38    |      73    |      85    |      47    |\\n|---------------------------------------------------------------------------|\\n| Non-releasable allocs |      13    |      34    |     475    |     462    |\\n|       from large pool |       8    |      28    |     269    |     261    |\\n|       from small pool |       5    |       9    |     206    |     201    |\\n|---------------------------------------------------------------------------|\\n| Oversize allocations  |       0    |       0    |       0    |       0    |\\n|---------------------------------------------------------------------------|\\n| Oversize GPU segments |       0    |       0    |       0    |       0    |\\n|===========================================================================|\\n'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 288
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "training_args = Seq2SeqTrainingArguments(\n",
        "    output_dir=\"test_dir\",\n",
        "    evaluation_strategy=\"epoch\",\n",
        "    learning_rate=2e-5,\n",
        "    per_device_train_batch_size=2,\n",
        "    per_device_eval_batch_size=2,\n",
        "    weight_decay=0.01,\n",
        "    save_total_limit=3,\n",
        "    num_train_epochs=2,\n",
        "    predict_with_generate=True,\n",
        ")\n",
        "\n",
        "trainer = Seq2SeqTrainer(\n",
        "    model=model,\n",
        "    args=training_args,\n",
        "    train_dataset=tokenized_sci_papers[\"train\"],\n",
        "    eval_dataset=tokenized_sci_papers[\"test\"],\n",
        "    tokenizer=tokenizer,\n",
        "    data_collator=data_collator,\n",
        "    compute_metrics=compute_metrics,\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 835
        },
        "id": "0kbdE0r5cjKq",
        "outputId": "b236f851-a693-458f-8b05-5d4f06604353"
      },
      "execution_count": 289,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "PyTorch: setting up devices\n",
            "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
            "The following columns in the training set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: abstract, id, text_no_abstract, full_text. If abstract, id, text_no_abstract, full_text are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/optimization.py:306: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
            "  warnings.warn(\n",
            "***** Running training *****\n",
            "  Num examples = 1403\n",
            "  Num Epochs = 2\n",
            "  Instantaneous batch size per device = 2\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 2\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 1404\n",
            "  Number of trainable parameters = 60506624\n",
            "You're using a T5TokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ],
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='1404' max='1404' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1404/1404 07:48, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              " <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Rouge1</th>\n",
              "      <th>Rouge2</th>\n",
              "      <th>Rougel</th>\n",
              "      <th>Rougelsum</th>\n",
              "      <th>Gen Len</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.312500</td>\n",
              "      <td>0.626091</td>\n",
              "      <td>0.218400</td>\n",
              "      <td>0.192100</td>\n",
              "      <td>0.214800</td>\n",
              "      <td>0.215000</td>\n",
              "      <td>19.000000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.836900</td>\n",
              "      <td>0.592188</td>\n",
              "      <td>0.218900</td>\n",
              "      <td>0.193100</td>\n",
              "      <td>0.215200</td>\n",
              "      <td>0.215700</td>\n",
              "      <td>19.000000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Saving model checkpoint to test_dir/checkpoint-500\n",
            "Configuration saved in test_dir/checkpoint-500/config.json\n",
            "Model weights saved in test_dir/checkpoint-500/pytorch_model.bin\n",
            "tokenizer config file saved in test_dir/checkpoint-500/tokenizer_config.json\n",
            "Special tokens file saved in test_dir/checkpoint-500/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: abstract, id, text_no_abstract, full_text. If abstract, id, text_no_abstract, full_text are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 351\n",
            "  Batch size = 2\n",
            "Saving model checkpoint to test_dir/checkpoint-1000\n",
            "Configuration saved in test_dir/checkpoint-1000/config.json\n",
            "Model weights saved in test_dir/checkpoint-1000/pytorch_model.bin\n",
            "tokenizer config file saved in test_dir/checkpoint-1000/tokenizer_config.json\n",
            "Special tokens file saved in test_dir/checkpoint-1000/special_tokens_map.json\n",
            "The following columns in the evaluation set don't have a corresponding argument in `T5ForConditionalGeneration.forward` and have been ignored: abstract, id, text_no_abstract, full_text. If abstract, id, text_no_abstract, full_text are not expected by `T5ForConditionalGeneration.forward`,  you can safely ignore this message.\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 351\n",
            "  Batch size = 2\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1404, training_loss=0.9900504870292468, metrics={'train_runtime': 469.3334, 'train_samples_per_second': 5.979, 'train_steps_per_second': 2.991, 'total_flos': 759538189860864.0, 'train_loss': 0.9900504870292468, 'epoch': 2.0})"
            ]
          },
          "metadata": {},
          "execution_count": 289
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "artigo=\"\"\"Recurrent models typically factor computation along the symbol positions of the input and output\n",
        "sequences. Aligning the positions to steps in computation time, they generate a sequence of hidden\n",
        "states ht, as a function of the previous hidden state ht−1 and the input for position t. This inherently\n",
        "sequential nature precludes parallelization within training examples, which becomes critical at longer\n",
        "sequence lengths, as memory constraints limit batching across examples. Recent work has achieved\n",
        "significant improvements in computational efficiency through factorization tricks [21] and conditional\n",
        "computation [32], while also improving model performance in case of the latter. The fundamental\n",
        "constraint of sequential computation, however, remains.\n",
        "\n",
        "Attention mechanisms have become an integral part of compelling sequence modeling and transduc-\n",
        "tion models in various tasks, allowing modeling of dependencies without regard to their distance in\n",
        "the input or output sequences [2, 19]. In all but a few cases [27], however, such attention mechanisms\n",
        "are used in conjunction with a recurrent network.\n",
        "\n",
        "In this work we propose the Transformer, a model architecture eschewing recurrence and instead\n",
        "relying entirely on an attention mechanism to draw global dependencies between input and output.\n",
        "The Transformer allows for significantly more parallelization and can reach a new state of the art in\n",
        "translation quality after being trained for as little as twelve hours on eight P100 GPUs.\"\"\""
      ],
      "metadata": {
        "id": "Vti6JEIaSjGK"
      },
      "execution_count": 290,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"t5-small\")\n",
        "inputs = tokenizer(artigo, return_tensors=\"pt\").input_ids"
      ],
      "metadata": {
        "id": "59GFJ4Tehzq0",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a80e371a-f28d-48bf-e153-2d22620eaa23"
      },
      "execution_count": 291,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/config.json\n",
            "Model config T5Config {\n",
            "  \"_name_or_path\": \"t5-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dense_act_fn\": \"relu\",\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"is_gated_act\": false,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_max_distance\": 128,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.25.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "loading file spiece.model from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/spiece.model\n",
            "loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/tokenizer.json\n",
            "loading file added_tokens.json from cache at None\n",
            "loading file special_tokens_map.json from cache at None\n",
            "loading file tokenizer_config.json from cache at None\n",
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/config.json\n",
            "Model config T5Config {\n",
            "  \"_name_or_path\": \"t5-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dense_act_fn\": \"relu\",\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"is_gated_act\": false,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_max_distance\": 128,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.25.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "/usr/local/lib/python3.8/dist-packages/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
            "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
            "- Be aware that you SHOULD NOT rely on t5-small automatically truncating your input to 512 when padding/encoding.\n",
            "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
            "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
            "  warnings.warn(\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoModelForSeq2SeqLM\n",
        "\n",
        "model = AutoModelForSeq2SeqLM.from_pretrained(\"t5-small\")\n",
        "outputs = model.generate(inputs, max_new_tokens=500, do_sample=False)"
      ],
      "metadata": {
        "id": "EbDVAe99iA7p",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "91922c52-bd69-442f-98b4-d4749c638f45"
      },
      "execution_count": 292,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/config.json\n",
            "Model config T5Config {\n",
            "  \"_name_or_path\": \"t5-small\",\n",
            "  \"architectures\": [\n",
            "    \"T5ForConditionalGeneration\"\n",
            "  ],\n",
            "  \"d_ff\": 2048,\n",
            "  \"d_kv\": 64,\n",
            "  \"d_model\": 512,\n",
            "  \"decoder_start_token_id\": 0,\n",
            "  \"dense_act_fn\": \"relu\",\n",
            "  \"dropout_rate\": 0.1,\n",
            "  \"eos_token_id\": 1,\n",
            "  \"feed_forward_proj\": \"relu\",\n",
            "  \"initializer_factor\": 1.0,\n",
            "  \"is_encoder_decoder\": true,\n",
            "  \"is_gated_act\": false,\n",
            "  \"layer_norm_epsilon\": 1e-06,\n",
            "  \"model_type\": \"t5\",\n",
            "  \"n_positions\": 512,\n",
            "  \"num_decoder_layers\": 6,\n",
            "  \"num_heads\": 8,\n",
            "  \"num_layers\": 6,\n",
            "  \"output_past\": true,\n",
            "  \"pad_token_id\": 0,\n",
            "  \"relative_attention_max_distance\": 128,\n",
            "  \"relative_attention_num_buckets\": 32,\n",
            "  \"task_specific_params\": {\n",
            "    \"summarization\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"length_penalty\": 2.0,\n",
            "      \"max_length\": 200,\n",
            "      \"min_length\": 30,\n",
            "      \"no_repeat_ngram_size\": 3,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"summarize: \"\n",
            "    },\n",
            "    \"translation_en_to_de\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to German: \"\n",
            "    },\n",
            "    \"translation_en_to_fr\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to French: \"\n",
            "    },\n",
            "    \"translation_en_to_ro\": {\n",
            "      \"early_stopping\": true,\n",
            "      \"max_length\": 300,\n",
            "      \"num_beams\": 4,\n",
            "      \"prefix\": \"translate English to Romanian: \"\n",
            "    }\n",
            "  },\n",
            "  \"transformers_version\": \"4.25.1\",\n",
            "  \"use_cache\": true,\n",
            "  \"vocab_size\": 32128\n",
            "}\n",
            "\n",
            "loading weights file pytorch_model.bin from cache at /root/.cache/huggingface/hub/models--t5-small/snapshots/02bc3551ce463fd96bd4c3735822469bace0396d/pytorch_model.bin\n",
            "All model checkpoint weights were used when initializing T5ForConditionalGeneration.\n",
            "\n",
            "All the weights of T5ForConditionalGeneration were initialized from the model checkpoint at t5-small.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use T5ForConditionalGeneration for predictions without further training.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.decode(outputs[0], skip_special_tokens=True)"
      ],
      "metadata": {
        "id": "LDRIDOGkiGot",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "00e76073-8646-4780-9243-2f8d6e431108"
      },
      "execution_count": 293,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "', allowing for more parallelization and can reach a new state of the art in translation quality. In this work we propose the Transformer, a model architecture eschewing recurrence. Attention mechanisms have become an integral part of compelling sequence modeling and transduc- tion models.'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 293
        }
      ]
    }
  ]
}