Abstractive_Text_Summarization_with_T5.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Abstractive Text Summarization with T5",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1duWhnWgbrRSTgu4IFb22xQpK5T3oSPLA",
      "authorship_tag": "ABX9TyPiuArhKpzIKhG0kFYED6bf"
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UaiQF1Bf6hPb",
        "outputId": "864363d9-9576-491e-e9f1-ad1deff8c67d"
      },
      "source": [
        "!pip install transformers -q\n",
        "!pip install wandb -q\n"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 2.1MB 5.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.3MB 38.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 901kB 39.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 2.1MB 4.2MB/s \n",
            "\u001b[K     |████████████████████████████████| 133kB 39.5MB/s \n",
            "\u001b[K     |████████████████████████████████| 163kB 37.3MB/s \n",
            "\u001b[K     |████████████████████████████████| 102kB 10.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 71kB 8.5MB/s \n",
            "\u001b[?25h  Building wheel for pathtools (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Building wheel for subprocess32 (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pqZ4SLAn6UuC"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import torch \n",
        "from torch.utils.data import DataLoader , Dataset ,RandomSampler,SequentialSampler  \n",
        "from transformers import T5Tokenizer , T5ForConditionalGeneration\n",
        "import wandb"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0OtFewnvO4dF"
      },
      "source": [
        "#about data\n",
        "\n",
        "# News Summary dataset available at Kaggle\n",
        "# There are4514 rows of data. \n",
        "# Where each row has the following data-point:\n",
        "# author : Author of the article\n",
        "# date : Date the article was published\n",
        "# headline: Headline for the published article\n",
        "# read_more : URL for the article to follow online\n",
        "# text: This is the summary of the article\n",
        "# ctext: This is the complete article\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pm5Y5zJZ6O-B",
        "outputId": "1326dc84-007f-48e1-c27d-b455f3f002ba"
      },
      "source": [
        "!nvidia-smi\n",
        "device   = \"cuda\""
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Sun Apr 25 13:23:30 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 465.19.01    Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla K80           Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   34C    P8    28W / 149W |      0MiB / 11441MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QD2nlE9m6P8M",
        "outputId": "b522a836-6267-48a2-f464-82f8ee7e2472"
      },
      "source": [
        "!wandb login"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mhusnain-ali21\u001b[0m (use `wandb login --relogin` to force relogin)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MGQHqZAV6qPw"
      },
      "source": [
        "class DataSet(Dataset):\n",
        "    def __init__(self , df , tokenizer ,src_len , sum_len ):\n",
        "        super(DataSet ,self).__init__()\n",
        "        self.data = df\n",
        "        self.tokenizer  = tokenizer\n",
        "        self.src_len = src_len\n",
        "        self.summary_len = sum_len\n",
        "        self.summaries = self.data.text \n",
        "        self.articles = self.data.ctext\n",
        "\n",
        "        def __len__(self):\n",
        "            return len(self.summary)\n",
        "\n",
        "        def __getitem__(self , index):\n",
        "            article = str(self.articles[index])\n",
        "            article = \" \".join(articles.split())\n",
        "            summary = str(self.summaries[index])\n",
        "            summary = \" \".join(summary.split())\n",
        "            #The tokenizer uses the batch_encode_plus method to perform tokenization and generate\n",
        "            # the necessary outputs, namely:source_id, source_mask \n",
        "            #from the actual text and target_id and target_mask from the summary text.\n",
        "\n",
        "            src = self.tokenizer.batch_encode_plus([article]\\\n",
        "                    ,max_length = self.src_len , pad_to_max_len = True ,return_tensors =\"pt\")\n",
        "            trg = self.tokenizer.batch_encode_plus([summary]\\\n",
        "                    ,max_length = self.summary_len , pad_to_max_len = True ,return_tensors =\"pt\")\n",
        "            \n",
        "            source_ids = src[\"input_ids\"].squeeze()\n",
        "            target_ids = trg[\"input_ids\"].squeeze()\n",
        "\n",
        "            source_masks = src[\"attention_mask\"].squeeze()\n",
        "            target_masks = trg[\"attention_mask\"].squeeze()\n",
        "\n",
        "            return {\n",
        "            'source_ids': source_ids.to(dtype=torch.long), \n",
        "            'source_masks': source_mask.to(dtype=torch.long), \n",
        "            'target_ids': target_ids.to(dtype=torch.long),\n",
        "            'target_ids_y': target_ids.to(dtype=torch.long)\n",
        "\n",
        "            }\n",
        "\n",
        "\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}