pytorch_fine_tuning_hf.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNuRRC2lpPl1WcSKSszjC0a",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shakilkhan24/Playground_DL/blob/main/pytorch_fine_tuning_hf.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# FINETUNING WITH PYTORCH\n",
        "\n"
      ],
      "metadata": {
        "id": "x1Zmp5gZgBtD"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install transformers\n"
      ],
      "metadata": {
        "id": "K6ssNj42PBnS"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install evaluate\n",
        "!pip install datasets"
      ],
      "metadata": {
        "id": "hPnzmOXmPhjh"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import pandas as pd\n",
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader\n",
        "from transformers import BertTokenizerFast, BertForSequenceClassification\n",
        "from tqdm.auto import tqdm\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import accuracy_score"
      ],
      "metadata": {
        "id": "ODNWqrbqaiMh"
      },
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        " PREPARING CUSTOM DATA FOR FINE TUNING HUGGING FACE MODDL\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "eQGsGie_fW8j"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# preprocessing methods\n",
        "AutoTokenizer (text) , AutoImageProcessor(image),\n",
        "\n",
        "AutoFeatureExtractor(audio) ,AutoProcessor(multi-model)"
      ],
      "metadata": {
        "id": "E5npM8VAgS-0"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "class MyDataset(Dataset):\n",
        "    def __init__(self, df):\n",
        "        self.df = df\n",
        "        self.tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.df)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        row = self.df.iloc[idx]\n",
        "        # use tokenizer to tokenize\n",
        "        encoding = self.tokenizer(row['text'], add_special_tokens=True, padding='max_length', truncation=True, max_length=128, return_attributions=False, return_tensors='pt')\n",
        "        label = torch.tensor(row['label'])    # tensor\n",
        "        return {'input': encoding, 'label': label}\n",
        "\n",
        ""
      ],
      "metadata": {
        "id": "fWjjUgyIalfB"
      },
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def prepare_data(filepath):\n",
        "    df = pd.read_csv(filepath)\n",
        "    X_train, X_val, y_train, y_val = train_test_split(df['text'].values, df['label'].values, test_size=0.2, random_state=42, stratify=df['label'])\n",
        "    # splitting data into x_train,x_test,y_train,y_test\n",
        "    return {'X_train': X_train, 'y_train': y_train, 'X_val': X_val, 'y_val': y_val}\n",
        "\n",
        ""
      ],
      "metadata": {
        "id": "NrL1mfICarWO"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "\n",
        "\n",
        "```\n",
        "TRAINING AND PREDICTING\n",
        "```\n",
        "\n"
      ],
      "metadata": {
        "id": "3WG_6QcefkRB"
      }
    },
    {
      "cell_type": "markdown",
      "source": [
        "# tqdm and learning_rate_schedular"
      ],
      "metadata": {
        "id": "ecO-7_M8hJvl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "\n",
        "def main(filepath):\n",
        "    raw_data = prepare_data(filepath)\n",
        "    train_data = MyDataset(pd.DataFrame({'text': raw_data['X_train'], 'label': list(raw_data['y_train'])}))\n",
        "    val_data = MyDataset(pd.DataFrame({'text': raw_data['X_val'], 'label': list(raw_data['y_val'])}))\n",
        "\n",
        "    train_loader = DataLoader(train_data, batch_size=32, shuffle=True)\n",
        "    val_loader = DataLoader(val_data, batch_size=32, shuffle=False)\n",
        "\n",
        "    model = BertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "    device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
        "    model.to(device)\n",
        "\n",
        "    criterion = torch.nn.CrossEntropyLoss()\n",
        "    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)\n",
        "\n",
        "    num_epochs = 3\n",
        "    total_steps = len(train_loader) * num_epochs\n",
        "\n",
        "    lr_scheduler = torch.optim.lr_scheduler.OneCycleLR(optimizer, max_lr=1e-5, steps_per_epoch=total_steps, pct_start=0.3, cycle_momentum=True, anneal_strategy='cos', div_factor=25.0, final_div_factor=10000.0, last_epoch=-1, verbose=True)\n",
        "\n",
        "    global_step = 0\n",
        "\n",
        "    for epoch in range(num_epochs):\n",
        "        print(f\"\\n\\nEpoch {epoch + 1}/{num_epochs}\\n-------------------------------\")\n",
        "        for _, batch in enumerate(train_loader):\n",
        "            input_ids = batch['input']['input_ids'].to(device)\n",
        "            attention_mask = batch['input']['attention_mask'].to(device)\n",
        "            labels = batch['label'].to(device)\n",
        "\n",
        "            optimizer.zero_grad()\n",
        "            outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
        "            loss = outputs.loss\n",
        "            loss.backward()\n",
        "            optimizer.step()\n",
        "\n",
        "            global_step += 1\n",
        "            if global_step % 10 == 0:\n",
        "                print(f\"Global Step: {global_step}, Loss: {loss.item()}\")\n",
        "\n",
        "        evaluations_accuracies = []\n",
        "\n",
        "        with torch.no_grad():\n",
        "            for batch in val_loader:\n",
        "                input_ids = batch['input']['input_ids'].to(device)\n",
        "                attention_mask = batch['input']['attention_mask'].to(device)\n",
        "                labels = batch['label'].to(device)\n",
        "\n",
        "                outputs = model(input_ids, attention_mask=attention_mask, labels=labels)\n",
        "                probs = torch.nn.functional.softmax(outputs.logits, dim=1)\n",
        "                pred = torch.argmax(probs, dim=1)\n",
        "\n",
        "                evaluations_accuracies.append(accuracy_score(labels.cpu().detach().numpy(), pred.cpu().detach().numpy()))\n",
        "\n",
        "        avg_evaluation_accuracy = sum(evaluations_accuracies)/len(evaluations_accuracies)\n",
        "        print(f\"Validation Average Accuracy: {avg_evaluation_accuracy}\")\n",
        "\n",
        "        lr_scheduler.step()\n",
        "\n",
        "if __name__ == \"__main__\":\n",
        "    # main('/path/to/custom/dataset.csv')\n",
        "    pass"
      ],
      "metadata": {
        "id": "yfCxsvHyTOhl"
      },
      "execution_count": 28,
      "outputs": []
    }
  ]
}