HF_DATA_TO_PYTORCH_CUSTOM_DATA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNsITPPcdlqgSbKTIQ/oAQ8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Shakilkhan24/Playground_DL/blob/main/HF_DATA_TO_PYTORCH_CUSTOM_DATA.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from datasets import load_dataset\n",
        "from transformers import DistilBertTokenizer, DistilBertForSequenceClassification\n",
        "from torch.utils.data import DataLoader, Dataset\n",
        "\n",
        "dataset = load_dataset(\"glue\", \"mrpc\")"
      ],
      "metadata": {
        "id": "bf0SFUemxviR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "oDCDeGWWxhB4"
      },
      "outputs": [],
      "source": [
        "\n",
        "def tokenize_function(example):\n",
        "    return tokenizer(example[\"sentence1\"], example[\"sentence2\"], truncation=True)\n",
        "\n",
        "tokenized_dataset = dataset.map(tokenize_function, batched=True)\n",
        "\n",
        "# Convert tokenized dataset to PyTorch Dataset\n",
        "class TorchDataset(Dataset):\n",
        "    def __init__(self, examples):\n",
        "        self.examples = examples\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.examples[\"input_ids\"])\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        return {\n",
        "            \"input_ids\": self.examples[\"input_ids\"][idx],\n",
        "            \"attention_mask\": self.examples[\"attention_mask\"][idx],\n",
        "            \"labels\": self.examples[\"label\"][idx]\n",
        "        }\n",
        "\n",
        "train_dataset = TorchDataset(tokenized_dataset[\"train\"])\n",
        "eval_dataset = TorchDataset(tokenized_dataset[\"validation\"])\n",
        "\n",
        "train_dataloader = DataLoader(train_dataset, batch_size=8, shuffle=True)\n",
        "eval_dataloader = DataLoader(eval_dataset, batch_size=8)\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import torch\n",
        "from datasets import load_dataset\n",
        "from torchvision import transforms\n",
        "from PIL import Image\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "class CustomImageDataset(Dataset):\n",
        "    def __init__(self, dataset_name, split_name, transform=None):\n",
        "        self.dataset = load_dataset(dataset_name, split=split_name)\n",
        "        self.transform = transform\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.dataset)\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        image_path = self.dataset[idx]['image_file_path']\n",
        "        image = Image.open(image_path)\n",
        "        label = self.dataset[idx]['label']\n",
        "\n",
        "        if self.transform:\n",
        "            image = self.transform(image)\n",
        "\n",
        "        return image, label\n",
        "\n",
        "# Define transforms for preprocessing\n",
        "transform = transforms.Compose([\n",
        "    transforms.Resize((224, 224)),  # Resize the image to 224x224\n",
        "    transforms.ToTensor(),           # Convert PIL Image to tensor\n",
        "    transforms.Normalize(            # Normalize image tensor\n",
        "      mean=[0.485, 0.456, 0.406],   # Mean values for RGB channels\n",
        "      std=[0.229, 0.224, 0.225]     # Standard deviations for RGB channels\n",
        "    )\n",
        "])\n",
        "\n",
        "# Create custom dataset instance\n",
        "dataset = CustomImageDataset(dataset_name=\"beans\", split_name=\"train\", transform=transform)\n",
        "\n",
        "# Example usage\n",
        "# Accessing a sample image and label\n",
        "sample_image, sample_label = dataset[0]\n",
        "print(\"Sample image shape:\", sample_image.shape)\n",
        "print(\"Sample label:\", sample_label)"
      ],
      "metadata": {
        "id": "SItwFt86x530"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}