TryingThings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"dataset.txt\") as f:\n",
    "    text = f.read()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1115394"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      " !$&',-.3:;?ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz\n",
      "65\n"
     ]
    }
   ],
   "source": [
    "chars = sorted(list(set(text)))\n",
    "vocab_size = len(chars)\n",
    "print(\"\".join(chars))\n",
    "print(vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[46, 43, 50, 50, 53]\n",
      "hello\n"
     ]
    }
   ],
   "source": [
    "stoi = { ch:i for i, ch in enumerate(chars) }\n",
    "itos = { i:ch for i, ch in enumerate(chars) }\n",
    "encode = lambda s: [stoi[ch] for ch in s]\n",
    "decode = lambda l: \"\".join([itos[i] for i in l])\n",
    "print(encode(\"hello\"))\n",
    "print(decode(encode(\"hello\")))\n",
    "\n",
    "#Sentence Piece, tiktoken"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([1115394]) torch.int64\n",
      "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58, 47])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "data = torch.tensor(encode(text), dtype=torch.long)\n",
    "print(data.shape, data.dtype)\n",
    "print(data[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "n = int(0.9*len(data))\n",
    "train_data = data[:n]\n",
    "val_data = data[n:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([18, 47, 56, 57, 58,  1, 15, 47, 58])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "block_size = 8\n",
    "train_data[:block_size+1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input is tensor([18]) and target is 47\n",
      "input is tensor([18, 47]) and target is 56\n",
      "input is tensor([18, 47, 56]) and target is 57\n",
      "input is tensor([18, 47, 56, 57]) and target is 58\n",
      "input is tensor([18, 47, 56, 57, 58]) and target is 1\n",
      "input is tensor([18, 47, 56, 57, 58,  1]) and target is 15\n",
      "input is tensor([18, 47, 56, 57, 58,  1, 15]) and target is 47\n",
      "input is tensor([18, 47, 56, 57, 58,  1, 15, 47]) and target is 58\n"
     ]
    }
   ],
   "source": [
    "x = train_data[:block_size]\n",
    "y = train_data[1:block_size+1]\n",
    "for t in range(block_size):\n",
    "    context = x[:t+1]\n",
    "    target = y[t]\n",
    "    print(f\"input is {context} and target is {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input:\n",
      "torch.Size([4, 8])\n",
      "tensor([[24, 43, 58,  5, 57,  1, 46, 43],\n",
      "        [44, 53, 56,  1, 58, 46, 39, 58],\n",
      "        [52, 58,  1, 58, 46, 39, 58,  1],\n",
      "        [25, 17, 27, 10,  0, 21,  1, 54]])\n",
      "target:\n",
      "torch.Size([4, 8])\n",
      "tensor([[43, 58,  5, 57,  1, 46, 43, 39],\n",
      "        [53, 56,  1, 58, 46, 39, 58,  1],\n",
      "        [58,  1, 58, 46, 39, 58,  1, 46],\n",
      "        [17, 27, 10,  0, 21,  1, 54, 39]])\n",
      "------\n",
      "input is tensor([24]) and target is 43\n",
      "input is tensor([24, 43]) and target is 58\n",
      "input is tensor([24, 43, 58]) and target is 5\n",
      "input is tensor([24, 43, 58,  5]) and target is 57\n",
      "input is tensor([24, 43, 58,  5, 57]) and target is 1\n",
      "input is tensor([24, 43, 58,  5, 57,  1]) and target is 46\n",
      "input is tensor([24, 43, 58,  5, 57,  1, 46]) and target is 43\n",
      "input is tensor([24, 43, 58,  5, 57,  1, 46, 43]) and target is 39\n",
      "input is tensor([44]) and target is 53\n",
      "input is tensor([44, 53]) and target is 56\n",
      "input is tensor([44, 53, 56]) and target is 1\n",
      "input is tensor([44, 53, 56,  1]) and target is 58\n",
      "input is tensor([44, 53, 56,  1, 58]) and target is 46\n",
      "input is tensor([44, 53, 56,  1, 58, 46]) and target is 39\n",
      "input is tensor([44, 53, 56,  1, 58, 46, 39]) and target is 58\n",
      "input is tensor([44, 53, 56,  1, 58, 46, 39, 58]) and target is 1\n",
      "input is tensor([52]) and target is 58\n",
      "input is tensor([52, 58]) and target is 1\n",
      "input is tensor([52, 58,  1]) and target is 58\n",
      "input is tensor([52, 58,  1, 58]) and target is 46\n",
      "input is tensor([52, 58,  1, 58, 46]) and target is 39\n",
      "input is tensor([52, 58,  1, 58, 46, 39]) and target is 58\n",
      "input is tensor([52, 58,  1, 58, 46, 39, 58]) and target is 1\n",
      "input is tensor([52, 58,  1, 58, 46, 39, 58,  1]) and target is 46\n",
      "input is tensor([25]) and target is 17\n",
      "input is tensor([25, 17]) and target is 27\n",
      "input is tensor([25, 17, 27]) and target is 10\n",
      "input is tensor([25, 17, 27, 10]) and target is 0\n",
      "input is tensor([25, 17, 27, 10,  0]) and target is 21\n",
      "input is tensor([25, 17, 27, 10,  0, 21]) and target is 1\n",
      "input is tensor([25, 17, 27, 10,  0, 21,  1]) and target is 54\n",
      "input is tensor([25, 17, 27, 10,  0, 21,  1, 54]) and target is 39\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(1337)\n",
    "batch_size = 4\n",
    "block_size = 8  \n",
    "\n",
    "def get_batch(split):\n",
    "    data = train_data if split == \"train\" else val_data\n",
    "    ix = torch.randint(len(data) - block_size, (batch_size,))\n",
    "    x = torch.stack([data[i:i+block_size] for i in ix])\n",
    "    y = torch.stack([data[i+1:i+block_size+1] for i in ix])\n",
    "    return x, y\n",
    "\n",
    "xb, yb = get_batch(\"train\")\n",
    "print('input:')\n",
    "print(xb.shape)\n",
    "print(xb)\n",
    "print('target:')\n",
    "print(yb.shape)\n",
    "print(yb)\n",
    "\n",
    "print(\"------\")\n",
    "\n",
    "for b in range(batch_size):\n",
    "    for t in range(block_size):\n",
    "        context = xb[b, :t+1]\n",
    "        target = yb[b, t]\n",
    "        print(f\"input is {context} and target is {target}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([32, 65])\n",
      "tensor(4.8786, grad_fn=<NllLossBackward0>)\n",
      "\n",
      "Sr?qP-QWktXoL&jLDJgOLVz'RIoDqHdhsV&vLLxatjscMpwLERSPyao.qfzs$Ys$zF-w,;eEkzxjgCKFChs!iWW.ObzDnxA Ms$3\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "torch.manual_seed(1337)\n",
    "\n",
    "class Bigram(nn.Module):\n",
    "    def __init__(self, vocab_size):\n",
    "        super().__init__()\n",
    "        self.token_embedding_table = nn.Embedding(vocab_size, vocab_size)\n",
    "    \n",
    "    def forward(self, idx, targets=None):\n",
    "        logits = self.token_embedding_table(idx) # [B,T,C]\n",
    "        if targets is None:\n",
    "            loss = None\n",
    "        else:\n",
    "            B,T,C = logits.shape\n",
    "            logits = logits.view(B*T,C)\n",
    "            # targets = targets.view(-1) same as below\n",
    "            targets = targets.view(B*T)\n",
    "            loss = F.cross_entropy(logits, targets)\n",
    "        return logits, loss\n",
    "\n",
    "    def generate(self, idx, max_new_tokens):\n",
    "        # idx is [B,T] \n",
    "        for _ in range(max_new_tokens):\n",
    "            # get the prediction\n",
    "            logits, loss = self(idx)\n",
    "            # focus on the last time step\n",
    "            logits = logits[:,-1,:] # becmoes [B,C]\n",
    "            probs = F.softmax(logits, dim=-1)\n",
    "            # sample from the distribution\n",
    "            idx_next = torch.multinomial(probs, num_samples=1) # [B,1]\n",
    "            idx = torch.cat((idx, idx_next), dim=1)  # [B,T+1]\n",
    "        return idx\n",
    "        \n",
    "\n",
    "m = Bigram(vocab_size)\n",
    "# logits, loss = m(xb, yb) #this gives because pytorch needs B/C/T NOT B/T/C\n",
    "logits, loss = m(xb, yb)\n",
    "print(logits.shape)\n",
    "print(loss) #-ln(1/65) = 4.17\n",
    "\n",
    "print(decode(m.generate(idx= torch.zeros((1,1), dtype=torch.long), max_new_tokens=100)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Visual Representation:\n",
    "Imagine you have a vocabulary of 5 words, and you want each word to be represented by a vector of size 3.\n",
    "\n",
    "Vocabulary: [\"apple\", \"banana\", \"cherry\", \"date\", \"elderberry\"]\n",
    "Indices: [0, 1, 2, 3, 4]\n",
    "Embedding Size: 3\n",
    "The nn.Embedding layer will create a matrix of size (vocab_size, n_embd), which in this case is (5, 3).'''\n",
    "'''\n",
    "Embedding Matrix (5 x 3):\n",
    "+---------+---------+---------+\n",
    "| 0.25    | -0.10   | 0.75    |  <- \"apple\" (index 0)\n",
    "| -0.50   | 0.60    | -0.30   |  <- \"banana\" (index 1)\n",
    "| 0.40    | -0.20   | 0.10    |  <- \"cherry\" (index 2)\n",
    "| -0.70   | 0.80    | -0.40   |  <- \"date\" (index 3)\n",
    "| 0.55    | -0.35   | 0.65    |  <- \"elderberry\" (index 4)\n",
    "+---------+---------+---------+\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "optimser = torch.optim.Adam(m.parameters(), lr=1e-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2.6783480644226074\n"
     ]
    }
   ],
   "source": [
    "batch_size = 32\n",
    "for steps in range(5000):\n",
    "    xb, yb = get_batch(\"train\")\n",
    "    logits, loss = m(xb, yb)\n",
    "    optimser.zero_grad()\n",
    "    loss.backward()\n",
    "    optimser.step()\n",
    "\n",
    "print(loss.item())  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Mave ghtang d ws amangT:\n",
      "D: nd TINT:\n",
      "Fof;AURKIf diTus, be:ghere\n",
      "Dyo.O'd, t fed inks cerVis benes. owepwnof pre, athar wowir W:\n",
      "ALI u\n",
      "INCAbOgfatho rend thic; is be wara!\n",
      "PO.\n",
      "WGListh R.\n",
      "JaveauCE! ce, ince:'\n",
      "t.\n",
      "Sin honk$zlerseestindovrer wat boue nodgh flle, mOM:\n",
      "\n",
      "SCKpoto AKIFwinthind d me sesete gkerw'DWhe arDERe\n",
      "\n",
      "HET:\n",
      "ROLINanxueile to he bis wllagGomieg.\n",
      "DWhelat s tropghallll,\n",
      "Wh trnenQJMmllel cLENzt iBOFokKENor-mu ed de atos are th, a!OFot I s l:\n",
      "E-de pond\n",
      "TII'henJAn?Bof bogave tthe s me : t\n",
      "Yo-\n"
     ]
    }
   ],
   "source": [
    "print(decode(m.generate(idx= torch.zeros((1,1), dtype=torch.long), max_new_tokens=500)[0].tolist()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cpu')"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Maths trick in self attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 8, 2])"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(1337)\n",
    "B, T, C = 4, 8, 2\n",
    "x = torch.randn(B,T,C)\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we want x[b,t] = mean_{i<=t} x[b,i]\n",
    "xbow = torch.zeros((B,T,C)) # xbow is x bag of words\n",
    "for b in range(B):\n",
    "    for t in range(T):\n",
    "        xprev = x[b,:t+1] # [t,C]\n",
    "        xbow[b,t] = torch.mean(xprev, 0)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.1808, -0.0700],\n",
       "        [-0.3596, -0.9152],\n",
       "        [ 0.6258,  0.0255],\n",
       "        [ 0.9545,  0.0643],\n",
       "        [ 0.3612,  1.1679],\n",
       "        [-1.3499, -0.5102],\n",
       "        [ 0.2360, -0.2398],\n",
       "        [-0.9211,  1.5433]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[ 0.1808, -0.0700],\n",
       "        [-0.0894, -0.4926],\n",
       "        [ 0.1490, -0.3199],\n",
       "        [ 0.3504, -0.2238],\n",
       "        [ 0.3525,  0.0545],\n",
       "        [ 0.0688, -0.0396],\n",
       "        [ 0.0927, -0.0682],\n",
       "        [-0.0341,  0.1332]])"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "xbow[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## above all are inequicent"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "a: tensor([[1.0000, 0.0000, 0.0000],\n",
      "        [0.5000, 0.5000, 0.0000],\n",
      "        [0.3333, 0.3333, 0.3333]])\n",
      "b: tensor([[2., 7.],\n",
      "        [6., 4.],\n",
      "        [6., 5.]])\n",
      "c: tensor([[2.0000, 7.0000],\n",
      "        [4.0000, 5.5000],\n",
      "        [4.6667, 5.3333]])\n"
     ]
    }
   ],
   "source": [
    "torch.manual_seed(42)\n",
    "a = torch.tril(torch.ones(3,3))\n",
    "a = a/ torch.sum(a, dim=1, keepdim=True)\n",
    "b = torch.randint(0,10,(3,2)).float()\n",
    "c = a @ b\n",
    "print(\"a:\", a)\n",
    "print(\"b:\", b)\n",
    "print(\"c:\", c)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Coontine"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## version was without matrix\n",
    "## version2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wei = torch.tril(torch.ones(T, T))\n",
    "wei = wei / wei.sum(dim=1, keepdim=True)\n",
    "xbow2 = wei @ x # (T, T) @ (B, T, C) ---- torch makes it (B, T, T) @ (B, T, C) = (B, T, C)\n",
    "torch.allclose(xbow, xbow2) # should give True maybe some update in torch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## version 3\n",
    "run one by one if u dont understand\n",
    "\n",
    "## We will use this"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "False"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tril = torch.tril(torch.ones(T,T))\n",
    "wei = torch.zeros(T,T) #interaction strength\n",
    "wei = wei.masked_fill(tril==0, float('-inf')) #tokens form past cant commmnunicate(basically future cant communicate with past)\n",
    "wei = F.softmax(wei, dim=1)\n",
    "xbow3 = wei @ x # aggregation\n",
    "torch.allclose(xbow, xbow3) #again "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## version 4: Self attention\n",
    "v3 has uniform numbers because different tokens find other interesting or not so to make it data dependent we will use self attention"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([4, 8, 16])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch.manual_seed(1337)\n",
    "B, T, C = 4, 8, 32\n",
    "x = torch.randn(B,T,C)\n",
    "\n",
    "head_size = 16  \n",
    "key = nn.Linear(C, head_size, bias=False) # what do i contain\n",
    "query = nn.Linear(C, head_size, bias=False) # what am i looking for\n",
    "value = nn.Linear(C, head_size, bias=False) # what do i want to know\n",
    "k = key(x) # [B,T,16]\n",
    "q = query(x) # [B,T,16]\n",
    "wei = q @ k.transpose(-2, -1) # [B,T,16] @ [B,16,T] = [B,T,T]\n",
    "\n",
    "\n",
    "tril = torch.tril(torch.ones(T,T))\n",
    "# wei = torch.zeros(T,T)\n",
    "wei = wei.masked_fill(tril==0, float('-inf')) # if u remove this line they will completefy communicate but if we keep it will communicate only in one direction i.e. past to future Ex: Sentimental analysis it needs future also\n",
    "wei = F.softmax(wei, dim=-1)\n",
    "\n",
    "v = value(x) # [B,T,16]\n",
    "out = wei @ v # [B,T,T] @ [B,T,16] = [B,T,16]\n",
    "\n",
    "\n",
    "# out = wei @ x\n",
    "out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[1.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.1574, 0.8426, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.2088, 0.1646, 0.6266, 0.0000, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.5792, 0.1187, 0.1889, 0.1131, 0.0000, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0294, 0.1052, 0.0469, 0.0276, 0.7909, 0.0000, 0.0000, 0.0000],\n",
       "        [0.0176, 0.2689, 0.0215, 0.0089, 0.6812, 0.0019, 0.0000, 0.0000],\n",
       "        [0.1691, 0.4066, 0.0438, 0.0416, 0.1048, 0.2012, 0.0329, 0.0000],\n",
       "        [0.0210, 0.0843, 0.0555, 0.2297, 0.0573, 0.0709, 0.2423, 0.2391]],\n",
       "       grad_fn=<SelectBackward0>)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wei[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Notes:\n",
    "- Attention is a **communication mechanism**. Can be seen as nodes in a directed graph looking at each other and aggregating information with a weighted sum from all nodes that point to them, with data-dependent weights.\n",
    "- There is no notion of space. Attention simply acts over a set of vectors. This is why we need to positionally encode tokens.\n",
    "- Each example across batch dimension is of course processed completely independently and never \"talk\" to each other\n",
    "- In an \"encoder\" attention block just delete the single line that does masking with `tril`, allowing all tokens to communicate. This block here is called a \"decoder\" attention block because it has triangular masking, and is usually used in autoregressive settings, like language modeling.\n",
    "- \"self-attention\" just means that the keys and values are produced from the same source as queries. In \"cross-attention\", the queries still get produced from x, but the keys and values come from some other, external source (e.g. an encoder module)\n",
    "- \"Scaled\" attention additional divides `wei` by 1/sqrt(head_size). This makes it so when input Q,K are unit variance, wei will be unit variance too and Softmax will stay diffuse and not saturate too much. Illustration below"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "k = torch.randn(B, T, head_size)\n",
    "q = torch.randn(B, T, head_size)\n",
    "wei = q @ k.transpose(-2,-1) * head_size**-0.5 #scaled attention(this is done to preserve variance which is close to 1 shown below)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor(1.0918)"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "k.var() \n",
    "q.var()\n",
    "wei.var() #before scaling it was 17.46 which is way greater than head_size = 16"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[[0.0900, 0.2447, 0.6652],\n",
      "         [0.0900, 0.2447, 0.6652],\n",
      "         [0.0900, 0.2447, 0.6652]],\n",
      "\n",
      "        [[0.0900, 0.2447, 0.6652],\n",
      "         [0.0900, 0.2447, 0.6652],\n",
      "         [0.0900, 0.2447, 0.6652]]])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import torch.nn.functional as F\n",
    "\n",
    "wei = torch.tensor([[[1.0, 2.0, 3.0],\n",
    "                     [1.0, 2.0, 3.0],\n",
    "                     [1.0, 2.0, 3.0]],\n",
    "\n",
    "                    [[1.0, 2.0, 3.0],\n",
    "                     [1.0, 2.0, 3.0],\n",
    "                     [1.0, 2.0, 3.0]]])\n",
    "\n",
    "softmax_wei = F.softmax(wei, dim=-1)\n",
    "print(softmax_wei)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## check batch norm code in make more and see layer norm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}