Study.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Convolution:\n",
    "    def __init__(self, W, b, stride=1, pad=0):\n",
    "        self.W = W\n",
    "        self.b = b\n",
    "        self.stride = stride\n",
    "        self.pad = pad\n",
    "    \n",
    "    def forward(self, x):\n",
    "        FN, C, FH, FW = self.W.shape\n",
    "        N, C, H, W = x.shape\n",
    "        out_h = int(1 + (H + 2*self.pad - FH) / self.stride)\n",
    "        out_w = int(1 + (W + 2*self.pad - FW) / self.stride)\n",
    "        \n",
    "        col = im2col(x, FH, FW, self.stride, self.pad)\n",
    "        col_W = self.W.reshape(Fn, -1).T # 필터 전개\n",
    "        out = np.dot(col, col_W) + self.b\n",
    "        \n",
    "        out = out.reshape(N, out_h, out_w, -1).transpose(0, 3, 1, 2)\n",
    "        \n",
    "        return out\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class Pooling:\n",
    "    def __init__(self, pool_h, pool_w, stride=1, pad=0):\n",
    "        self.pool_h = pool_h\n",
    "        self.pool_w = pool_w\n",
    "        self.stride = stride\n",
    "        self.pad = pad\n",
    "        \n",
    "    def forward(self, x):\n",
    "        N, C, H, W = x.shape\n",
    "        out_h = int(1 + (H - self.pool_h) / self.stride)\n",
    "        out_w = int(1 + (W - self.pool_w) / self.stride)\n",
    "        \n",
    "        # 전개 (1)\n",
    "        col = im2col(x, self.pool_h, self.pool_w, self.stride, self.pad)\n",
    "        col = col.reshape(-1, self.pool_h*self.pool_w)\n",
    "        \n",
    "        # 최댓값 (2)\n",
    "        out = np.max(col, axis=1)\n",
    "        \n",
    "        # 성형 (3)\n",
    "        out = out.reshape(N, out_h, out_w, C).transpose(0, 3, 1, 2)\n",
    "        \n",
    "        return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def preprocess(text):\n",
    "    text = text.lower()\n",
    "    text = text.replace('.', ' .')\n",
    "    \n",
    "    words = text.split(' ')\n",
    "    \n",
    "    word_to_id = {}\n",
    "    id_to_word = {}\n",
    "\n",
    "    for word in words:\n",
    "        if word not in word_to_id:\n",
    "            new_id = len(word_to_id)\n",
    "            word_to_id[word] = new_id\n",
    "            id_to_word[new_id] = word\n",
    "    \n",
    "    corpus = [word_to_id[w] for w in words]\n",
    "    corpus = np.array(corpus)\n",
    "    \n",
    "    return corpus, word_to_id, id_to_word"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_co_matrix(corpus, vocab_size, window_size=1):\n",
    "    corpus_size = len(corpus)\n",
    "    co_matrix = np.zeros((vocab_size, vocab_size), dtype=np.int32)\n",
    "    \n",
    "    for idx, word_id in enumerate(corpus):\n",
    "        for i in range(1, window_size + 1):\n",
    "            left_idx = idx - i\n",
    "            right_idx = idx + i\n",
    "            \n",
    "            if left_idx >= 0:\n",
    "                left_word_id = corpus[left_idx]\n",
    "                co_matrix[word_id, left_word_id] += 1\n",
    "                \n",
    "            if right_idx < corpus_size:\n",
    "                right_word_id = corpus[right_idx]\n",
    "                co_matrix[word_id, right_word_id] += 1\n",
    "                \n",
    "    return co_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cos_similarity(x, y, eps=1e-8):\n",
    "    nx = x / (np.sqrt(np.sum(x**2)) + eps) # x의 정규화\n",
    "    ny = y / (np.sqrt(np.sum(y**2)) + eps)# y의 정규화\n",
    "    return np.dot(nx, ny)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def ppmi(C, verbose=False, eps=1e-8):\n",
    "    M = np.zeros_like(C, dtype=np.float32)\n",
    "    N = np.sum(C)\n",
    "    S = np.sum(C, axis=0)\n",
    "    total = C.shape[0] * C.shape[1]\n",
    "    cnt = 0\n",
    "    \n",
    "    for i in range(C.shape[0]):\n",
    "        for j in range(C.shape[1]):\n",
    "            pmi = np.log2(C[i, j] * N / (S[j] * S[i] + eps))\n",
    "            M[i, j] = max(0, pmi)\n",
    "            \n",
    "            if verbose:\n",
    "                cnt += 1\n",
    "                if cnt % (total//100) == 0:\n",
    "                    print('%.1f%% 완료' % (100*cnt/total))\n",
    "                    \n",
    "    return M"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_contexts_target(corpus, window_size=1):\n",
    "    target = corpus[window_size:-window_size]\n",
    "    contexts = []\n",
    "    \n",
    "    for idx in range(window_size, len(corpus)-window_size):\n",
    "        cs = []\n",
    "        for t in range(-window_size, window_size + 1):\n",
    "            if t == 0:\n",
    "                continue\n",
    "            cs.append(corpus[idx + t])\n",
    "        contexts.append(cs)\n",
    "    \n",
    "    return np.array(contexts), np.array(target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCBOW:\n",
    "    def __init__(self, vocab_size, hidden_size):\n",
    "        V, H = vocab_size, hidden_size\n",
    "        \n",
    "        # 가중치 초기화\n",
    "        W_in = 0.01 * np.random.randn(V, H).astype('f')\n",
    "        W_out = 0.01 * np.random.randn(H, V).astype('f')\n",
    "        \n",
    "        # 계층 생성\n",
    "        self.in_layer0 = MatMul(W_in)\n",
    "        self.in_layer1 = MatMul(W_in)\n",
    "        self.out_layer = MatMul(W_out)\n",
    "        self.loss_layer = SoftmaxWithLoss()\n",
    "        \n",
    "        # 모등 가중치와 기울기를 리스트에 모은다.\n",
    "        layers = [self.in_layer0, self.in_layer1, self.out_layer]\n",
    "        self.params, self.grads = [], []\n",
    "        for layer in layers:\n",
    "            self.params += layer.params\n",
    "            self.grads += layer.grads\n",
    "        \n",
    "        #인스턴스 변수에 단어의 분산 표현을 저장한다.\n",
    "        self.word_vecs = W_in\n",
    "        \n",
    "    \n",
    "    def forward(self, contexts, target):\n",
    "        h0 = self.in_layer0.forward(contexts[:, 0])\n",
    "        h1 = self.in_layer1.forward(contexts[:, 1])\n",
    "        h = (h0 + h1) * 0.5\n",
    "        score = self.out_layer.forward(h)\n",
    "        loss = self.loss_layer.forward(score, target)\n",
    "        return loss\n",
    "    \n",
    "    \n",
    "    def backward(self, dout=1):\n",
    "        ds = self.loss_layer.backward(dout)\n",
    "        da = self.out_layer.backward(ds)\n",
    "        da *= 0.5\n",
    "        self.in_layer1.backward(da)\n",
    "        self.in_layer0.backward(da)\n",
    "        return None"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}