1_1_1-全连接神经网络做线性回归.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 一、定义前向、后向传播\n",
    "\n",
    "本文将用numpy实现全连接层的前向过程和反向过程，并使用一个线性回归作为例子进行测试；\n",
    "\n",
    "如果对神经网络的反向传播过程还有不清楚的，可以[0_1-全连接层、损失函数的反向传播](0_1-全连接层、损失函数的反向传播.md)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "def fc_forword(z, W, b):\n",
    "    \"\"\"\n",
    "    全连接层的前向传播\n",
    "    :param z: 当前层的输出\n",
    "    :param W: 当前层的权重\n",
    "    :param b: 当前层的偏置\n",
    "    :return: 下一层的输出\n",
    "    \"\"\"\n",
    "    return np.dot(z, W) + b\n",
    "\n",
    "\n",
    "def fc_backword(next_dz, W, z):\n",
    "    \"\"\"\n",
    "    全连接层的反向传播\n",
    "    :param next_dz: 下一层的梯度\n",
    "    :param W: 当前层的权重\n",
    "    :param z: 当前层的输出\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    N = z.shape[0]\n",
    "    dz = np.dot(next_dz, W.T)  # 当前层的梯度\n",
    "    dw = np.dot(z.T, next_dz)  # 当前层权重的梯度\n",
    "    db = np.sum(next_dz,axis=0)  # 当前层偏置的梯度, N个样本的梯度求和\n",
    "    return dw/N, db/N, dz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 二、定义损失函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def mean_squared_loss(y_predict,y_true):\n",
    "    \"\"\"\n",
    "    均方误差损失函数\n",
    "    :param y_predict: 预测值,shape (N,d)，N为批量样本数\n",
    "    :param y_true: 真实值\n",
    "    :return:\n",
    "    \"\"\"\n",
    "    loss = np.mean(np.sum(np.square(y_predict-y_true),axis=-1))  # 损失函数值\n",
    "    dy = y_predict - y_true  # 损失函数关于网络输出的梯度\n",
    "    return loss, dy"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 三、初始化数据"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "x.shape:(500, 2),y.shape:(500, 3)\n"
     ]
    }
   ],
   "source": [
    "# 实际的权重和偏置\n",
    "W = np.array([[3,7,4],\n",
    "              [5,2,6]])\n",
    "b = np.array([2,9,3])\n",
    "\n",
    "# 产生训练样本\n",
    "x_data = np.random.randint(0,10,1000).reshape(500,2)\n",
    "y_data = np.dot(x_data,W)+b\n",
    "\n",
    "def next_sample(batch_size=1):\n",
    "    idx=np.random.randint(500)\n",
    "    return x_data[idx:idx+batch_size],y_data[idx:idx+batch_size]\n",
    "\n",
    "print(\"x.shape:{},y.shape:{}\".format(x_data.shape,y_data.shape))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 四、定义网络、使用SGD训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "迭代1000次，当前loss:0.43387298848896233, 当前权重:[[3.01734672 7.12785625 4.02756123]\n",
      " [5.0221794  2.16347613 6.0352396 ]],当前偏置[1.81757802 7.65543542 2.71016   ]\n",
      "\n",
      "迭代2000次，当前loss:0.024775748245913158, 当前权重:[[3.00242166 7.01784918 4.00384764]\n",
      " [5.00295757 2.02179914 6.00469912]],当前偏置[1.96775495 8.76233376 2.94876766]\n",
      "\n",
      "迭代3000次，当前loss:0.00014564406568725818, 当前权重:[[3.00082136 7.00605396 4.00130502]\n",
      " [5.00061563 2.00453758 6.00097814]],当前偏置[1.99381124 8.95438495 2.99016703]\n",
      "\n",
      "迭代4000次，当前loss:2.6237167410353415e-05, 当前权重:[[3.0001119  7.00082475 4.00017779]\n",
      " [5.00008191 2.0006037  6.00013014]],当前偏置[1.99885749 8.99157899 2.99818473]\n",
      "\n",
      "迭代5000次，当前loss:3.713805657221762e-07, 当前权重:[[3.00002322 7.00017112 4.00003689]\n",
      " [5.00001109 2.00008176 6.00001763]],当前偏置[1.99979785 8.99851001 2.99967881]\n",
      "\n",
      "迭代6000次，当前loss:8.807646869757514e-09, 当前权重:[[3.0000031  7.00002283 4.00000492]\n",
      " [5.00000397 2.00002927 6.00000631]],当前偏置[1.99996212 8.9997208  2.99993981]\n",
      "\n",
      "迭代7000次，当前loss:1.536245925844849e-10, 当前权重:[[3.00000073 7.00000539 4.00000116]\n",
      " [5.00000067 2.00000494 6.00000106]],当前偏置[1.99999324 8.99995017 2.99998926]\n",
      "\n",
      "迭代7398次，当前loss:3.3297294256090265e-16, 当前权重:[[3.00000043 7.00000318 4.00000069]\n",
      " [5.0000004  2.00000294 6.00000063]],当前偏置[1.99999655 8.99997456 2.99999452]\n"
     ]
    }
   ],
   "source": [
    "# 随机初始化参数\n",
    "W1 = np.random.randn(2,3)\n",
    "b1 = np.zeros([3])\n",
    "loss = 100.0\n",
    "lr = 0.01\n",
    "i = 0 \n",
    "\n",
    "while loss > 1e-15:\n",
    "    x,y_true=next_sample(2)  # 获取当前样本\n",
    "    # 前向传播\n",
    "    y = fc_forword(x,W1,b1)\n",
    "    # 反向传播更新梯度\n",
    "    loss,dy=mean_squared_loss(y,y_true)\n",
    "    dw,db,_ = fc_backword(dy,W,x)\n",
    "    \n",
    "    # 在一个batch中梯度取均值\n",
    "    #print(dw)\n",
    "    \n",
    "    # 更新梯度\n",
    "    W1 -= lr*dw\n",
    "    b1 -= lr*db\n",
    "    \n",
    "    # 更新迭代次数\n",
    "    i += 1\n",
    "    if i % 1000 == 0:\n",
    "        print(\"\\n迭代{}次，当前loss:{}, 当前权重:{},当前偏置{}\".format(i,loss,W1,b1))   \n",
    "\n",
    "# 打印最终结果\n",
    "print(\"\\n迭代{}次，当前loss:{}, 当前权重:{},当前偏置{}\".format(i,loss,W1,b1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "W1==W: True \n",
      "b1==b:  True\n"
     ]
    }
   ],
   "source": [
    "print(\"W1==W: {} \\nb1==b:  {}\".format(np.allclose(W1,W),np.allclose(b1,b)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}