main.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "61d38595",
   "metadata": {},
   "source": [
    "# Install Dependencies\n",
    "\n",
    "```bash\n",
    "# pip install pytorch=1.6.0 torchvision=0.7.0 cudatoolkit=10.2 -c pytorch\n",
    "pip install matplotlib path.py tqdm\n",
    "pip install tensorboard tensorboardX\n",
    "pip install scipy scikit-image opencv-contrib-python\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6adf2773",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: matplotlib in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (3.4.2)\n",
      "Requirement already satisfied: path.py in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (12.5.0)\n",
      "Requirement already satisfied: tqdm in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (4.61.1)\n",
      "Requirement already satisfied: pillow>=6.2.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib) (8.2.0)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib) (0.10.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib) (2.4.7)\n",
      "Requirement already satisfied: numpy>=1.16 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib) (1.20.3)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib) (1.3.1)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib) (2.8.1)\n",
      "Requirement already satisfied: path in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from path.py) (16.0.0)\n",
      "Requirement already satisfied: six in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from cycler>=0.10->matplotlib) (1.16.0)\n",
      "Requirement already satisfied: tensorboard in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (2.5.0)\n",
      "Requirement already satisfied: tensorboardX in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (2.2)\n",
      "Requirement already satisfied: google-auth-oauthlib<0.5,>=0.4.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (0.4.4)\n",
      "Requirement already satisfied: grpcio>=1.24.3 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (1.38.0)\n",
      "Requirement already satisfied: tensorboard-plugin-wit>=1.6.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (1.8.0)\n",
      "Requirement already satisfied: wheel>=0.26; python_version >= \"3\" in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (0.36.2)\n",
      "Requirement already satisfied: tensorboard-data-server<0.7.0,>=0.6.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (0.6.1)\n",
      "Requirement already satisfied: werkzeug>=0.11.15 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (2.0.1)\n",
      "Requirement already satisfied: setuptools>=41.0.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (44.0.0)\n",
      "Requirement already satisfied: numpy>=1.12.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (1.20.3)\n",
      "Requirement already satisfied: protobuf>=3.6.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (3.17.3)\n",
      "Requirement already satisfied: requests<3,>=2.21.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (2.25.1)\n",
      "Requirement already satisfied: markdown>=2.6.8 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (3.3.4)\n",
      "Requirement already satisfied: google-auth<2,>=1.6.3 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (1.31.0)\n",
      "Requirement already satisfied: absl-py>=0.4 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from tensorboard) (0.13.0)\n",
      "Requirement already satisfied: requests-oauthlib>=0.7.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (1.3.0)\n",
      "Requirement already satisfied: six>=1.5.2 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from grpcio>=1.24.3->tensorboard) (1.16.0)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard) (4.0.0)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard) (2.10)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard) (1.26.5)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from requests<3,>=2.21.0->tensorboard) (2021.5.30)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard) (0.2.8)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4; python_version >= \"3.6\" in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard) (4.7.2)\n",
      "Requirement already satisfied: cachetools<5.0,>=2.0.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from google-auth<2,>=1.6.3->tensorboard) (4.2.2)\n",
      "Requirement already satisfied: oauthlib>=3.0.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard) (3.1.1)\n",
      "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard) (0.4.8)\n",
      "Requirement already satisfied: scipy in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (1.6.3)\n",
      "Requirement already satisfied: scikit-image in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (0.18.1)\n",
      "Collecting ipykernel\n",
      "  Using cached ipykernel-5.5.5-py3-none-any.whl (120 kB)\n",
      "Requirement already satisfied: numpy<1.23.0,>=1.16.5 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scipy) (1.20.3)\n",
      "Requirement already satisfied: networkx>=2.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scikit-image) (2.5.1)\n",
      "Requirement already satisfied: tifffile>=2019.7.26 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scikit-image) (2021.6.14)\n",
      "Requirement already satisfied: imageio>=2.3.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scikit-image) (2.9.0)\n",
      "Requirement already satisfied: PyWavelets>=1.1.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scikit-image) (1.1.1)\n",
      "Requirement already satisfied: pillow!=7.1.0,!=7.1.1,>=4.3.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scikit-image) (8.2.0)\n",
      "Requirement already satisfied: matplotlib!=3.0.0,>=2.0.0 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from scikit-image) (3.4.2)\n",
      "Collecting jupyter-client\n",
      "  Using cached jupyter_client-6.1.12-py3-none-any.whl (112 kB)\n",
      "Collecting traitlets>=4.1.0\n",
      "  Using cached traitlets-5.0.5-py3-none-any.whl (100 kB)\n",
      "Collecting tornado>=4.2\n",
      "  Using cached tornado-6.1-cp38-cp38-manylinux2010_x86_64.whl (427 kB)\n",
      "Collecting ipython>=5.0.0\n",
      "  Downloading ipython-7.24.1-py3-none-any.whl (785 kB)\n",
      "\u001b[K     |████████████████████████████████| 785 kB 3.2 MB/s eta 0:00:01\n",
      "\u001b[?25hRequirement already satisfied: decorator<5,>=4.3 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from networkx>=2.0->scikit-image) (4.4.2)\n",
      "Requirement already satisfied: python-dateutil>=2.7 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image) (2.8.1)\n",
      "Requirement already satisfied: kiwisolver>=1.0.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image) (1.3.1)\n",
      "Requirement already satisfied: cycler>=0.10 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image) (0.10.0)\n",
      "Requirement already satisfied: pyparsing>=2.2.1 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from matplotlib!=3.0.0,>=2.0.0->scikit-image) (2.4.7)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Collecting jupyter-core>=4.6.0\n",
      "  Using cached jupyter_core-4.7.1-py3-none-any.whl (82 kB)\n",
      "Collecting pyzmq>=13\n",
      "  Downloading pyzmq-22.1.0-cp38-cp38-manylinux2010_x86_64.whl (1.1 MB)\n",
      "\u001b[K     |████████████████████████████████| 1.1 MB 10.9 MB/s eta 0:00:01\n",
      "\u001b[?25hCollecting ipython-genutils\n",
      "  Using cached ipython_genutils-0.2.0-py2.py3-none-any.whl (26 kB)\n",
      "Collecting pygments\n",
      "  Using cached Pygments-2.9.0-py3-none-any.whl (1.0 MB)\n",
      "Collecting matplotlib-inline\n",
      "  Using cached matplotlib_inline-0.1.2-py3-none-any.whl (8.2 kB)\n",
      "Collecting prompt-toolkit!=3.0.0,!=3.0.1,<3.1.0,>=2.0.0\n",
      "  Using cached prompt_toolkit-3.0.18-py3-none-any.whl (367 kB)\n",
      "Collecting jedi>=0.16\n",
      "  Using cached jedi-0.18.0-py2.py3-none-any.whl (1.4 MB)\n",
      "Collecting pexpect>4.3; sys_platform != \"win32\"\n",
      "  Downloading pexpect-4.8.0-py2.py3-none-any.whl (59 kB)\n",
      "\u001b[K     |████████████████████████████████| 59 kB 6.8 MB/s eta 0:00:011\n",
      "\u001b[?25hCollecting pickleshare\n",
      "  Using cached pickleshare-0.7.5-py2.py3-none-any.whl (6.9 kB)\n",
      "Requirement already satisfied: setuptools>=18.5 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from ipython>=5.0.0->ipykernel) (44.0.0)\n",
      "Collecting backcall\n",
      "  Using cached backcall-0.2.0-py2.py3-none-any.whl (11 kB)\n",
      "Requirement already satisfied: six>=1.5 in /home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages (from python-dateutil>=2.7->matplotlib!=3.0.0,>=2.0.0->scikit-image) (1.16.0)\n",
      "Collecting wcwidth\n",
      "  Using cached wcwidth-0.2.5-py2.py3-none-any.whl (30 kB)\n",
      "Collecting parso<0.9.0,>=0.8.0\n",
      "  Using cached parso-0.8.2-py2.py3-none-any.whl (94 kB)\n",
      "Collecting ptyprocess>=0.5\n",
      "  Using cached ptyprocess-0.7.0-py2.py3-none-any.whl (13 kB)\n",
      "Installing collected packages: ipython-genutils, traitlets, jupyter-core, pyzmq, tornado, jupyter-client, pygments, matplotlib-inline, wcwidth, prompt-toolkit, parso, jedi, ptyprocess, pexpect, pickleshare, backcall, ipython, ipykernel\n",
      "Successfully installed backcall-0.2.0 ipykernel-5.5.5 ipython-7.24.1 ipython-genutils-0.2.0 jedi-0.18.0 jupyter-client-6.1.12 jupyter-core-4.7.1 matplotlib-inline-0.1.2 parso-0.8.2 pexpect-4.8.0 pickleshare-0.7.5 prompt-toolkit-3.0.18 ptyprocess-0.7.0 pygments-2.9.0 pyzmq-22.1.0 tornado-6.1 traitlets-5.0.5 wcwidth-0.2.5\n"
     ]
    }
   ],
   "source": [
    "!pip install matplotlib path.py tqdm\n",
    "!pip install tensorboard tensorboardX\n",
    "!pip install scipy scikit-image ipykernel\n",
    "# pip install opencv-contrib-python"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "93fd408c",
   "metadata": {},
   "source": [
    "# Download Dataset\n",
    "\n",
    "To evaluate/train our LEAStereo network, you will need to download the required datasets.\n",
    "\n",
    "* [SceneFlow](https://lmb.informatik.uni-freiburg.de/resources/datasets/SceneFlowDatasets.en.html)\n",
    "\n",
    "* [KITTI2015](http://www.cvlibs.net/datasets/kitti/eval_scene_flow.php?benchmark=stereo)\n",
    "\n",
    "* [KITTI2012](http://www.cvlibs.net/datasets/kitti/eval_stereo_flow.php?benchmark=stereo)\n",
    "\n",
    "* [Middlebury 2014](https://vision.middlebury.edu/stereo/submit3/)\n",
    "\n",
    "Change the first column path in file `create_link.sh` with your actual dataset location. Then run `create_link.sh` that will create symbolic links to wherever the datasets were downloaded in the `datasets` folder. For Middlebury 2014 dataset, we perform our network on half resolution images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "4422725c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mkdir: cannot create directory ‘/storage/datasets’: No such file or directory\r\n",
      "mkdir: cannot create directory ‘/storage/datasets/kitti2012’: No such file or directory\r\n",
      "/storage/kitti2012/kitti2012.zip: No such file or directory\r\n",
      "unzip:  cannot find or open /storage/kitti2012/kitti2012.zip, /storage/kitti2012/kitti2012.zip.zip or /storage/kitti2012/kitti2012.zip.ZIP.\r\n"
     ]
    }
   ],
   "source": [
    "# TODO : download and link dataset dir\n",
    "!sh download_kitti12.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d345895",
   "metadata": {},
   "outputs": [],
   "source": [
    "!sh download_kitti15.sh"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "565495d4",
   "metadata": {},
   "source": [
    "# Test Predictions\n",
    "\n",
    "You can evaluate a trained model using prediction.sh for each dataset, that would help you generate *.png or *.pfm images correspoding to different datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "7b766409",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Namespace(cell_arch_fea='run/sceneflow/best/architecture/feature_genotype.npy', cell_arch_mat='run/sceneflow/best/architecture/matching_genotype.npy', crop_height=384, crop_width=1248, cuda=True, data_path='./dataset/kitti2012/testing/', fea_block_multiplier=4, fea_filter_multiplier=8, fea_num_layers=6, fea_step=3, kitti2012=1, kitti2015=0, mat_block_multiplier=4, mat_filter_multiplier=8, mat_num_layers=12, mat_step=3, maxdisp=192, middlebury=0, net_arch_fea='run/sceneflow/best/architecture/feature_network_path.npy', net_arch_mat='run/sceneflow/best/architecture/matching_network_path.npy', resume='./run/Kitti12/best/best_1.16.pth', save_path='./predict/kitti2012/images/', sceneflow=0, test_list='./dataloaders/lists/kitti2012_test.list')\n",
      "===> Building LEAStereo model\n",
      "Feature network path:[1 0 1 0 0 0]\n",
      "Matching network path:[1 1 2 2 1 2 2 2 1 1 0 1] \n",
      "\n",
      "Total Params = 1.81MB\n",
      "Feature Net Params = 0.10MB\n",
      "Matching Net Params = 1.71MB\n",
      "Traceback (most recent call last):\n",
      "  File \"predict.py\", line 48, in <module>\n",
      "    mult_adds = comp_multadds(model, input_size=(3,opt.crop_height, opt.crop_width)) #(3,192, 192))\n",
      "  File \"/mnt/HDD/home/aditya/Projects/LEAStereo/utils/multadds_count.py\", line 21, in comp_multadds\n",
      "    _ = model(input_data, input_data)\n",
      "  File \"/home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 889, in _call_impl\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/mnt/HDD/home/aditya/Projects/LEAStereo/retrain/LEAStereo.py\", line 41, in forward\n",
      "    cost = self.matching(cost)     \n",
      "  File \"/home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 889, in _call_impl\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/mnt/HDD/home/aditya/Projects/LEAStereo/retrain/skip_model_3d.py\", line 140, in forward\n",
      "    stem1 = self.stem1(stem0)\n",
      "  File \"/home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 889, in _call_impl\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/mnt/HDD/home/aditya/Projects/LEAStereo/models/operations_3d.py\", line 42, in forward\n",
      "    x = self.conv(x)\n",
      "  File \"/home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 889, in _call_impl\n",
      "    result = self.forward(*input, **kwargs)\n",
      "  File \"/home/aditya/VSProjects/LEAStereo/LEAStereo/lib/python3.8/site-packages/torch/nn/modules/conv.py\", line 520, in forward\n",
      "    return F.conv3d(input, self.weight, self.bias, self.stride,\n",
      "RuntimeError: CUDA out of memory. Tried to allocate 476.00 MiB (GPU 0; 3.82 GiB total capacity; 1.65 GiB already allocated; 492.50 MiB free; 2.12 GiB reserved in total by PyTorch)\n"
     ]
    }
   ],
   "source": [
    "!sh predict_kitti12.sh"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9cfe970",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}