main_TSP_edge_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main Driver Notebook for Training Graph NNs on TSP for Edge Classification"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MODELS\n",
    "- GatedGCN \n",
    "- GCN \n",
    "- GAT \n",
    "- GraphSage \n",
    "- GIN  \n",
    "- MoNet  \n",
    "- MLP\n",
    "\n",
    "### DATASET\n",
    "- TSP\n",
    "\n",
    "### TASK\n",
    "- Edge Classification, i.e. Classifying each edge as belonging/not belonging to the optimal TSP solution set."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    IMPORTING LIBS\n",
    "\"\"\"\n",
    "import dgl\n",
    "\n",
    "import numpy as np\n",
    "import os\n",
    "import socket\n",
    "import time\n",
    "import random\n",
    "import glob\n",
    "import argparse, json\n",
    "import pickle\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import DataLoader\n",
    "\n",
    "from tensorboardX import SummaryWriter\n",
    "from tqdm import tqdm\n",
    "\n",
    "class DotDict(dict):\n",
    "    def __init__(self, **kwds):\n",
    "        self.update(kwds)\n",
    "        self.__dict__ = self\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True\n"
     ]
    }
   ],
   "source": [
    "# \"\"\"\n",
    "#     AUTORELOAD IPYTHON EXTENSION FOR RELOADING IMPORTED MODULES\n",
    "# \"\"\"\n",
    "\n",
    "def in_ipynb():\n",
    "    try:\n",
    "        cfg = get_ipython().config \n",
    "        return True\n",
    "    except NameError:\n",
    "        return False\n",
    "    \n",
    "notebook_mode = in_ipynb()\n",
    "print(notebook_mode)\n",
    "\n",
    "if notebook_mode == True:\n",
    "    %load_ext autoreload\n",
    "    %autoreload 2\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    IMPORTING CUSTOM MODULES/METHODS\n",
    "\"\"\"\n",
    "from nets.TSP_edge_classification.load_net import gnn_model # import all GNNS\n",
    "from data.data import LoadData # import dataset\n",
    "from train.train_TSP_edge_classification import train_epoch, evaluate_network # import train functions\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    GPU Setup\n",
    "\"\"\"\n",
    "def gpu_setup(use_gpu, gpu_id):\n",
    "    os.environ[\"CUDA_DEVICE_ORDER\"] = \"PCI_BUS_ID\"\n",
    "    os.environ[\"CUDA_VISIBLE_DEVICES\"] = str(gpu_id)  \n",
    "\n",
    "    if torch.cuda.is_available() and use_gpu:\n",
    "        print('cuda available with GPU:',torch.cuda.get_device_name(0))\n",
    "        device = torch.device(\"cuda\")\n",
    "    else:\n",
    "        print('cuda not available')\n",
    "        device = torch.device(\"cpu\")\n",
    "    return device\n",
    "\n",
    "\n",
    "use_gpu = True\n",
    "gpu_id = -1\n",
    "device = None\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[I] Loading data (notebook) ...\n",
      "[I] Loading dataset TSP...\n",
      "train, test, val sizes : 10000 1000 1000\n",
      "[I] Finished loading.\n",
      "[I] Data load time: 20.0447s\n",
      "[I] Finished loading.\n"
     ]
    }
   ],
   "source": [
    "# \"\"\"\n",
    "#     USER CONTROLS\n",
    "# \"\"\"\n",
    "if notebook_mode == True:\n",
    "    \n",
    "    MODEL_NAME = 'MLP'\n",
    "    # MODEL_NAME = 'GCN'\n",
    "    MODEL_NAME = 'GatedGCN'\n",
    "    # MODEL_NAME = 'GAT'\n",
    "    # MODEL_NAME = 'GraphSage'\n",
    "    # MODEL_NAME = 'DiffPool'\n",
    "    # MODEL_NAME = 'GIN'\n",
    "\n",
    "    DATASET_NAME = 'TSP'\n",
    "\n",
    "    out_dir = 'out/TSP_edge_classification/'\n",
    "    root_log_dir = out_dir + 'logs/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n",
    "    root_ckpt_dir = out_dir + 'checkpoints/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n",
    "\n",
    "    print(\"[I] Loading data (notebook) ...\")\n",
    "    dataset = LoadData(DATASET_NAME)\n",
    "    trainset, valset, testset = dataset.train, dataset.val, dataset.test\n",
    "    print(\"[I] Finished loading.\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME = 'GatedGCN'\n",
    "MODEL_NAME = 'GCN'\n",
    "MODEL_NAME = 'GAT'\n",
    "#MODEL_NAME = 'GraphSage'\n",
    "#MODEL_NAME = 'MLP'\n",
    "#MODEL_NAME = 'GIN'\n",
    "#MODEL_NAME = 'MoNet'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# \"\"\"\n",
    "#     PARAMETERS\n",
    "# \"\"\"\n",
    "if notebook_mode == True:\n",
    "    \n",
    "    #MODEL_NAME = 'GCN'\n",
    "    \n",
    "    n_heads = -1\n",
    "    edge_feat = False\n",
    "    pseudo_dim_MoNet = -1\n",
    "    kernel = -1\n",
    "    gnn_per_block = -1\n",
    "    embedding_dim = -1\n",
    "    pool_ratio = -1\n",
    "    n_mlp_GIN = -1\n",
    "    gated = False\n",
    "    self_loop = False\n",
    "    max_time = 48\n",
    "\n",
    "    \n",
    "    if MODEL_NAME == 'MLP':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=3; hidden_dim=144; out_dim=hidden_dim; dropout=0.0; readout='mean'; gated = False  # Change gated = True for Gated MLP model\n",
    "    \n",
    "    if MODEL_NAME == 'GCN':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=4; hidden_dim=128; out_dim=hidden_dim; dropout=0.0; readout='mean';\n",
    "        \n",
    "    if MODEL_NAME == 'GraphSage':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=4; hidden_dim=96; out_dim=hidden_dim; dropout=0.0; readout='mean';\n",
    "\n",
    "    if MODEL_NAME == 'GAT':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=4; n_heads=8; hidden_dim=16; out_dim=128; dropout=0.0; readout='mean';\n",
    "    \n",
    "    if MODEL_NAME == 'GIN':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=4; hidden_dim=112; out_dim=hidden_dim; dropout=0.0; readout='mean';\n",
    "        \n",
    "    if MODEL_NAME == 'MoNet':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=4; hidden_dim=80; out_dim=hidden_dim; dropout=0.0; readout='mean';\n",
    "        \n",
    "    if MODEL_NAME == 'GatedGCN':\n",
    "        seed=41; epochs=500; batch_size=64; init_lr=0.001; lr_reduce_factor=0.5; lr_schedule_patience=10; min_lr = 1e-5; weight_decay=0\n",
    "        L=4; hidden_dim=64; out_dim=hidden_dim; dropout=0.0; readout='mean'; edge_feat = True\n",
    "\n",
    "    # generic new_params\n",
    "    net_params = {}\n",
    "    net_params['device'] = device\n",
    "    net_params['in_dim'] = trainset[0][0].ndata['feat'][0].size(0)\n",
    "    net_params['in_dim_edge'] = trainset[0][0].edata['feat'][0].size(0)\n",
    "    net_params['residual'] = True\n",
    "    net_params['hidden_dim'] = hidden_dim\n",
    "    net_params['out_dim'] = out_dim\n",
    "    num_classes = len(np.unique(np.concatenate(trainset[:][1])))\n",
    "    net_params['n_classes'] = num_classes\n",
    "    net_params['n_heads'] = n_heads\n",
    "    net_params['L'] = L  # min L should be 2\n",
    "    net_params['readout'] = \"mean\"\n",
    "    net_params['graph_norm'] = True\n",
    "    net_params['batch_norm'] = True\n",
    "    net_params['in_feat_dropout'] = 0.0\n",
    "    net_params['dropout'] = 0.0\n",
    "    net_params['edge_feat'] = edge_feat\n",
    "    net_params['self_loop'] = self_loop\n",
    "    \n",
    "    # for MLPNet \n",
    "    net_params['gated'] = gated\n",
    "    \n",
    "    # specific for MoNet\n",
    "    net_params['pseudo_dim_MoNet'] = 2\n",
    "    net_params['kernel'] = 3\n",
    "    \n",
    "    # specific for GIN\n",
    "    net_params['n_mlp_GIN'] = 2\n",
    "    net_params['learn_eps_GIN'] = True\n",
    "    net_params['neighbor_aggr_GIN'] = 'sum'\n",
    "    \n",
    "    # specific for graphsage\n",
    "    net_params['sage_aggregator'] = 'meanpool'    \n",
    "\n",
    "    # setting seeds\n",
    "    random.seed(seed)\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "    if device == 'cuda':\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL DETAILS:\n",
      "\n",
      "MODEL/Total parameters: GAT 109250\n"
     ]
    }
   ],
   "source": [
    "\"\"\"\n",
    "    VIEWING MODEL CONFIG AND PARAMS\n",
    "\"\"\"\n",
    "def view_model_param(MODEL_NAME, net_params):\n",
    "    model = gnn_model(MODEL_NAME, net_params)\n",
    "    total_param = 0\n",
    "    print(\"MODEL DETAILS:\\n\")\n",
    "    #print(model)\n",
    "    for param in model.parameters():\n",
    "        # print(param.data.size())\n",
    "        total_param += np.prod(list(param.data.size()))\n",
    "    print('MODEL/Total parameters:', MODEL_NAME, total_param)\n",
    "    return total_param\n",
    "\n",
    "\n",
    "if notebook_mode == True:\n",
    "    view_model_param(MODEL_NAME, net_params)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "    TRAINING CODE\n",
    "\"\"\"\n",
    "\n",
    "def train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs):\n",
    "    t0 = time.time()\n",
    "    per_epoch_time = []\n",
    "        \n",
    "    DATASET_NAME = dataset.name\n",
    "    \n",
    "    #assert net_params['self_loop'] == False, \"No self-loop support for %s dataset\" % DATASET_NAME\n",
    "    \n",
    "    trainset, valset, testset = dataset.train, dataset.val, dataset.test\n",
    "        \n",
    "    root_log_dir, root_ckpt_dir, write_file_name, write_config_file = dirs\n",
    "    device = net_params['device']\n",
    "    \n",
    "    # Write the network and optimization hyper-parameters in folder config/\n",
    "    with open(write_config_file + '.txt', 'w') as f:\n",
    "        f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n\\nTotal Parameters: {}\\n\\n\"\"\"\\\n",
    "                .format(DATASET_NAME, MODEL_NAME, params, net_params, net_params['total_param']))\n",
    "        \n",
    "    log_dir = os.path.join(root_log_dir, \"RUN_\" + str(0))\n",
    "    writer = SummaryWriter(log_dir=log_dir)\n",
    "\n",
    "    # setting seeds\n",
    "    random.seed(params['seed'])\n",
    "    np.random.seed(params['seed'])\n",
    "    torch.manual_seed(params['seed'])\n",
    "    if device == 'cuda':\n",
    "        torch.cuda.manual_seed(params['seed'])\n",
    "    \n",
    "    print(\"Training Graphs: \", len(trainset))\n",
    "    print(\"Validation Graphs: \", len(valset))\n",
    "    print(\"Test Graphs: \", len(testset))\n",
    "    print(\"Number of Classes: \", net_params['n_classes'])\n",
    "\n",
    "    model = gnn_model(MODEL_NAME, net_params)\n",
    "    model = model.to(device)\n",
    "\n",
    "    optimizer = optim.Adam(model.parameters(), lr=params['init_lr'], weight_decay=params['weight_decay'])\n",
    "    scheduler = optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='min',\n",
    "                                                     factor=params['lr_reduce_factor'],\n",
    "                                                     patience=params['lr_schedule_patience'],\n",
    "                                                     verbose=True)\n",
    "    \n",
    "    epoch_train_losses, epoch_val_losses = [], []\n",
    "    epoch_train_f1s, epoch_val_f1s = [], [] \n",
    "    \n",
    "    train_loader = DataLoader(trainset, batch_size=params['batch_size'], shuffle=True, collate_fn=dataset.collate)\n",
    "    val_loader = DataLoader(valset, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n",
    "    test_loader = DataLoader(testset, batch_size=params['batch_size'], shuffle=False, collate_fn=dataset.collate)\n",
    "\n",
    "    # At any point you can hit Ctrl + C to break out of training early.\n",
    "    try:\n",
    "        with tqdm(range(params['epochs'])) as t:\n",
    "            for epoch in t:\n",
    "\n",
    "                t.set_description('Epoch %d' % epoch)    \n",
    "\n",
    "                start = time.time()\n",
    "\n",
    "                epoch_train_loss, epoch_train_f1, optimizer = train_epoch(model, optimizer, device, train_loader, epoch)\n",
    "                epoch_val_loss, epoch_val_f1 = evaluate_network(model, device, val_loader, epoch)\n",
    "\n",
    "                epoch_train_losses.append(epoch_train_loss)\n",
    "                epoch_val_losses.append(epoch_val_loss)\n",
    "                epoch_train_f1s.append(epoch_train_f1)\n",
    "                epoch_val_f1s.append(epoch_val_f1)\n",
    "\n",
    "                writer.add_scalar('train/_loss', epoch_train_loss, epoch)\n",
    "                writer.add_scalar('val/_loss', epoch_val_loss, epoch)\n",
    "                writer.add_scalar('train/_f1', epoch_train_f1, epoch)\n",
    "                writer.add_scalar('val/_f1', epoch_val_f1, epoch)\n",
    "                writer.add_scalar('learning_rate', optimizer.param_groups[0]['lr'], epoch)   \n",
    "\n",
    "                _, epoch_test_f1 = evaluate_network(model, device, test_loader, epoch)                        \n",
    "                t.set_postfix(time=time.time()-start, lr=optimizer.param_groups[0]['lr'],\n",
    "                              train_loss=epoch_train_loss, val_loss=epoch_val_loss,\n",
    "                              train_f1=epoch_train_f1, val_f1=epoch_val_f1,\n",
    "                              test_f1=epoch_test_f1) \n",
    "\n",
    "                per_epoch_time.append(time.time()-start)\n",
    "\n",
    "                # Saving checkpoint\n",
    "                ckpt_dir = os.path.join(root_ckpt_dir, \"RUN_\")\n",
    "                if not os.path.exists(ckpt_dir):\n",
    "                    os.makedirs(ckpt_dir)\n",
    "                torch.save(model.state_dict(), '{}.pkl'.format(ckpt_dir + \"/epoch_\" + str(epoch)))\n",
    "\n",
    "                files = glob.glob(ckpt_dir + '/*.pkl')\n",
    "                for file in files:\n",
    "                    epoch_nb = file.split('_')[-1]\n",
    "                    epoch_nb = int(epoch_nb.split('.')[0])\n",
    "                    if epoch_nb < epoch-1:\n",
    "                        os.remove(file)\n",
    "\n",
    "                scheduler.step(epoch_val_loss)\n",
    "\n",
    "                if optimizer.param_groups[0]['lr'] < params['min_lr']:\n",
    "                    print(\"\\n!! LR EQUAL TO MIN LR SET.\")\n",
    "                    break\n",
    "                    \n",
    "                # Stop training after params['max_time'] hours\n",
    "                if time.time()-t0 > params['max_time']*3600:\n",
    "                    print('-' * 89)\n",
    "                    print(\"Max_time for training elapsed {:.2f} hours, so stopping\".format(params['max_time']))\n",
    "                    break\n",
    "    \n",
    "    except KeyboardInterrupt:\n",
    "        print('-' * 89)\n",
    "        print('Exiting from training early because of KeyboardInterrupt')\n",
    "    \n",
    "    _, test_f1 = evaluate_network(model, device, test_loader, epoch)\n",
    "    _, train_f1 = evaluate_network(model, device, train_loader, epoch)\n",
    "    print(\"Test F1: {:.4f}\".format(test_f1))\n",
    "    print(\"Train F1: {:.4f}\".format(train_f1))\n",
    "    print(\"TOTAL TIME TAKEN: {:.4f}s\".format(time.time()-t0))\n",
    "    print(\"AVG TIME PER EPOCH: {:.4f}s\".format(np.mean(per_epoch_time)))\n",
    "\n",
    "    writer.close()\n",
    "\n",
    "    \"\"\"\n",
    "        Write the results in out_dir/results folder\n",
    "    \"\"\"\n",
    "    with open(write_file_name + '.txt', 'w') as f:\n",
    "        f.write(\"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n{}\\n\\nTotal Parameters: {}\\n\\n\n",
    "    FINAL RESULTS\\nTEST F1: {:.4f}\\nTRAIN F1: {:.4f}\\n\\n\n",
    "    Total Time Taken: {:.4f}hrs\\nAverage Time Per Epoch: {:.4f}s\\n\\n\\n\"\"\"\\\n",
    "          .format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],\n",
    "                  np.mean(np.array(test_f1)), np.mean(np.array(train_f1)), (time.time()-t0)/3600, np.mean(per_epoch_time)))\n",
    "        \n",
    "\n",
    "    # send results to gmail\n",
    "    try:\n",
    "        from gmail import send\n",
    "        subject = 'Result for Dataset: {}, Model: {}'.format(DATASET_NAME, MODEL_NAME)\n",
    "        body = \"\"\"Dataset: {},\\nModel: {}\\n\\nparams={}\\n\\nnet_params={}\\n\\n{}\\n\\nTotal Parameters: {}\\n\\n\n",
    "    FINAL RESULTS\\nTEST F1: {:.4f}\\nTRAIN F1: {:.4f}\\n\\n\n",
    "    Total Time Taken: {:.4f}hrs\\nAverage Time Per Epoch: {:.4f}s\\n\\n\\n\"\"\"\\\n",
    "          .format(DATASET_NAME, MODEL_NAME, params, net_params, model, net_params['total_param'],\n",
    "                  np.mean(np.array(test_f1)), np.mean(np.array(train_f1)), (time.time()-t0)/3600, np.mean(per_epoch_time))\n",
    "        send(subject, body)\n",
    "    except:\n",
    "        pass\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Convert main_TSP_edge_classification.ipynb to main_TSP_edge_classification.py\n",
      "Clean main_TSP_edge_classification.py\n",
      "Done. \n",
      "[I] Loading dataset TSP...\n",
      "train, test, val sizes : 10000 1000 1000\n",
      "[I] Finished loading.\n",
      "[I] Data load time: 24.1761s\n",
      "cuda not available\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 0/500 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MODEL DETAILS:\n",
      "\n",
      "MODEL/Total parameters: GAT 109250\n",
      "Training Graphs:  10000\n",
      "Validation Graphs:  1000\n",
      "Test Graphs:  1000\n",
      "Number of Classes:  2\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Epoch 0:   0%|          | 0/500 [01:00<?, ?it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-----------------------------------------------------------------------------------------\n",
      "Exiting from training early because of KeyboardInterrupt\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-10-91d2b7a98636>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m    229\u001b[0m     \u001b[0mcleaner_main\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'main_TSP_edge_classification'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    230\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 231\u001b[0;31m     \u001b[0mmain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0mconfig\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    232\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    233\u001b[0m \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-10-91d2b7a98636>\u001b[0m in \u001b[0;36mmain\u001b[0;34m(notebook_mode, config)\u001b[0m\n\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m     \u001b[0mnet_params\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'total_param'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mview_model_param\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMODEL_NAME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet_params\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 191\u001b[0;31m     \u001b[0mtrain_val_pipeline\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mMODEL_NAME\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnet_params\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdirs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    192\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    193\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-9-6c6332224cca>\u001b[0m in \u001b[0;36mtrain_val_pipeline\u001b[0;34m(MODEL_NAME, dataset, params, net_params, dirs)\u001b[0m\n\u001b[1;32m    113\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    114\u001b[0m     \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_f1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtest_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 115\u001b[0;31m     \u001b[0m_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_f1\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mevaluate_network\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdevice\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepoch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    116\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Test F1: {:.4f}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtest_f1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"Train F1: {:.4f}\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_f1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/benchmarking-gnn/train/train_TSP_edge_classification.py\u001b[0m in \u001b[0;36mevaluate_network\u001b[0;34m(model, device, data_loader, epoch)\u001b[0m\n\u001b[1;32m     52\u001b[0m             \u001b[0mbatch_snorm_n\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mbatch_snorm_n\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 54\u001b[0;31m             \u001b[0mbatch_scores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_graphs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_x\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_e\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_snorm_n\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_snorm_e\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     55\u001b[0m             \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch_scores\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_labels\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     56\u001b[0m             \u001b[0mepoch_test_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdetach\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/benchmarking-gnn/nets/TSP_edge_classification/gat_net.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, g, h, e, snorm_n, snorm_e)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0min_feat_dropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mconv\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlayers\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m             \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mconv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msnorm_n\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     48\u001b[0m         \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'h'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     49\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/benchmarking-gnn/layers/gat_layer.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, g, h, snorm_n)\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msnorm_n\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mh_in\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh\u001b[0m \u001b[0;31m# for residual connection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m         \u001b[0mhead_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mattn_head\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msnorm_n\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mattn_head\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mheads\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'cat'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/benchmarking-gnn/layers/gat_layer.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     69\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msnorm_n\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     70\u001b[0m         \u001b[0mh_in\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mh\u001b[0m \u001b[0;31m# for residual connection\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 71\u001b[0;31m         \u001b[0mhead_outs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mattn_head\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mg\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msnorm_n\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mattn_head\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mheads\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     72\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     73\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmerge\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'cat'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m    491\u001b[0m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_slow_forward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    492\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 493\u001b[0;31m             \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    494\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mhook\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_forward_hooks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    495\u001b[0m             \u001b[0mhook_result\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/benchmarking-gnn/layers/gat_layer.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, g, h, snorm_n)\u001b[0m\n\u001b[1;32m     38\u001b[0m         \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'z'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mz\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     39\u001b[0m         \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mapply_edges\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0medge_attention\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 40\u001b[0;31m         \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmessage_func\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreduce_func\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     41\u001b[0m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mg\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mndata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'h'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     42\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgraph_norm\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/dgl/graph.py\u001b[0m in \u001b[0;36mupdate_all\u001b[0;34m(self, message_func, reduce_func, apply_node_func)\u001b[0m\n\u001b[1;32m   2745\u001b[0m                                           \u001b[0mreduce_func\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mreduce_func\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2746\u001b[0m                                           apply_func=apply_node_func)\n\u001b[0;32m-> 2747\u001b[0;31m             \u001b[0mRuntime\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mprog\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2748\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2749\u001b[0m     def prop_nodes(self,\n",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/dgl/runtime/runtime.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(prog)\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mexe\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mprog\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0;31m# prog.pprint_exe(exe)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m             \u001b[0mexe\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/dgl/runtime/ir/executor.py\u001b[0m in \u001b[0;36mrun\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    130\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    131\u001b[0m             \u001b[0mmail_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfdmail\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 132\u001b[0;31m             \u001b[0mudf_ret\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnode_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmail_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    133\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mret\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mFrameRef\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mFrame\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mudf_ret\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/dgl/runtime/degree_bucketing.py\u001b[0m in \u001b[0;36m_rfunc_wrapper\u001b[0;34m(node_data, mail_data)\u001b[0m\n\u001b[1;32m    151\u001b[0m         \u001b[0mreshaped_mail_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mutils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLazyDict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_reshaped_getter\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmail_data\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mkeys\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    152\u001b[0m         \u001b[0mnbatch\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mNodeBatch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvbkt\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnode_data\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mreshaped_mail_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 153\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mreduce_udf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    154\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0m_rfunc_wrapper\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    155\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/benchmarking-gnn/layers/gat_layer.py\u001b[0m in \u001b[0;36mreduce_func\u001b[0;34m(self, nodes)\u001b[0m\n\u001b[1;32m     31\u001b[0m         \u001b[0malpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msoftmax\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnodes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmailbox\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'e'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     32\u001b[0m         \u001b[0malpha\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mF\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdropout\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtraining\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtraining\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 33\u001b[0;31m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0malpha\u001b[0m \u001b[0;34m*\u001b[0m \u001b[0mnodes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmailbox\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'z'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdim\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     34\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'h'\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mh\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     35\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/benchmark_gnn/lib/python3.7/site-packages/dgl/udf.py\u001b[0m in \u001b[0;36mmailbox\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    121\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 122\u001b[0;31m     \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    123\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmailbox\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    124\u001b[0m         \"\"\"Return the received messages.\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "def main(notebook_mode=False,config=None):\n",
    "    \n",
    "    \"\"\"\n",
    "        USER CONTROLS\n",
    "    \"\"\"\n",
    "    \n",
    "    # terminal mode\n",
    "    if notebook_mode==False:\n",
    "        \n",
    "        parser = argparse.ArgumentParser()\n",
    "        parser.add_argument('--config', help=\"Please give a config.json file with training/model/data/param details\")\n",
    "        parser.add_argument('--gpu_id', help=\"Please give a value for gpu id\")\n",
    "        parser.add_argument('--model', help=\"Please give a value for model name\")\n",
    "        parser.add_argument('--dataset', help=\"Please give a value for dataset name\")\n",
    "        parser.add_argument('--out_dir', help=\"Please give a value for out_dir\")\n",
    "        parser.add_argument('--seed', help=\"Please give a value for seed\")\n",
    "        parser.add_argument('--epochs', help=\"Please give a value for epochs\")\n",
    "        parser.add_argument('--batch_size', help=\"Please give a value for batch_size\")\n",
    "        parser.add_argument('--init_lr', help=\"Please give a value for init_lr\")\n",
    "        parser.add_argument('--lr_reduce_factor', help=\"Please give a value for lr_reduce_factor\")\n",
    "        parser.add_argument('--lr_schedule_patience', help=\"Please give a value for lr_schedule_patience\")\n",
    "        parser.add_argument('--min_lr', help=\"Please give a value for min_lr\")\n",
    "        parser.add_argument('--weight_decay', help=\"Please give a value for weight_decay\")\n",
    "        parser.add_argument('--print_epoch_interval', help=\"Please give a value for print_epoch_interval\")    \n",
    "        parser.add_argument('--L', help=\"Please give a value for L\")\n",
    "        parser.add_argument('--hidden_dim', help=\"Please give a value for hidden_dim\")\n",
    "        parser.add_argument('--out_dim', help=\"Please give a value for out_dim\")\n",
    "        parser.add_argument('--residual', help=\"Please give a value for residual\")\n",
    "        parser.add_argument('--edge_feat', help=\"Please give a value for edge_feat\")\n",
    "        parser.add_argument('--readout', help=\"Please give a value for readout\")\n",
    "        parser.add_argument('--kernel', help=\"Please give a value for kernel\")\n",
    "        parser.add_argument('--n_heads', help=\"Please give a value for n_heads\")\n",
    "        parser.add_argument('--gated', help=\"Please give a value for gated\")\n",
    "        parser.add_argument('--in_feat_dropout', help=\"Please give a value for in_feat_dropout\")\n",
    "        parser.add_argument('--dropout', help=\"Please give a value for dropout\")\n",
    "        parser.add_argument('--graph_norm', help=\"Please give a value for graph_norm\")\n",
    "        parser.add_argument('--batch_norm', help=\"Please give a value for batch_norm\")\n",
    "        parser.add_argument('--sage_aggregator', help=\"Please give a value for sage_aggregator\")\n",
    "        parser.add_argument('--data_mode', help=\"Please give a value for data_mode\")\n",
    "        parser.add_argument('--num_pool', help=\"Please give a value for num_pool\")\n",
    "        parser.add_argument('--gnn_per_block', help=\"Please give a value for gnn_per_block\")\n",
    "        parser.add_argument('--embedding_dim', help=\"Please give a value for embedding_dim\")\n",
    "        parser.add_argument('--pool_ratio', help=\"Please give a value for pool_ratio\")\n",
    "        parser.add_argument('--linkpred', help=\"Please give a value for linkpred\")\n",
    "        parser.add_argument('--cat', help=\"Please give a value for cat\")\n",
    "        parser.add_argument('--self_loop', help=\"Please give a value for self_loop\")\n",
    "        parser.add_argument('--max_time', help=\"Please give a value for max_time\")\n",
    "        args = parser.parse_args()\n",
    "        with open(args.config) as f:\n",
    "            config = json.load(f)\n",
    "            \n",
    "\n",
    "        # device\n",
    "        if args.gpu_id is not None:\n",
    "            config['gpu']['id'] = int(args.gpu_id)\n",
    "            config['gpu']['use'] = True\n",
    "        device = gpu_setup(config['gpu']['use'], config['gpu']['id'])\n",
    "\n",
    "        # model, dataset, out_dir\n",
    "        if args.model is not None:\n",
    "            MODEL_NAME = args.model\n",
    "        else:\n",
    "            MODEL_NAME = config['model']\n",
    "        if args.dataset is not None:\n",
    "            DATASET_NAME = args.dataset\n",
    "        else:\n",
    "            DATASET_NAME = config['dataset']\n",
    "        dataset = LoadData(DATASET_NAME)\n",
    "        if args.out_dir is not None:\n",
    "            out_dir = args.out_dir\n",
    "        else:\n",
    "            out_dir = config['out_dir']\n",
    "\n",
    "        # parameters\n",
    "        params = config['params']\n",
    "        if args.seed is not None:\n",
    "            params['seed'] = int(args.seed)\n",
    "        if args.epochs is not None:\n",
    "            params['epochs'] = int(args.epochs)\n",
    "        if args.batch_size is not None:\n",
    "            params['batch_size'] = int(args.batch_size)\n",
    "        if args.init_lr is not None:\n",
    "            params['init_lr'] = float(args.init_lr)\n",
    "        if args.lr_reduce_factor is not None:\n",
    "            params['lr_reduce_factor'] = float(args.lr_reduce_factor)\n",
    "        if args.lr_schedule_patience is not None:\n",
    "            params['lr_schedule_patience'] = int(args.lr_schedule_patience)\n",
    "        if args.min_lr is not None:\n",
    "            params['min_lr'] = float(args.min_lr)\n",
    "        if args.weight_decay is not None:\n",
    "            params['weight_decay'] = float(args.weight_decay)\n",
    "        if args.print_epoch_interval is not None:\n",
    "            params['print_epoch_interval'] = int(args.print_epoch_interval)\n",
    "        if args.max_time is not None:\n",
    "            params['max_time'] = float(args.max_time)\n",
    "\n",
    "        # network parameters\n",
    "        net_params = config['net_params']\n",
    "        net_params['device'] = device\n",
    "        net_params['gpu_id'] = config['gpu']['id']\n",
    "        net_params['batch_size'] = params['batch_size']\n",
    "        if args.L is not None:\n",
    "            net_params['L'] = int(args.L)\n",
    "        if args.hidden_dim is not None:\n",
    "            net_params['hidden_dim'] = int(args.hidden_dim)\n",
    "        if args.out_dim is not None:\n",
    "            net_params['out_dim'] = int(args.out_dim)   \n",
    "        if args.residual is not None:\n",
    "            net_params['residual'] = True if args.residual=='True' else False\n",
    "        if args.edge_feat is not None:\n",
    "            net_params['edge_feat'] = True if args.edge_feat=='True' else False\n",
    "        if args.readout is not None:\n",
    "            net_params['readout'] = args.readout\n",
    "        if args.kernel is not None:\n",
    "            net_params['kernel'] = int(args.kernel)\n",
    "        if args.n_heads is not None:\n",
    "            net_params['n_heads'] = int(args.n_heads)\n",
    "        if args.gated is not None:\n",
    "            net_params['gated'] = True if args.gated=='True' else False\n",
    "        if args.in_feat_dropout is not None:\n",
    "            net_params['in_feat_dropout'] = float(args.in_feat_dropout)\n",
    "        if args.dropout is not None:\n",
    "            net_params['dropout'] = float(args.dropout)\n",
    "        if args.graph_norm is not None:\n",
    "            net_params['graph_norm'] = True if args.graph_norm=='True' else False\n",
    "        if args.batch_norm is not None:\n",
    "            net_params['batch_norm'] = True if args.batch_norm=='True' else False\n",
    "        if args.sage_aggregator is not None:\n",
    "            net_params['sage_aggregator'] = args.sage_aggregator\n",
    "        if args.data_mode is not None:\n",
    "            net_params['data_mode'] = args.data_mode\n",
    "        if args.num_pool is not None:\n",
    "            net_params['num_pool'] = int(args.num_pool)\n",
    "        if args.gnn_per_block is not None:\n",
    "            net_params['gnn_per_block'] = int(args.gnn_per_block)\n",
    "        if args.embedding_dim is not None:\n",
    "            net_params['embedding_dim'] = int(args.embedding_dim)\n",
    "        if args.pool_ratio is not None:\n",
    "            net_params['pool_ratio'] = float(args.pool_ratio)\n",
    "        if args.linkpred is not None:\n",
    "            net_params['linkpred'] = True if args.linkpred=='True' else False\n",
    "        if args.cat is not None:\n",
    "            net_params['cat'] = True if args.cat=='True' else False\n",
    "        if args.self_loop is not None:\n",
    "            net_params['self_loop'] = True if args.self_loop=='True' else False\n",
    " \n",
    "\n",
    "    # notebook mode\n",
    "    if notebook_mode:\n",
    "        \n",
    "        # parameters\n",
    "        params = config['params']\n",
    "        \n",
    "        # dataset\n",
    "        DATASET_NAME = config['dataset']\n",
    "        dataset = LoadData(DATASET_NAME)\n",
    "        \n",
    "        # device\n",
    "        device = gpu_setup(config['gpu']['use'], config['gpu']['id'])\n",
    "        out_dir = config['out_dir']\n",
    "        \n",
    "        # GNN model\n",
    "        MODEL_NAME = config['model']\n",
    "        \n",
    "        # network parameters\n",
    "        net_params = config['net_params']\n",
    "        net_params['device'] = device\n",
    "        net_params['gpu_id'] = config['gpu']['id']\n",
    "        net_params['batch_size'] = params['batch_size']\n",
    "      \n",
    "    \n",
    "    # TSP\n",
    "    net_params['in_dim'] = dataset.train[0][0].ndata['feat'][0].shape[0]\n",
    "    net_params['in_dim_edge'] = dataset.train[0][0].edata['feat'][0].size(0)\n",
    "    num_classes = len(np.unique(np.concatenate(dataset.train[:][1])))\n",
    "    net_params['n_classes'] = num_classes\n",
    "    \n",
    "    root_log_dir = out_dir + 'logs/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n",
    "    root_ckpt_dir = out_dir + 'checkpoints/' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n",
    "    write_file_name = out_dir + 'results/result_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n",
    "    write_config_file = out_dir + 'configs/config_' + MODEL_NAME + \"_\" + DATASET_NAME + \"_GPU\" + str(config['gpu']['id']) + \"_\" + time.strftime('%Hh%Mm%Ss_on_%b_%d_%Y')\n",
    "    dirs = root_log_dir, root_ckpt_dir, write_file_name, write_config_file\n",
    "\n",
    "    if not os.path.exists(out_dir + 'results'):\n",
    "        os.makedirs(out_dir + 'results')\n",
    "        \n",
    "    if not os.path.exists(out_dir + 'configs'):\n",
    "        os.makedirs(out_dir + 'configs')\n",
    "\n",
    "    net_params['total_param'] = view_model_param(MODEL_NAME, net_params)\n",
    "    train_val_pipeline(MODEL_NAME, dataset, params, net_params, dirs)\n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "if notebook_mode==True:\n",
    "    \n",
    "    config = {}\n",
    "    # gpu config\n",
    "    gpu = {}\n",
    "    gpu['use'] = use_gpu\n",
    "    gpu['id'] = gpu_id\n",
    "    config['gpu'] = gpu\n",
    "    # GNN model, dataset, out_dir\n",
    "    config['model'] = MODEL_NAME\n",
    "    config['dataset'] = DATASET_NAME\n",
    "    config['out_dir'] = out_dir\n",
    "    # parameters\n",
    "    params = {}\n",
    "    params['seed'] = seed\n",
    "    params['epochs'] = epochs\n",
    "    params['batch_size'] = batch_size\n",
    "    params['init_lr'] = init_lr\n",
    "    params['lr_reduce_factor'] = lr_reduce_factor \n",
    "    params['lr_schedule_patience'] = lr_schedule_patience\n",
    "    params['min_lr'] = min_lr\n",
    "    params['weight_decay'] = weight_decay\n",
    "    params['print_epoch_interval'] = 5\n",
    "    params['max_time'] = max_time\n",
    "    config['params'] = params\n",
    "    # network parameters\n",
    "    config['net_params'] = net_params\n",
    "    \n",
    "    # convert to .py format\n",
    "    from utils.cleaner_main import *\n",
    "    cleaner_main('main_TSP_edge_classification')\n",
    "    \n",
    "    main(True,config)\n",
    "    \n",
    "else:\n",
    "    \n",
    "    main()\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}