test.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "slideshow": {
     "slide_type": "subslide"
    }
   },
   "outputs": [],
   "source": [
    "import time\n",
    "import pickle\n",
    "import argparse\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from torch.utils.data import DataLoader\n",
    "from esim.data import NLIDataset\n",
    "from esim.utils import correct_predictions, masked_softmax\n",
    "from sklearn import metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InterAttention(nn.Module):\n",
    "    \"\"\"\n",
    "    Compute the inter-attention with softmax attention.\n",
    "    \"\"\"\n",
    "\n",
    "    def forward(self,\n",
    "                premise_batch,\n",
    "                premise_mask,\n",
    "                hypothesis_batch,\n",
    "                hypothesis_mask):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            premise_batch: A batch of sequences of vectors representing the\n",
    "                premises in some NLI task. The batch is assumed to have the\n",
    "                size (batch, sequences, vector_dim).\n",
    "            premise_mask: A mask for the sequences in the premise batch, to\n",
    "                ignore padding data in the sequences during the computation of\n",
    "                the attention.\n",
    "            hypothesis_batch: A batch of sequences of vectors representing the\n",
    "                hypotheses in some NLI task. The batch is assumed to have the\n",
    "                size (batch, sequences, vector_dim).\n",
    "            hypothesis_mask: A mask for the sequences in the hypotheses batch,\n",
    "                to ignore padding data in the sequences during the computation\n",
    "                of the attention.\n",
    "\n",
    "        Returns:\n",
    "            attended_premises: The sequences of attention vectors for the\n",
    "                premises in the input batch.\n",
    "            attended_hypotheses: The sequences of attention vectors for the\n",
    "                hypotheses in the input batch.\n",
    "        \"\"\"\n",
    "        # Dot product between premises and hypotheses in each sequence of\n",
    "        # the batch.\n",
    "        similarity_matrix = premise_batch.bmm(hypothesis_batch.transpose(2, 1).contiguous())\n",
    "\n",
    "        # Softmax attention weights.\n",
    "        prem_hyp_attn = masked_softmax(similarity_matrix, hypothesis_mask)\n",
    "        hyp_prem_attn = masked_softmax(similarity_matrix.transpose(1, 2).contiguous(), premise_mask)\n",
    "\n",
    "        return prem_hyp_attn, hyp_prem_attn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [],
   "source": [
    "class IntraAttention(nn.Module):\n",
    "    \"\"\"\n",
    "    Compute the intra-attention with adaptive attention.\n",
    "    \"\"\"\n",
    "    def __init__(self, batch_size, hidden_size):\n",
    "        super(IntraAttention, self).__init__()\n",
    "        self.hidden_size2=hidden_size*2\n",
    "        self.W = torch.nn.Parameter(torch.randn(batch_size, self.hidden_size2, self.hidden_size2))\n",
    "        self.W.requires_grad = True\n",
    "        self.b = torch.nn.Parameter(torch.randn(batch_size, self.hidden_size2, 1))\n",
    "        self.b.requires_grad = True\n",
    "        self.v = torch.nn.Parameter(torch.randn(batch_size, 1, self.hidden_size2))\n",
    "        self.v.requires_grad = True\n",
    "        self.batch_size = batch_size\n",
    "\n",
    "\n",
    "    def forward(self,\n",
    "                premise_batch):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            premise_batch: A batch of sequences of vectors representing the\n",
    "                premises in some NLI task. The batch is assumed to have the\n",
    "                size (batch, sequences, vector_dim).\n",
    "\n",
    "        Returns:\n",
    "            beta_batch: the intra attention\n",
    "        \"\"\"\n",
    "        # premise_batch's size is [32(batch size), 25(words_num), 600(embedding size)]\n",
    "        W_batch=self.W.repeat(self.batch_size,1,1)\n",
    "        words_num=premise_batch.size()[1]\n",
    "        b_batch=b.repeat(self.batch_size,1,words_num)\n",
    "        v_batch=self.v.repeat(self.batch_size,1,1)\n",
    "        premise_batch_t=torch.transpose(premise_batch,1,2).contiguous()\n",
    "        f_batch=torch.bmm(v_batch, torch.tanh(torch.bmm(W_batch, premise_batch_t)+b_batch))\n",
    "        alpha_batch=torch.softmax(f_batch, 2)\n",
    "        alpha_batch_t=torch.transpose(alpha_batch,1,2).contiguous()\n",
    "        c_batch=torch.bmm(premise_batch_t, alpha_batch_t)\n",
    "        g_batch=torch.bmm(premise_batch, c_batch)\n",
    "        beta_batch=torch.softmax(g_batch,1)\n",
    "        \n",
    "        return beta_batch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [],
   "source": [
    "class AttentionCombination(nn.Module):\n",
    "    \"\"\"\n",
    "    Attention layer taking premises and hypotheses encoded by an RNN as input\n",
    "    and computing the soft attention between their elements.\n",
    "\n",
    "    The dot product of the encoded vectors in the premises and hypotheses is\n",
    "    first computed. The softmax of the result is then used in a weighted sum\n",
    "    of the vectors of the premises for each element of the hypotheses, and\n",
    "    conversely for the elements of the premises.\n",
    "    \"\"\"\n",
    "\n",
    "    def forward(self,\n",
    "                premise_batch,\n",
    "                premise_mask,\n",
    "                hypothesis_batch,\n",
    "                hypothesis_mask,\n",
    "                prem_hyp_attn,\n",
    "                hyp_prem_attn,\n",
    "                beta_batch):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            premise_batch: A batch of sequences of vectors representing the\n",
    "                premises in some NLI task. The batch is assumed to have the\n",
    "                size (batch, sequences, vector_dim).\n",
    "            premise_mask: A mask for the sequences in the premise batch, to\n",
    "                ignore padding data in the sequences during the computation of\n",
    "                the attention.\n",
    "            hypothesis_batch: A batch of sequences of vectors representing the\n",
    "                hypotheses in some NLI task. The batch is assumed to have the\n",
    "                size (batch, sequences, vector_dim).\n",
    "            hypothesis_mask: A mask for the sequences in the hypotheses batch,\n",
    "                to ignore padding data in the sequences during the computation\n",
    "                of the attention.\n",
    "\n",
    "        Returns:\n",
    "            attended_premises: The sequences of attention vectors for the\n",
    "                premises in the input batch.\n",
    "            attended_hypotheses: The sequences of attention vectors for the\n",
    "                hypotheses in the input batch.\n",
    "        \"\"\"\n",
    "        # Weighted sums of the hypotheses for the the premises attention,\n",
    "        # and vice-versa for the attention of the hypotheses.\n",
    "        attended_premises = weighted_sum(hypothesis_batch,\n",
    "                                         prem_hyp_attn,\n",
    "                                         premise_mask)\n",
    "        attended_hypotheses = weighted_sum(premise_batch,\n",
    "                                           hyp_prem_attn,\n",
    "                                           hypothesis_mask)\n",
    "\n",
    "        return attended_premises, attended_hypotheses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_non_zeros(hypotheses):\n",
    "    shape=hypotheses.size()\n",
    "    count_list=[]\n",
    "    for i in range(shape[0]):\n",
    "        count=0\n",
    "        for j in range(shape[1]):\n",
    "            if hypotheses[i][j].item()!=0:\n",
    "                count+=1\n",
    "        count_list.append(count)\n",
    "    return count_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_non_zeros_2D(hypotheses):\n",
    "    count_list=[]\n",
    "    for row in hypotheses:\n",
    "        count=0\n",
    "        for j in row:\n",
    "            if j!=0:\n",
    "                count+=1\n",
    "        count_list.append(count)\n",
    "    return count_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_non_zeros_3D(encoded_premises):\n",
    "    shape=encoded_premises.size()\n",
    "    print(shape)\n",
    "    count_list1=[]\n",
    "    for i in range(shape[0]):\n",
    "        count_list2=[]\n",
    "        for j in range(shape[1]):\n",
    "            count=0\n",
    "            for k in range(shape[2]):\n",
    "                if encoded_premises[i][j][k].item()!=0:\n",
    "                    count+=1\n",
    "            count_list2.append(count)\n",
    "        count_list1.append(count_list2)\n",
    "    return count_list1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "Definition of the ESIM model.\n",
    "\"\"\"\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "\n",
    "from esim.layers import RNNDropout, Seq2SeqEncoder, SoftmaxAttention\n",
    "from esim.utils import get_mask, replace_masked\n",
    "\n",
    "\n",
    "class ESIM(nn.Module):\n",
    "    \"\"\"\n",
    "    Implementation of the ESIM model presented in the paper \"Enhanced LSTM for\n",
    "    Natural Language Inference\" by Chen et al.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self,\n",
    "                 vocab_size,\n",
    "#                  batch_size,\n",
    "                 embedding_dim,\n",
    "                 hidden_size,\n",
    "                 embeddings=None,\n",
    "                 padding_idx=0,\n",
    "                 dropout=0.5,\n",
    "                 num_classes=2,\n",
    "                 device=\"cpu\"):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            vocab_size: The size of the vocabulary of embeddings in the model.\n",
    "            embedding_dim: The dimension of the word embeddings.\n",
    "            hidden_size: The size of all the hidden layers in the network.\n",
    "            embeddings: A tensor of size (vocab_size, embedding_dim) containing\n",
    "                pretrained word embeddings. If None, word embeddings are\n",
    "                initialised randomly. Defaults to None.\n",
    "            padding_idx: The index of the padding token in the premises and\n",
    "                hypotheses passed as input to the model. Defaults to 0.\n",
    "            dropout: The dropout rate to use between the layers of the network.\n",
    "                A dropout rate of 0 corresponds to using no dropout at all.\n",
    "                Defaults to 0.5.\n",
    "            num_classes: The number of classes in the output of the network.\n",
    "                Defaults to 3.\n",
    "            device: The name of the device on which the model is being\n",
    "                executed. Defaults to 'cpu'.\n",
    "        \"\"\"\n",
    "        super(ESIM, self).__init__()\n",
    "\n",
    "        self.vocab_size = vocab_size\n",
    "#         self.batch_size = batch_size\n",
    "        self.embedding_dim = embedding_dim\n",
    "        self.hidden_size = hidden_size\n",
    "        self.num_classes = num_classes\n",
    "        self.dropout = dropout\n",
    "        self.device = device\n",
    "        print(\"vocab_size:\",vocab_size)\n",
    "        print(\"embedding_dim:\",embedding_dim)\n",
    "        print(\"hidden_size:\",hidden_size)\n",
    "        print(\"num_classes:\",num_classes)\n",
    "        print(\"dropout:\",dropout)\n",
    "        print(\"device:\",device)\n",
    "\n",
    "        self._word_embedding = nn.Embedding(self.vocab_size,\n",
    "                                            self.embedding_dim,\n",
    "                                            padding_idx=padding_idx,\n",
    "                                            _weight=embeddings)\n",
    "\n",
    "        if self.dropout:\n",
    "            self._rnn_dropout = RNNDropout(p=self.dropout)\n",
    "            # self._rnn_dropout = nn.Dropout(p=self.dropout)\n",
    "\n",
    "        self._encoding = Seq2SeqEncoder(nn.LSTM,\n",
    "                                        self.embedding_dim,\n",
    "                                        self.hidden_size,\n",
    "                                        bidirectional=True)\n",
    "\n",
    "        self._attention = SoftmaxAttention()\n",
    "        self._interattention = InterAttention()\n",
    "#         self._intraattention = IntraAttention(self.batch_size, self.embedding_dim)\n",
    "\n",
    "        self._projection = nn.Sequential(nn.Linear(4*2*self.hidden_size,\n",
    "                                                   self.hidden_size),\n",
    "                                         nn.ReLU())\n",
    "\n",
    "        self._composition = Seq2SeqEncoder(nn.LSTM,\n",
    "                                           self.hidden_size,\n",
    "                                           self.hidden_size,\n",
    "                                           bidirectional=True)\n",
    "\n",
    "        self._classification = nn.Sequential(nn.Dropout(p=self.dropout),\n",
    "                                             nn.Linear(2*4*self.hidden_size,\n",
    "                                                       self.hidden_size),\n",
    "                                             nn.Tanh(),\n",
    "                                             nn.Dropout(p=self.dropout),\n",
    "                                             nn.Linear(self.hidden_size,\n",
    "                                                       self.num_classes))\n",
    "\n",
    "        # Initialize all weights and biases in the model.\n",
    "        self.apply(_init_esim_weights)\n",
    "\n",
    "    def forward(self,\n",
    "                premises,\n",
    "                premises_lengths,\n",
    "                hypotheses,\n",
    "                hypotheses_lengths):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            premises: A batch of varaible length sequences of word indices\n",
    "                representing premises. The batch is assumed to be of size\n",
    "                (batch, premises_length).\n",
    "            premises_lengths: A 1D tensor containing the lengths of the\n",
    "                premises in 'premises'.\n",
    "            hypothesis: A batch of varaible length sequences of word indices\n",
    "                representing hypotheses. The batch is assumed to be of size\n",
    "                (batch, hypotheses_length).\n",
    "            hypotheses_lengths: A 1D tensor containing the lengths of the\n",
    "                hypotheses in 'hypotheses'.\n",
    "\n",
    "        Returns:\n",
    "            logits: A tensor of size (batch, num_classes) containing the\n",
    "                logits for each output class of the model.\n",
    "            probabilities: A tensor of size (batch, num_classes) containing\n",
    "                the probabilities of each output class in the model.\n",
    "        \"\"\"\n",
    "        print(\"premises:\",premises,premises.size(),premises[0])\n",
    "        print(\"premises_lengths:\",premises_lengths)\n",
    "        premises_mask = get_mask(premises, premises_lengths).to(self.device)\n",
    "        print(\"premises_mask:\",premises_mask,premises_mask.size(),premises_mask[0],sum(premises_mask[0]))\n",
    "        hypotheses_mask = get_mask(hypotheses, hypotheses_lengths).to(self.device)\n",
    "\n",
    "        embedded_premises = self._word_embedding(premises)\n",
    "        print(\"embedded_premises:\",embedded_premises,embedded_premises.size())\n",
    "        embedded_premises_3D=get_non_zeros_3D(embedded_premises)\n",
    "        print(\"embedded_premises_3D:\",embedded_premises_3D)\n",
    "        embedded_premises_2D=get_non_zeros_2D(embedded_premises_3D)\n",
    "        print(\"embedded_premises_2D:\",embedded_premises_2D)\n",
    "        embedded_hypotheses = self._word_embedding(hypotheses)\n",
    "        \n",
    "        if self.dropout:\n",
    "            embedded_premises = self._rnn_dropout(embedded_premises)\n",
    "            embedded_hypotheses = self._rnn_dropout(embedded_hypotheses)\n",
    "\n",
    "        embedded_premises_dropout_3D=get_non_zeros_3D(embedded_premises)\n",
    "        print(\"embedded_premises_dropout_3D:\",embedded_premises_dropout_3D)\n",
    "        embedded_premises_dropout_2D=get_non_zeros_2D(embedded_premises_dropout_3D)\n",
    "        print(\"embedded_premises_dropout_2D:\",embedded_premises_dropout_2D)\n",
    "        encoded_premises = self._encoding(embedded_premises,\n",
    "                                          premises_lengths)\n",
    "        print(\"encoded_premises:\",encoded_premises,encoded_premises.size())\n",
    "        encoded_premises_3D=get_non_zeros_3D(encoded_premises)\n",
    "        print(\"encoded_premises_3D:\",encoded_premises_3D)\n",
    "        encoded_premises_2D=get_non_zeros_2D(encoded_premises_3D)\n",
    "        print(\"encoded_premises_2D:\",encoded_premises_2D)\n",
    "        encoded_hypotheses = self._encoding(embedded_hypotheses,\n",
    "                                            hypotheses_lengths)\n",
    "        print(\"encoded_hypotheses:\",encoded_hypotheses,encoded_hypotheses.size())\n",
    "\n",
    "        prem_hyp_attn, hyp_prem_attn = self._interattention(encoded_premises, premises_mask,\n",
    "                            encoded_hypotheses, hypotheses_mask)\n",
    "        print(\"prem_hyp_attn:\",prem_hyp_attn,prem_hyp_attn.size())\n",
    "        print(\"hyp_prem_attn:\",hyp_prem_attn,hyp_prem_attn.size())\n",
    "#         beta_premises=self._intraattention(encoded_premises)\n",
    "#         beta_hypotheses=self._intraattention(encoded_hypotheses)\n",
    "        attended_premises, attended_hypotheses =\\\n",
    "            self._attention(encoded_premises, premises_mask,\n",
    "                            encoded_hypotheses, hypotheses_mask)\n",
    "        print(\"attended_premises:\",attended_premises,attended_premises.size())\n",
    "\n",
    "        enhanced_premises = torch.cat([encoded_premises,\n",
    "                                       attended_premises,\n",
    "                                       encoded_premises - attended_premises,\n",
    "                                       encoded_premises * attended_premises],\n",
    "                                      dim=-1)\n",
    "        enhanced_hypotheses = torch.cat([encoded_hypotheses,\n",
    "                                         attended_hypotheses,\n",
    "                                         encoded_hypotheses - attended_hypotheses,\n",
    "                                         encoded_hypotheses * attended_hypotheses],\n",
    "                                        dim=-1)\n",
    "\n",
    "        projected_premises = self._projection(enhanced_premises)\n",
    "        projected_hypotheses = self._projection(enhanced_hypotheses)\n",
    "\n",
    "        if self.dropout:\n",
    "            projected_premises = self._rnn_dropout(projected_premises)\n",
    "            projected_hypotheses = self._rnn_dropout(projected_hypotheses)\n",
    "\n",
    "        v_ai = self._composition(projected_premises, premises_lengths)\n",
    "        v_bj = self._composition(projected_hypotheses, hypotheses_lengths)\n",
    "\n",
    "        v_a_avg = torch.sum(v_ai * premises_mask.unsqueeze(1)\n",
    "                                                .transpose(2, 1), dim=1)\\\n",
    "            / torch.sum(premises_mask, dim=1, keepdim=True)\n",
    "        v_b_avg = torch.sum(v_bj * hypotheses_mask.unsqueeze(1)\n",
    "                                                  .transpose(2, 1), dim=1)\\\n",
    "            / torch.sum(hypotheses_mask, dim=1, keepdim=True)\n",
    "\n",
    "        v_a_max, _ = replace_masked(v_ai, premises_mask, -1e7).max(dim=1)\n",
    "        v_b_max, _ = replace_masked(v_bj, hypotheses_mask, -1e7).max(dim=1)\n",
    "\n",
    "        v = torch.cat([v_a_avg, v_a_max, v_b_avg, v_b_max], dim=1)\n",
    "\n",
    "        logits = self._classification(v)\n",
    "        probabilities = nn.functional.softmax(logits, dim=-1)\n",
    "\n",
    "        return logits, probabilities\n",
    "\n",
    "\n",
    "def _init_esim_weights(module):\n",
    "    \"\"\"\n",
    "    Initialise the weights of the ESIM model.\n",
    "    \"\"\"\n",
    "    if isinstance(module, nn.Linear):\n",
    "        nn.init.xavier_uniform_(module.weight.data)\n",
    "        nn.init.constant_(module.bias.data, 0.0)\n",
    "\n",
    "    elif isinstance(module, nn.LSTM):\n",
    "        nn.init.xavier_uniform_(module.weight_ih_l0.data)\n",
    "        nn.init.orthogonal_(module.weight_hh_l0.data)\n",
    "        nn.init.constant_(module.bias_ih_l0.data, 0.0)\n",
    "        nn.init.constant_(module.bias_hh_l0.data, 0.0)\n",
    "        hidden_size = module.bias_hh_l0.data.shape[0] // 4\n",
    "        module.bias_hh_l0.data[hidden_size:(2*hidden_size)] = 1.0\n",
    "\n",
    "        if (module.bidirectional):\n",
    "            nn.init.xavier_uniform_(module.weight_ih_l0_reverse.data)\n",
    "            nn.init.orthogonal_(module.weight_hh_l0_reverse.data)\n",
    "            nn.init.constant_(module.bias_ih_l0_reverse.data, 0.0)\n",
    "            nn.init.constant_(module.bias_hh_l0_reverse.data, 0.0)\n",
    "            module.bias_hh_l0_reverse.data[hidden_size:(2*hidden_size)] = 1.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [],
   "source": [
    "def test(model, dataloader):\n",
    "    \"\"\"\n",
    "    Test the accuracy of a model on some labelled test dataset.\n",
    "\n",
    "    Args:\n",
    "        model: The torch module on which testing must be performed.\n",
    "        dataloader: A DataLoader object to iterate over some dataset.\n",
    "\n",
    "    Returns:\n",
    "        batch_time: The average time to predict the classes of a batch.\n",
    "        total_time: The total time to process the whole dataset.\n",
    "        accuracy: The accuracy of the model on the input data.\n",
    "    \"\"\"\n",
    "    # Switch the model to eval mode.\n",
    "    model.eval()\n",
    "    device = model.device\n",
    "\n",
    "    time_start = time.time()\n",
    "    batch_time = 0.0\n",
    "    accuracy = 0.0\n",
    "\n",
    "    all_labels = []\n",
    "    all_out_classes = []\n",
    "\n",
    "    # Deactivate autograd for evaluation.\n",
    "    with torch.no_grad():\n",
    "        for batch in dataloader:\n",
    "            batch_start = time.time()\n",
    "\n",
    "            # Move input and output data to the GPU if one is used.\n",
    "            premises = batch[\"premise\"].to(device)\n",
    "            premises_lengths = batch[\"premise_length\"].to(device)\n",
    "            hypotheses = batch[\"hypothesis\"].to(device)\n",
    "            hypotheses_lengths = batch[\"hypothesis_length\"].to(device)\n",
    "            labels = batch[\"label\"]\n",
    "            all_labels.extend(labels.tolist())\n",
    "            labels = labels.to(device)\n",
    "\n",
    "            _, probs = model(premises,\n",
    "                             premises_lengths,\n",
    "                             hypotheses,\n",
    "                             hypotheses_lengths)\n",
    "            _, out_classes = probs.max(dim=1)\n",
    "            all_out_classes.extend(out_classes.tolist())\n",
    "\n",
    "            accuracy += correct_predictions(probs, labels)\n",
    "            batch_time += time.time() - batch_start\n",
    "            break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [],
   "source": [
    "def main(test_file, pretrained_file, batch_size=32):\n",
    "    \"\"\"\n",
    "    Test the ESIM model with pretrained weights on some dataset.\n",
    "\n",
    "    Args:\n",
    "        test_file: The path to a file containing preprocessed NLI data.\n",
    "        pretrained_file: The path to a checkpoint produced by the\n",
    "            'train_model' script.\n",
    "        vocab_size: The number of words in the vocabulary of the model\n",
    "            being tested.\n",
    "        embedding_dim: The size of the embeddings in the model.\n",
    "        hidden_size: The size of the hidden layers in the model. Must match\n",
    "            the size used during training. Defaults to 300.\n",
    "        num_classes: The number of classes in the output of the model. Must\n",
    "            match the value used during training. Defaults to 3.\n",
    "        batch_size: The size of the batches used for testing. Defaults to 32.\n",
    "    \"\"\"\n",
    "    print(20 * \"=\", \" Preparing for testing \", 20 * \"=\")\n",
    "\n",
    "    #gpu\n",
    "    device = torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\")\n",
    "    checkpoint = torch.load(pretrained_file)\n",
    "    \n",
    "    #cpu\n",
    "#     device = torch.device(\"cpu\")\n",
    "#     checkpoint = torch.load(pretrained_file, map_location=lambda storage, loc: storage)\n",
    "\n",
    "    # Retrieving model parameters from checkpoint.\n",
    "    vocab_size = checkpoint[\"model\"][\"_word_embedding.weight\"].size(0)\n",
    "    embedding_dim = checkpoint[\"model\"]['_word_embedding.weight'].size(1)\n",
    "    hidden_size = checkpoint[\"model\"][\"_projection.0.weight\"].size(0)\n",
    "    num_classes = checkpoint[\"model\"][\"_classification.4.weight\"].size(0)\n",
    "\n",
    "    print(\"\\t* Loading test data...\")\n",
    "    with open(test_file, \"rb\") as pkl:\n",
    "        test_data = NLIDataset(pickle.load(pkl))\n",
    "\n",
    "    test_loader = DataLoader(test_data, shuffle=False, batch_size=batch_size)\n",
    "\n",
    "    print(\"\\t* Building model...\")\n",
    "    model = ESIM(vocab_size,\n",
    "#                  batch_size,\n",
    "                 embedding_dim,\n",
    "                 hidden_size,\n",
    "                 num_classes=num_classes,\n",
    "                 device=device).to(device)\n",
    "\n",
    "    model.load_state_dict(checkpoint[\"model\"])\n",
    "\n",
    "    print(20 * \"=\",\n",
    "          \" Testing ESIM model on device: {} \".format(device),\n",
    "          20 * \"=\")\n",
    "    test(model, test_loader)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================  Preparing for testing  ====================\n",
      "\t* Loading test data...\n",
      "\t* Building model...\n",
      "vocab_size: 128447\n",
      "embedding_dim: 300\n",
      "hidden_size: 300\n",
      "num_classes: 2\n",
      "dropout: 0.5\n",
      "device: cuda:0\n",
      "====================  Testing ESIM model on device: cuda:0  ====================\n",
      "premises: tensor([[  2,   6,  43,  ...,   0,   0,   0],\n",
      "        [  2,  76, 206,  ...,   0,   0,   0],\n",
      "        [  2,  11,  20,  ...,   0,   0,   0],\n",
      "        ...,\n",
      "        [  2,  11, 162,  ...,   0,   0,   0],\n",
      "        [  2,   6,   8,  ...,   0,   0,   0],\n",
      "        [  2,   6,   8,  ...,   0,   0,   0]], device='cuda:0') torch.Size([32, 71]) tensor([   2,    6,   43,    7,   14,   10,  689, 1534,   12,  394,    4,    3,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0,\n",
      "           0,    0,    0,    0,    0,    0,    0,    0,    0,    0,    0],\n",
      "       device='cuda:0')\n",
      "premises_lengths: tensor([12, 12,  9,  9, 11, 13, 18,  8, 11, 11, 16,  9, 14, 23,  9, 14, 10,  8,\n",
      "        13, 14, 11, 25, 10, 10, 12, 11, 14, 15,  9, 11, 13, 17],\n",
      "       device='cuda:0')\n",
      "premises_mask: tensor([[1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "         1., 1., 1., 1., 1., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.,\n",
      "         1., 1., 1., 1., 1., 1., 1.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.],\n",
      "        [1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0.,\n",
      "         0., 0., 0., 0., 0., 0., 0.]], device='cuda:0') torch.Size([32, 25]) tensor([1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 0., 0., 0., 0., 0., 0.,\n",
      "        0., 0., 0., 0., 0., 0., 0.], device='cuda:0') tensor(12., device='cuda:0')\n",
      "embedded_premises: tensor([[[ 0.8348, -1.6945, -0.1358,  ..., -0.0093, -0.5467, -0.8310],\n",
      "         [-0.0677,  0.1734, -0.1585,  ..., -0.0101,  0.3993, -0.1043],\n",
      "         [ 0.0518,  0.2009, -0.5014,  ..., -0.7342,  0.5498, -0.0517],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[ 0.8348, -1.6945, -0.1358,  ..., -0.0093, -0.5467, -0.8310],\n",
      "         [-0.2788,  0.0265, -0.0899,  ...,  0.1175,  0.6031,  0.0351],\n",
      "         [ 0.1322,  0.1786,  0.3311,  ..., -0.3074, -0.1526, -0.6386],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[ 0.8348, -1.6945, -0.1358,  ..., -0.0093, -0.5467, -0.8310],\n",
      "         [-0.5439,  0.4943, -0.3240,  ...,  0.4797,  0.2097,  0.2665],\n",
      "         [-0.2126,  0.2967, -0.3012,  ..., -0.2815,  0.3559,  0.1709],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[ 0.8348, -1.6945, -0.1358,  ..., -0.0093, -0.5467, -0.8310],\n",
      "         [-0.5439,  0.4943, -0.3240,  ...,  0.4797,  0.2097,  0.2665],\n",
      "         [ 0.1206,  0.1605, -0.4313,  ..., -0.0754, -0.2768,  0.0665],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[ 0.8348, -1.6945, -0.1358,  ..., -0.0093, -0.5467, -0.8310],\n",
      "         [-0.0677,  0.1734, -0.1585,  ..., -0.0101,  0.3993, -0.1043],\n",
      "         [ 0.0271,  0.4079, -0.0936,  ..., -0.3907, -0.2550,  0.0795],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[ 0.8348, -1.6945, -0.1358,  ..., -0.0093, -0.5467, -0.8310],\n",
      "         [-0.0677,  0.1734, -0.1585,  ..., -0.0101,  0.3993, -0.1043],\n",
      "         [ 0.0271,  0.4079, -0.0936,  ..., -0.3907, -0.2550,  0.0795],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]]],\n",
      "       device='cuda:0') torch.Size([32, 71, 300])\n",
      "torch.Size([32, 71, 300])\n",
      "embedded_premises_3D: [[300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
      "embedded_premises_2D: [12, 12, 9, 9, 11, 13, 18, 8, 11, 11, 16, 9, 14, 23, 9, 14, 10, 8, 13, 14, 11, 25, 10, 10, 12, 11, 14, 15, 9, 11, 13, 17]\n",
      "torch.Size([32, 71, 300])\n",
      "embedded_premises_dropout_3D: [[300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 300, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
      "embedded_premises_dropout_2D: [12, 12, 9, 9, 11, 13, 18, 8, 11, 11, 16, 9, 14, 23, 9, 14, 10, 8, 13, 14, 11, 25, 10, 10, 12, 11, 14, 15, 9, 11, 13, 17]\n",
      "encoded_premises: tensor([[[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -5.4054e-01,\n",
      "           1.2787e-03, -9.2722e-03],\n",
      "         [ 1.1285e-02,  3.8569e-02, -2.2332e-02,  ..., -4.9448e-01,\n",
      "          -5.2409e-02,  5.4005e-02],\n",
      "         [ 7.8599e-02, -1.7487e-02, -6.9358e-03,  ..., -2.9580e-01,\n",
      "          -1.0149e-01,  2.1289e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -5.8502e-02,\n",
      "           7.9375e-02,  5.0712e-02],\n",
      "         [ 1.1739e-02,  1.2942e-02, -8.4645e-03,  ..., -5.8990e-02,\n",
      "           1.7680e-01,  2.7989e-01],\n",
      "         [ 1.1469e-02, -7.9014e-02, -6.8430e-03,  ..., -5.6960e-03,\n",
      "           5.0429e-01,  1.9876e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -4.6929e-01,\n",
      "           1.9260e-02, -3.1866e-02],\n",
      "         [ 4.7479e-03,  1.7939e-03, -5.7994e-02,  ..., -4.4894e-01,\n",
      "           1.2460e-02,  6.5918e-02],\n",
      "         [ 1.0311e-02, -3.6480e-04, -2.7885e-02,  ..., -1.9096e-01,\n",
      "          -5.9211e-02,  1.3224e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -6.6577e-01,\n",
      "           3.6574e-03,  1.8616e-02],\n",
      "         [ 4.7479e-03,  1.7939e-03, -5.7994e-02,  ..., -7.0695e-01,\n",
      "          -2.9958e-02,  1.3486e-01],\n",
      "         [ 2.8876e-02,  5.3424e-05, -1.9253e-02,  ..., -3.1221e-01,\n",
      "          -1.4753e-01, -8.9768e-02],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -6.5041e-03,\n",
      "           4.2628e-02, -7.8253e-02],\n",
      "         [ 1.1285e-02,  3.8569e-02, -2.2332e-02,  ..., -3.3161e-03,\n",
      "           1.2263e-02,  6.1674e-02],\n",
      "         [ 1.2194e-02, -8.3124e-03,  7.1329e-04,  ..., -4.4870e-04,\n",
      "          -1.8203e-01,  2.8278e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -4.9902e-01,\n",
      "           2.7760e-02, -4.7625e-02],\n",
      "         [ 1.1285e-02,  3.8569e-02, -2.2332e-02,  ..., -3.8990e-01,\n",
      "          -1.7000e-02,  7.3292e-02],\n",
      "         [ 1.2194e-02, -8.3124e-03,  7.1329e-04,  ..., -1.0585e-01,\n",
      "          -3.3637e-01,  1.8775e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]]], device='cuda:0') torch.Size([32, 25, 600])\n",
      "torch.Size([32, 25, 600])\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/rongz/anaconda3/lib/python3.6/site-packages/esim/utils.py:41: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than tensor.new_tensor(sourceTensor).\n",
      "  sequences_lengths.new_tensor(torch.arange(0, len(sequences_lengths)))\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "encoded_premises_3D: [[600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 600, 0, 0, 0, 0, 0, 0, 0, 0]]\n",
      "encoded_premises_2D: [12, 12, 9, 9, 11, 13, 18, 8, 11, 11, 16, 9, 14, 23, 9, 14, 10, 8, 13, 14, 11, 25, 10, 10, 12, 11, 14, 15, 9, 11, 13, 17]\n",
      "encoded_hypotheses: tensor([[[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -5.7516e-01,\n",
      "          -1.6266e-03, -3.4965e-02],\n",
      "         [ 4.7479e-03,  1.7939e-03, -5.7994e-02,  ..., -4.8976e-01,\n",
      "          -2.4047e-02,  8.4553e-02],\n",
      "         [ 1.2991e-02,  2.4097e-04, -1.1024e-02,  ..., -1.7458e-01,\n",
      "          -1.0438e-02, -3.8154e-02],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -2.8034e-01,\n",
      "           6.2853e-02,  9.6896e-02],\n",
      "         [ 1.1739e-02,  1.2942e-02, -8.4645e-03,  ..., -2.4166e-01,\n",
      "           1.2888e-01,  3.6894e-01],\n",
      "         [ 1.1469e-02, -7.9014e-02, -6.8430e-03,  ..., -8.3039e-02,\n",
      "           3.2844e-01,  1.0515e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -4.1224e-01,\n",
      "           1.7047e-02, -4.5035e-02],\n",
      "         [ 4.7479e-03,  1.7939e-03, -5.7994e-02,  ..., -3.7850e-01,\n",
      "           4.1407e-03,  6.6716e-02],\n",
      "         [ 1.2991e-02,  2.4097e-04, -1.1024e-02,  ..., -1.3519e-01,\n",
      "          -1.0303e-02, -5.9935e-02],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -7.9673e-01,\n",
      "           9.1784e-03, -1.5037e-02],\n",
      "         [ 1.6875e-01,  5.6955e-03, -8.4483e-02,  ..., -6.7831e-01,\n",
      "          -3.3241e-03, -4.4260e-02],\n",
      "         [ 1.5561e-02,  1.1482e-01, -3.0691e-01,  ..., -4.5323e-01,\n",
      "          -3.8985e-01, -9.2195e-02],\n",
      "         ...,\n",
      "         [ 1.1477e-03,  2.4211e-02, -4.6120e-03,  ..., -7.4807e-01,\n",
      "          -7.2896e-04,  1.4317e-03],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -8.2817e-03,\n",
      "           4.4726e-02, -8.3980e-02],\n",
      "         [ 1.1285e-02,  3.8569e-02, -2.2332e-02,  ..., -3.5402e-03,\n",
      "           1.6790e-02,  6.8174e-02],\n",
      "         [ 1.2194e-02, -8.3124e-03,  7.1329e-04,  ..., -7.5431e-04,\n",
      "          -1.6124e-01,  3.2376e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -9.5547e-01,\n",
      "           6.1612e-02, -3.2314e-02],\n",
      "         [ 4.7479e-03,  1.7939e-03, -5.7994e-02,  ..., -9.2981e-01,\n",
      "           7.8597e-02,  7.9214e-02],\n",
      "         [ 7.7428e-02,  4.9459e-03, -4.1619e-02,  ..., -6.7914e-01,\n",
      "           1.1409e-02, -1.3894e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  0.0000e+00,\n",
      "           0.0000e+00,  0.0000e+00]]], device='cuda:0') torch.Size([32, 32, 600])\n",
      "prem_hyp_attn: tensor([[[1.0000e+00, 7.4624e-12, 2.4159e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [6.9497e-05, 9.9952e-01, 3.5165e-04,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.9370e-05, 2.6037e-04, 9.9531e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 1.4261e-11, 5.7601e-13,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.8353e-09, 1.0000e+00, 8.8564e-08,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.2887e-09, 8.0234e-07, 1.0000e+00,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [1.0000e-01, 1.0000e-01, 1.0000e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.0000e-01, 1.0000e-01, 1.0000e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.0000e-01, 1.0000e-01, 1.0000e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 2.3542e-11, 5.9019e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [4.6543e-08, 9.9998e-01, 1.6532e-05,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.1660e-07, 1.5350e-04, 9.9932e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [1.0000e-01, 1.0000e-01, 1.0000e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.0000e-01, 1.0000e-01, 1.0000e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.0000e-01, 1.0000e-01, 1.0000e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[1.0000e+00, 7.5562e-11, 1.1326e-11,  ..., 7.8218e-09,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [3.1390e-03, 5.5442e-04, 3.4392e-05,  ..., 4.9251e-03,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.3133e-02, 8.5992e-02, 7.2154e-02,  ..., 7.0505e-03,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [3.3333e-02, 3.3333e-02, 3.3333e-02,  ..., 3.3333e-02,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [3.3333e-02, 3.3333e-02, 3.3333e-02,  ..., 3.3333e-02,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [3.3333e-02, 3.3333e-02, 3.3333e-02,  ..., 3.3333e-02,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 1.1024e-11, 5.9715e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.4308e-06, 9.9981e-01, 9.2860e-05,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.4048e-06, 1.1119e-05, 9.9976e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [7.6923e-02, 7.6923e-02, 7.6923e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.6923e-02, 7.6923e-02, 7.6923e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.6923e-02, 7.6923e-02, 7.6923e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 9.0272e-12, 1.1020e-12,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.0984e-03, 9.9555e-01, 5.1162e-04,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.2107e-01, 2.7302e-02, 4.2504e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [7.1429e-02, 7.1429e-02, 7.1429e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.1429e-02, 7.1429e-02, 7.1429e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.1429e-02, 7.1429e-02, 7.1429e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]]], device='cuda:0') torch.Size([32, 25, 32])\n",
      "hyp_prem_attn: tensor([[[1.0000e+00, 1.5872e-11, 2.3981e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [3.2628e-05, 9.9807e-01, 9.2955e-04,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.2155e-05, 7.3647e-05, 7.4526e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 1.5238e-11, 1.6190e-13,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.3326e-09, 1.0000e+00, 2.9183e-08,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.1429e-09, 2.4349e-06, 1.0000e+00,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [8.3333e-02, 8.3333e-02, 8.3333e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 4.1846e-11, 8.9624e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.6184e-08, 9.9998e-01, 2.1352e-05,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [4.7214e-07, 1.1891e-04, 9.9983e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [1.1111e-01, 1.1111e-01, 1.1111e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.1111e-01, 1.1111e-01, 1.1111e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.1111e-01, 1.1111e-01, 1.1111e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[1.0000e+00, 3.1076e-10, 2.1679e-12,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [3.7767e-01, 2.7433e-01, 7.0948e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.8308e-01, 5.5037e-02, 1.9253e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [1.0578e-10, 6.5941e-12, 1.5740e-14,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [9.0909e-02, 9.0909e-02, 9.0909e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [9.0909e-02, 9.0909e-02, 9.0909e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 1.0696e-11, 5.0363e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [2.5053e-06, 9.9983e-01, 9.0588e-05,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.6656e-06, 1.1398e-05, 9.9975e-01,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [7.6923e-02, 7.6923e-02, 7.6923e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.6923e-02, 7.6923e-02, 7.6923e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [7.6923e-02, 7.6923e-02, 7.6923e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]],\n",
      "\n",
      "        [[1.0000e+00, 1.4302e-11, 4.6636e-11,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [1.3124e-03, 9.8649e-01, 2.2545e-04,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [4.9378e-03, 1.5624e-02, 1.0817e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         ...,\n",
      "         [5.8824e-02, 5.8824e-02, 5.8824e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [5.8824e-02, 5.8824e-02, 5.8824e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00],\n",
      "         [5.8824e-02, 5.8824e-02, 5.8824e-02,  ..., 0.0000e+00,\n",
      "          0.0000e+00, 0.0000e+00]]], device='cuda:0') torch.Size([32, 32, 25])\n",
      "similarity_matrix: tensor([[[33.9158,  8.2947,  9.4694,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 9.0493, 18.6231, 10.6707,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 9.4621, 11.6442, 19.8929,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[35.7491, 10.7756,  7.5665,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [10.8419, 29.5065, 13.2670,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 6.2973, 12.1569, 26.1926,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[32.4318,  7.9596,  8.8787,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 8.5348, 25.4177, 14.4075,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 9.2964, 14.6634, 23.4445,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[28.7207,  5.4146,  3.5167,  ..., 10.0543,  0.0000,  0.0000],\n",
      "         [ 6.8287,  5.0949,  2.3148,  ...,  7.2791,  0.0000,  0.0000],\n",
      "         [ 1.8634,  3.7425,  3.5671,  ...,  1.2414,  0.0000,  0.0000],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[32.4313,  7.2003,  8.8898,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 7.1701, 20.0972, 10.8130,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 8.7195, 10.7882, 22.1949,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]],\n",
      "\n",
      "        [[33.0795,  7.6488,  5.5457,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 8.1089, 14.2711,  6.6976,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 9.2909,  5.8872,  6.3299,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         ...,\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000],\n",
      "         [ 0.0000,  0.0000,  0.0000,  ...,  0.0000,  0.0000,  0.0000]]],\n",
      "       device='cuda:0') torch.Size([32, 25, 32]) tensor([[ 3.3916e+01,  8.2947e+00,  9.4694e+00,  9.1447e+00,  2.6815e+00,\n",
      "          4.7125e+00,  4.9206e+00,  6.4053e+00,  1.8766e+00,  3.5781e+00,\n",
      "          5.4223e+00,  8.9269e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 9.0493e+00,  1.8623e+01,  1.0671e+01,  8.7471e+00, -7.5870e-01,\n",
      "          3.4543e+00,  1.5691e+00,  2.8649e+00, -1.0832e+00,  1.9796e+00,\n",
      "          6.7392e+00,  4.6237e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 9.4621e+00,  1.1644e+01,  1.9893e+01,  1.4472e+01,  6.3226e+00,\n",
      "          3.1758e+00,  3.5397e+00,  2.8372e+00, -4.3427e-01, -5.4107e-01,\n",
      "          4.2424e+00,  4.6882e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 9.1680e+00,  1.1325e+01,  1.2403e+01,  2.2441e+01,  6.4364e+00,\n",
      "          6.5430e+00,  3.7944e+00,  3.9384e+00,  1.8843e+00,  1.2639e+00,\n",
      "          2.7455e+00,  5.9331e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 8.0148e+00,  9.9082e+00,  1.8802e+01,  1.5574e+01,  9.4902e+00,\n",
      "          6.7771e+00,  4.2930e+00,  4.0441e+00,  1.5625e+00, -9.7409e-03,\n",
      "          3.3299e+00,  6.1775e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 7.9908e+00,  9.6251e+00,  1.4644e+01,  1.7867e+01,  8.4616e+00,\n",
      "          7.3147e+00,  5.4043e+00,  5.4099e+00,  1.0781e-01, -7.8990e-01,\n",
      "          2.8629e+00,  6.4755e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 3.2730e+00,  3.3327e+00,  4.9274e+00,  9.4712e+00,  1.5913e+01,\n",
      "          1.0249e+01,  3.3037e+00,  3.6429e+00,  6.9502e+00,  7.9871e-01,\n",
      "          1.3727e+00,  2.7753e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 4.4353e+00,  1.9100e+00,  2.6021e+00,  3.6102e+00,  6.4107e+00,\n",
      "          1.7784e+01,  8.5830e+00,  7.0934e+00,  7.7404e+00,  4.8884e+00,\n",
      "          4.3400e+00,  4.1865e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 5.2046e+00,  1.5138e+00,  2.8796e+00,  3.4564e+00,  1.7442e+00,\n",
      "          7.2710e+00,  1.6624e+01,  1.3557e+01,  6.6804e+00,  6.9418e+00,\n",
      "          6.6447e+00,  6.3859e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 1.3788e+00,  2.2730e-01, -1.3757e+00,  6.1979e-01, -2.5020e-01,\n",
      "          3.1483e+00,  5.9965e+00,  8.6405e+00,  1.1256e+01,  2.0529e+01,\n",
      "          7.6619e+00,  3.5247e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 4.4079e+00,  4.6035e+00,  3.5892e+00,  2.9269e+00,  8.7959e-01,\n",
      "          5.9485e+00,  9.1439e+00,  1.1758e+01,  1.0398e+01,  1.0539e+01,\n",
      "          2.2795e+01,  9.8823e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 9.3910e+00,  4.7511e+00,  6.4666e+00,  7.0902e+00,  2.7125e+00,\n",
      "          5.1518e+00,  7.1142e+00,  1.1576e+01,  5.7492e+00,  4.9108e+00,\n",
      "          9.7990e+00,  3.4695e+01,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00],\n",
      "        [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,  0.0000e+00,\n",
      "          0.0000e+00,  0.0000e+00]], device='cuda:0')\n",
      "attended_premises: tensor([[[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -5.7516e-01,\n",
      "          -1.6266e-03, -3.4965e-02],\n",
      "         [ 4.7528e-03,  1.7988e-03, -5.7970e-02,  ..., -4.8965e-01,\n",
      "          -2.4044e-02,  8.4500e-02],\n",
      "         [ 1.3116e-02,  6.7998e-04, -1.0980e-02,  ..., -1.7578e-01,\n",
      "          -1.0810e-02, -3.7606e-02],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -2.8034e-01,\n",
      "           6.2853e-02,  9.6896e-02],\n",
      "         [ 1.1739e-02,  1.2942e-02, -8.4647e-03,  ..., -2.4166e-01,\n",
      "           1.2887e-01,  3.6894e-01],\n",
      "         [ 1.1469e-02, -7.9014e-02, -6.8430e-03,  ..., -8.3039e-02,\n",
      "           3.2844e-01,  1.0515e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -4.1224e-01,\n",
      "           1.7047e-02, -4.5035e-02],\n",
      "         [ 4.7481e-03,  1.7939e-03, -5.7993e-02,  ..., -3.7850e-01,\n",
      "           4.1404e-03,  6.6714e-02],\n",
      "         [ 1.3005e-02,  2.9324e-04, -1.1024e-02,  ..., -1.3521e-01,\n",
      "          -1.0401e-02, -5.9861e-02],\n",
      "         ...,\n",
      "         [-0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00],\n",
      "         [-0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00],\n",
      "         [-0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        ...,\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -7.9673e-01,\n",
      "           9.1784e-03, -1.5037e-02],\n",
      "         [ 4.0575e-01, -2.3528e-02, -3.7478e-03,  ..., -1.4537e-01,\n",
      "          -6.9058e-02,  5.0329e-02],\n",
      "         [-3.3924e-02,  1.2995e-01, -3.4473e-02,  ..., -1.7149e-01,\n",
      "          -2.2418e-01, -6.0051e-02],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -8.2817e-03,\n",
      "           4.4726e-02, -8.3980e-02],\n",
      "         [ 1.1287e-02,  3.8561e-02, -2.2328e-02,  ..., -3.5437e-03,\n",
      "           1.6791e-02,  6.8216e-02],\n",
      "         [ 1.2196e-02, -8.3119e-03,  7.1304e-04,  ..., -7.6322e-04,\n",
      "          -1.6115e-01,  3.2375e-01],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00, -0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00,  0.0000e+00]],\n",
      "\n",
      "        [[ 1.7491e-04,  3.8436e-04, -2.6168e-05,  ..., -9.5547e-01,\n",
      "           6.1612e-02, -3.2314e-02],\n",
      "         [ 4.8013e-03,  1.8222e-03, -5.8141e-02,  ..., -9.2841e-01,\n",
      "           7.8365e-02,  7.8690e-02],\n",
      "         [ 4.5226e-03,  3.6509e-03, -7.7981e-03,  ..., -8.9911e-01,\n",
      "           5.1250e-02, -3.3133e-02],\n",
      "         ...,\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00],\n",
      "         [ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ..., -0.0000e+00,\n",
      "          -0.0000e+00, -0.0000e+00]]], device='cuda:0') torch.Size([32, 25, 600])\n"
     ]
    }
   ],
   "source": [
    "test_data=\"/home/rongz/paraphrase_recognition/src/data/preprocessed/quora/test_data.pkl\"\n",
    "checkpoint=\"/home/rongz/paraphrase_recognition/exp/saved_model/ESIM/qqp/best.pth.tar\"\n",
    "batch_size=32\n",
    "main(test_data,\n",
    "     checkpoint,\n",
    "     batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "a = torch.rand(3,2,2)\n",
    "b = torch.rand(3,2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "c = b.unsqueeze(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0071, 0.0527],\n",
       "         [0.2866, 0.1333]],\n",
       "\n",
       "        [[0.4576, 0.5175],\n",
       "         [0.0553, 0.0382]],\n",
       "\n",
       "        [[0.3357, 0.4629],\n",
       "         [0.1772, 0.4650]]])"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a * c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[[0.0670, 0.4984],\n",
       "         [0.9579, 0.4455]],\n",
       "\n",
       "        [[0.8613, 0.9739],\n",
       "         [0.7200, 0.4967]],\n",
       "\n",
       "        [[0.5814, 0.8015],\n",
       "         [0.2819, 0.7397]]])"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "a"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tensor([[0.1057, 0.2992],\n",
       "        [0.5314, 0.0768],\n",
       "        [0.5775, 0.6286]])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "b"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}