11785_HW1_submission/HW1P2_S24_final_submission.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"F9ERgBpbcMmB"},"source":["# HW1: Frame-Level Speech Recognition"]},{"cell_type":"markdown","metadata":{"id":"CLkH6GMGcWcE"},"source":["In this homework, you will be working with MFCC data consisting of 27 features at each time step/frame. Your model should be able to recognize the phoneme occured in that frame."]},{"cell_type":"markdown","metadata":{"id":"z4vZbDmJvMp1"},"source":["# Libraries"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"rwYu9sSUnSho"},"outputs":[],"source":["!pip install torchsummaryX wandb --quiet"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"qI4qfx7tiBZt","outputId":"bc29b725-05db-49da-fe58-3780fe46eba3","executionInfo":{"status":"ok","timestamp":1707420954578,"user_tz":300,"elapsed":6237,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["Device:  cuda\n"]}],"source":["import torch\n","import numpy as np\n","from torchsummaryX import summary\n","import sklearn\n","import gc\n","import zipfile\n","import pandas as pd\n","from tqdm.auto import tqdm\n","import os\n","import datetime\n","import wandb\n","device = 'cuda' if torch.cuda.is_available() else 'cpu'\n","print(\"Device: \", device)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"8yBgXjKV1O0Z"},"outputs":[],"source":["### If you are using colab, you can import google drive to save model checkpoints in a folder\n","# from google.colab import drive\n","# drive.mount('/content/drive')"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"N-9qE20hmCgQ"},"outputs":[],"source":["### PHONEME LIST\n","PHONEMES = [\n","            '[SIL]',   'AA',    'AE',    'AH',    'AO',    'AW',    'AY',\n","            'B',     'CH',    'D',     'DH',    'EH',    'ER',    'EY',\n","            'F',     'G',     'HH',    'IH',    'IY',    'JH',    'K',\n","            'L',     'M',     'N',     'NG',    'OW',    'OY',    'P',\n","            'R',     'S',     'SH',    'T',     'TH',    'UH',    'UW',\n","            'V',     'W',     'Y',     'Z',     'ZH',    '[SOS]', '[EOS]']"]},{"cell_type":"markdown","metadata":{"id":"ZIi0Big7vPa9"},"source":["# Kaggle"]},{"cell_type":"markdown","metadata":{"id":"BBCbeRhixGM7"},"source":["This section contains code that helps you install kaggle's API, creating kaggle.json with you username and API key details. Make sure to input those in the given code to ensure you can download data from the competition successfully."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"TPBUd7Cnl-Rx"},"outputs":[],"source":["# !pip install --upgrade --force-reinstall --no-deps kaggle==1.5.8\n","# !mkdir /root/.kaggle\n","\n","# with open(\"/root/.kaggle/kaggle.json\", \"w+\") as f:\n","#     f.write('{\"username\":\"jianches\",\"key\":\"c1bda8e22ca7a63a4fe3cd4bc80e5823\"}')\n","#     # Put your kaggle username & key here\n","\n","# !chmod 600 /root/.kaggle/kaggle.json"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"if2Somqfbje1"},"outputs":[],"source":["# # commands to download data from kaggle\n","# !kaggle competitions download -c 11785-hw1p2-s24\n","\n","# !unzip -qo /content/11785-hw1p2-s24.zip -d '/content'"]},{"cell_type":"code","source":["# !rm -rf /content/11785-s24-hw1p2\n"],"metadata":{"id":"jKS3OMKYVieO"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["#print(root)  /content/11-785-s24-hw1p2/train-clean-100/mfcc/103-1240-0000.npy"],"metadata":{"id":"1TB7YgF0upZs"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["np.load('/content/11-785-s24-hw1p2/train-clean-100/mfcc/103-1240-0000.npy')"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"FUH02QCVThVk","outputId":"9eb670a2-da9b-4c71-e403-7c34b21c9500","executionInfo":{"status":"ok","timestamp":1707195601190,"user_tz":300,"elapsed":4,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}}},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["array([[ 1.10716181e+01,  1.29364958e+01,  2.11114330e+01, ...,\n","        -1.52556181e-01,  3.08246484e-13,  9.49062500e+01],\n","       [ 1.14747839e+01,  1.38595648e+01,  2.18050823e+01, ...,\n","        -2.66239017e-01,  3.11884416e-13,  9.56741714e+01],\n","       [ 1.12015600e+01,  1.66964321e+01,  2.29666538e+01, ...,\n","        -5.54687679e-01,  3.03849421e-13,  9.54803467e+01],\n","       ...,\n","       [ 2.51545238e+00, -1.28244057e+01,  2.81143970e+01, ...,\n","        -1.46406519e+00,  4.07646790e-13,  1.16040901e+02],\n","       [-3.72080021e-02, -1.21763525e+01,  2.60421238e+01, ...,\n","        -9.37022150e-01,  4.09246665e-13,  1.17274742e+02],\n","       [-1.40476334e+00, -1.17058363e+01,  2.43025761e+01, ...,\n","        -1.07578576e+00,  4.07066769e-13,  1.18305443e+02]], dtype=float32)"]},"metadata":{},"execution_count":9}]},{"cell_type":"markdown","metadata":{"id":"Vuzce0_TdcaR"},"source":["# Dataset"]},{"cell_type":"markdown","metadata":{"id":"2_7QgMbBdgPp"},"source":["This section covers the dataset/dataloader class for speech data. You will have to spend time writing code to create this class successfully. We have given you a lot of comments guiding you on what code to write at each stage, from top to bottom of the class. Please try and take your time figuring this out, as it will immensely help in creating dataset/dataloader classes for future homeworks.\n","\n","Before running the following cells, please take some time to analyse the structure of data. Try loading a single MFCC and its transcipt, print out the shapes and print out the values. Do the transcripts look like phonemes?"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"YpLCvi3AJC5z"},"outputs":[],"source":["# Dataset class to load train and validation data\n","\n","class AudioDataset(torch.utils.data.Dataset):\n","\n","    def __init__(self, root, phonemes = PHONEMES, context=0, partition= \"train-clean-100/\", limit = None): # Feel free to add more arguments\n","\n","        self.context    = context\n","        self.phonemes   = phonemes\n","\n","        # TODO: MFCC directory - use partition to acces train/dev directories from kaggle data using root\n","        self.mfcc_dir       = os.path.join(root,partition,'mfcc')\n","        # TODO: Transcripts directory - use partition to acces train/dev directories from kaggle data using root\n","        self.transcript_dir = os.path.join(root,partition,'transcript')\n","\n","        if limit is None:\n","            mfcc_names = sorted(os.listdir(self.mfcc_dir))\n","            transcript_names = sorted(os.listdir(self.transcript_dir))\n","        else:\n","            mfcc_names = sorted(os.listdir(self.mfcc_dir))[:limit]\n","            transcript_names = sorted(os.listdir(self.transcript_dir))[:limit]\n","\n","\n","        # Making sure that we have the same no. of mfcc and transcripts\n","        assert len(mfcc_names) == len(transcript_names)\n","\n","        # total_timesteps = 0\n","        # # store the length of time step\n","        # for mfcc_file in mfcc_names:\n","        #   mfcc_path = os.path.join(self.mfcc_dir, mfcc_file)\n","        #   mfcc = np.load(mfcc_path)\n","        #   total_timesteps += mfcc.shape[0]\n","\n","        # mfcc_placeholder = np.zeros((total_timesteps, 27), dtype=np.float32)  #\n","        # transcript_placeholder = np.zeros((total_timesteps,), dtype=np.uint8)  #\n","\n","        self.mfccs, self.transcripts = [], []\n","\n","        # current_index = 0\n","        # TODO: Iterate through mfccs and transcripts\n","        for i in range(len(mfcc_names)):\n","        #   Load a single mfcc\n","            mfcc        = np.load(os.path.join(self.mfcc_dir,mfcc_names[i]))\n","        #   Do Cepstral Normalization of mfcc (explained in writeup)\n","            mfcc = (mfcc - np.mean(mfcc)) / np.std(mfcc)\n","        #   Load the corresponding transcript\n","            transcript  = np.load(os.path.join(self.transcript_dir,transcript_names[i])) # Remove [SOS] and [EOS] from the transcript\n","            transcript = transcript[1:-1]\n","            # (Is there an efficient way to do this without traversing through the transcript?)\n","            # Note that SOS will always be in the starting and EOS at end, as the name suggests.\n","            #num_timesteps = mfcc.shape[0]\n","\n","            # mfcc_placeholder[current_index : num_timesteps + current_index] = mfcc\n","            # transcript_placeholder[current_index : num_timesteps + current_index] = transcript\n","\n","            #current_index += mfcc.shape[0]\n","            self.mfccs.append(mfcc)\n","            self.transcripts.append(transcript)\n","\n","        # NOTE:\n","        # Each mfcc is of shape T1 x 27, T2 x 27, ...\n","        # Each transcript is of shape (T1+2) x 27, (T2+2) x 27 before removing [SOS] and [EOS]\n","\n","        # TODO: Concatenate all mfccs in self.mfccs such that\n","        # the final shape is T x 27 (Where T = T1 + T2 + ...)\n","        self.mfccs          = np.concatenate(self.mfccs,axis=0)\n","\n","        # TODO: Concatenate all transcripts in self.transcripts such that\n","        # the final shape is (T,) meaning, each time step has one phoneme output\n","        self.transcripts    = np.concatenate(self.transcripts,axis=0)\n","        # Hint: Use numpy to concatenate\n","\n","        # Length of the dataset is now the length of concatenated mfccs/transcripts\n","        self.length = len(self.mfccs)\n","\n","        # Take some time to think about what we have done.\n","        # self.mfcc is an array of the format (Frames x Features).\n","        # Our goal is to recognize phonemes of each frame\n","        # From hw0, you will be knowing what context is.\n","        # We can introduce context by padding zeros on top and bottom of self.mfcc\n","\n","        # zero_padding_matrix = np.zeros((self.mfccs.shape[0]+2*context,self.mfccs.shape[1]))\n","        # zero_padding_matrix[0+context:-context,:] = self.mfccs# TODO\n","        self.mfccs = np.pad(self.mfccs, pad_width = ((context, context), (0,0)), mode = 'constant')\n","\n","        # The available phonemes in the transcript are of string data type\n","        # But the neural network cannot predict strings as such.\n","        # Hence, we map these phonemes to integers\n","\n","        # TODO: Map the phonemes to their corresponding list indexes in self.phonemes\n","        phonemes_map = {phoneme: index for index, phoneme in enumerate(self.phonemes)}\n","        for i, elements in enumerate(self.transcripts):\n","          self.transcripts[i] = phonemes_map[elements]\n","\n","        # Now, if an element in self.transcript is 0, it means that it is 'SIL' (as per the above example)\n","\n","    def __len__(self):\n","        return self.length\n","\n","    def __getitem__(self, ind):\n","\n","        # TODO: Based on context and offset, return a frame at given index with context frames to the left, and right.\n","        frames = self.mfccs[ind:ind+2*self.context+1,:]\n","        # After slicing, you get an array of shape 2*context+1 x 27. But our MLP needs 1d data and not 2d.\n","        frames = frames.reshape(-1) # TODO: Flatten to get 1d data\n","\n","        frames      = torch.FloatTensor(frames) # Convert to tensors\n","        phonemes    = torch.tensor(int(self.transcripts[ind]))\n","\n","        return frames, phonemes"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"e8KfVP39S6o7"},"outputs":[],"source":["class AudioTestDataset(torch.utils.data.Dataset):\n","   # TODO: Create a test dataset class similar to the previous class but you dont have transcripts for this\n","    # Imp: Read the mfccs in sorted order, do NOT shuffle the data here or in your dataloader.\n","    def __init__(self, root, phonemes = PHONEMES, context=0, partition= \"test-clean\", limit = None):\n","        super().__init__()\n","        self.context    = context\n","        self.phonemes   = phonemes\n","\n","        self.mfcc_dir = os.path.join(root, partition,'mfcc')\n","\n","        if limit is None:\n","            mfcc_names = sorted(os.listdir(self.mfcc_dir))\n","        else:\n","            mfcc_names = sorted(os.listdir(self.mfcc_dir))[:limit]\n","\n","        # total_timesteps = 0\n","        # # store the length of time step\n","        # for mfcc_file in mfcc_names:\n","        #   mfcc_path = os.path.join(self.mfcc_dir, mfcc_file)\n","        #   mfcc = np.load(mfcc_path)\n","        #   total_timesteps += mfcc.shape[0]\n","\n","        #mfcc_placeholder = np.zeros((total_timesteps, 27), dtype=np.float32)  #\n","\n","        self.mfccs = []\n","        #current_index = 0\n","        for i in range(len(mfcc_names)):\n","          mfcc = np.load(os.path.join(self.mfcc_dir,mfcc_names[i]))\n","          mfcc = (mfcc - np.mean(mfcc)) / np.std(mfcc)\n","          # num_step = mfcc.shape[0]\n","          # mfcc_placeholder[current_index : current_index + num_step] = mfcc\n","          # current_index += num_step\n","          self.mfccs.append(mfcc)\n","\n","        self.mfccs = np.concatenate(self.mfccs, axis=0)\n","\n","\n","        self.length = len(self.mfccs)\n","\n","        self.mfccs = np.pad(self.mfccs, pad_width = ((context, context), (0,0)), mode = 'constant')\n","\n","\n","    def __len__(self):\n","      return self.length\n","\n","    def __getitem__(self, ind):\n","\n","      frames = self.mfccs[ind:ind+2*self.context+1, :]\n","      frames = frames.reshape(-1)\n","\n","      frames = torch.FloatTensor(frames)\n","\n","      return frames"]},{"cell_type":"markdown","metadata":{"id":"qNacQ8bpt9nw"},"source":["# Parameters Configuration"]},{"cell_type":"markdown","metadata":{"id":"WE7tsinAuLNy"},"source":["Storing your parameters and hyperparameters in a single configuration dictionary makes it easier to keep track of them during each experiment. It can also be used with weights and biases to log your parameters for each experiment and keep track of them across multiple experiments."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"PmKwlFqgt_Zq"},"outputs":[],"source":["config = {\n","    'epochs'        : 30,\n","    'batch_size'    : 2048,\n","    'context'       : 50,\n","    'init_lr'       : 1e-5/2,\n","    'architecture'  : 'a-model',\n","    'dropout'    : 0.2\n","    # Add more as you need them - e.g dropout values, weight decay, scheduler parameters\n","}"]},{"cell_type":"markdown","metadata":{"id":"2mlwaKlDt_2c"},"source":["# Create Datasets"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"7xi7V8x8W9z4","colab":{"base_uri":"https://localhost:8080/"},"outputId":"f996e5d2-6029-4c8a-b9fd-19c128299eaf","executionInfo":{"status":"ok","timestamp":1707421095234,"user_tz":300,"elapsed":139605,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}}},"outputs":[{"output_type":"stream","name":"stdout","text":["(1406,)\n","tensor([ 0.0000,  0.0000,  0.0000,  ..., -0.3837, -0.2862,  5.6537]) tensor(0) torch.Size([2727]) torch.Size([])\n"]}],"source":["#TODO: Create a dataset object using the AudioDataset class for the training data\n","x=np.load('/content/11-785-s24-hw1p2/train-clean-100/transcript/103-1240-0000.npy')\n","print(x.shape)\n","train_data = AudioDataset(root = \"/content/11-785-s24-hw1p2/\", context=config[\"context\"], limit = None)\n","#\n","i,j = train_data[8]\n","print(i,j,i.shape,j.shape)\n","# TODO: Create a dataset object using the AudioDataset class for the validation data\n","val_data = AudioDataset(partition=\"dev-clean\", root = \"/content/11-785-s24-hw1p2/\", context=config[\"context\"], limit = None)\n","\n","# TODO: Create a dataset object using the AudioTestDataset class for the test data\n","test_data = AudioTestDataset(root = \"/content/11-785-s24-hw1p2/\", context=config[\"context\"], limit = None)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"4mzoYfTKu14s","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1707421095234,"user_tz":300,"elapsed":11,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"05203ec7-3f60-4a0e-98db-b3464675b948"},"outputs":[{"output_type":"stream","name":"stdout","text":["Batch size     :  2048\n","Context        :  50\n","Input size     :  2727\n","Output symbols :  42\n","Train dataset samples = 36091157, batches = 17623\n","Validation dataset samples = 1928204, batches = 942\n","Test dataset samples = 1934138, batches = 945\n"]}],"source":["# Define dataloaders for train, val and test datasets\n","# Dataloaders will yield a batch of frames and phonemes of given batch_size at every iteration\n","# We shuffle train dataloader but not val & test dataloader. Why?\n","\n","train_loader = torch.utils.data.DataLoader(\n","    dataset     = train_data,\n","    num_workers = 0,\n","    batch_size  = config['batch_size'],\n","    pin_memory  = True,\n","    shuffle     = True\n",")\n","\n","val_loader = torch.utils.data.DataLoader(\n","    dataset     = val_data,\n","    num_workers = 0,\n","    batch_size  = config['batch_size'],\n","    pin_memory  = True,\n","    shuffle     = False\n",")\n","\n","test_loader = torch.utils.data.DataLoader(\n","    dataset     = test_data,\n","    num_workers = 0,\n","    batch_size  = config['batch_size'],\n","    pin_memory  = True,\n","    shuffle     = False\n",")\n","\n","\n","print(\"Batch size     : \", config['batch_size'])\n","print(\"Context        : \", config['context'])\n","print(\"Input size     : \", (2*config['context']+1)*27)\n","print(\"Output symbols : \", len(PHONEMES))\n","\n","print(\"Train dataset samples = {}, batches = {}\".format(train_data.__len__(), len(train_loader)))\n","print(\"Validation dataset samples = {}, batches = {}\".format(val_data.__len__(), len(val_loader)))\n","print(\"Test dataset samples = {}, batches = {}\".format(test_data.__len__(), len(test_loader)))"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"n-GV3UvgLSoF","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1707421097736,"user_tz":300,"elapsed":2503,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"991773c1-9e96-4067-f0ad-66694ff06341"},"outputs":[{"output_type":"stream","name":"stdout","text":["torch.Size([2048, 2727]) torch.Size([2048])\n"]}],"source":["\n","# Testing code to check if your data loaders are working\n","for i, data in enumerate(train_loader):\n","    frames, phoneme = data\n","    print(frames.shape, phoneme.shape)\n","    break"]},{"cell_type":"markdown","metadata":{"id":"Nxjwve20JRJ2"},"source":["# Network Architecture\n"]},{"cell_type":"markdown","metadata":{"id":"3NJzT-mRw6iy"},"source":["This section defines your network architecture for the homework. We have given you a sample architecture that can easily clear the very low cutoff for the early submission deadline."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"OvcpontXQq9j"},"outputs":[],"source":["\n","# This architecture will make you cross the very low cutoff\n","# However, you need to run a lot of experiments to cross the medium or high cutoff\n","# class Network(torch.nn.Module):\n","\n","class Network(torch.nn.Module):\n","    def __init__(self, input_size, output_size):\n","        super(Network, self).__init__()\n","\n","        self.model = torch.nn.Sequential(\n","            torch.nn.Linear(input_size, 512*4),\n","            torch.nn.BatchNorm1d(512*4),\n","            torch.nn.GELU(),\n","            torch.nn.Dropout(0.2),\n","            torch.nn.Linear(512*4, 512*4),\n","            torch.nn.BatchNorm1d(512*4),\n","            torch.nn.GELU(),\n","            torch.nn.Dropout(0.2),\n","            torch.nn.Linear(512*4, 512*4),\n","            torch.nn.BatchNorm1d(512*4),\n","            torch.nn.GELU(),\n","            torch.nn.Dropout(0.2),\n","            torch.nn.Linear(512*4, 512*4),\n","            torch.nn.BatchNorm1d(512*4),\n","            torch.nn.GELU(),\n","            torch.nn.Dropout(0.2),\n","            torch.nn.Linear(512*4, 512*4),\n","            torch.nn.BatchNorm1d(512*4),\n","            torch.nn.GELU(),\n","            torch.nn.Dropout(0.2),\n","            torch.nn.Linear(512*4, output_size)\n","        )\n","\n","    def forward(self, x):\n","        out = self.model(x)\n","        return out"]},{"cell_type":"markdown","metadata":{"id":"HejoSXe3vMVU"},"source":["# Define Model, Loss Function and Optimizer"]},{"cell_type":"markdown","metadata":{"id":"xAhGBH7-xxth"},"source":["Here we define the model, loss function, optimizer and optionally a learning rate scheduler."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"_qtrEM1ZvLje","colab":{"base_uri":"https://localhost:8080/","height":1000},"executionInfo":{"status":"ok","timestamp":1707421207413,"user_tz":300,"elapsed":370,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"a5fd9ec1-df02-4de0-e60d-77bf7ebd322e"},"outputs":[{"output_type":"stream","name":"stdout","text":["=========================================================================\n","                         Kernel Shape  Output Shape     Params  Mult-Adds\n","Layer                                                                    \n","0_model.Linear_0         [2727, 2048]  [2048, 2048]  5.586944M  5.584896M\n","1_model.BatchNorm1d_1          [2048]  [2048, 2048]     4.096k     2.048k\n","2_model.GELU_2                      -  [2048, 2048]          -          -\n","3_model.Dropout_3                   -  [2048, 2048]          -          -\n","4_model.Linear_4         [2048, 2048]  [2048, 2048]  4.196352M  4.194304M\n","5_model.BatchNorm1d_5          [2048]  [2048, 2048]     4.096k     2.048k\n","6_model.GELU_6                      -  [2048, 2048]          -          -\n","7_model.Dropout_7                   -  [2048, 2048]          -          -\n","8_model.Linear_8         [2048, 2048]  [2048, 2048]  4.196352M  4.194304M\n","9_model.BatchNorm1d_9          [2048]  [2048, 2048]     4.096k     2.048k\n","10_model.GELU_10                    -  [2048, 2048]          -          -\n","11_model.Dropout_11                 -  [2048, 2048]          -          -\n","12_model.Linear_12       [2048, 2048]  [2048, 2048]  4.196352M  4.194304M\n","13_model.BatchNorm1d_13        [2048]  [2048, 2048]     4.096k     2.048k\n","14_model.GELU_14                    -  [2048, 2048]          -          -\n","15_model.Dropout_15                 -  [2048, 2048]          -          -\n","16_model.Linear_16       [2048, 2048]  [2048, 2048]  4.196352M  4.194304M\n","17_model.BatchNorm1d_17        [2048]  [2048, 2048]     4.096k     2.048k\n","18_model.GELU_18                    -  [2048, 2048]          -          -\n","19_model.Dropout_19                 -  [2048, 2048]          -          -\n","20_model.Linear_20         [2048, 42]    [2048, 42]    86.058k    86.016k\n","-------------------------------------------------------------------------\n","                          Totals\n","Total params           22.47889M\n","Trainable params       22.47889M\n","Non-trainable params         0.0\n","Mult-Adds             22.458368M\n","=========================================================================\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/torchsummaryX/torchsummaryX.py:101: FutureWarning: The default value of numeric_only in DataFrame.sum is deprecated. In a future version, it will default to False. In addition, specifying 'numeric_only=None' is deprecated. Select only valid columns or specify the value of numeric_only to silence this warning.\n","  df_sum = df.sum()\n"]},{"output_type":"execute_result","data":{"text/plain":["                         Kernel Shape  Output Shape     Params  Mult-Adds\n","Layer                                                                    \n","0_model.Linear_0         [2727, 2048]  [2048, 2048]  5586944.0  5584896.0\n","1_model.BatchNorm1d_1          [2048]  [2048, 2048]     4096.0     2048.0\n","2_model.GELU_2                      -  [2048, 2048]        NaN        NaN\n","3_model.Dropout_3                   -  [2048, 2048]        NaN        NaN\n","4_model.Linear_4         [2048, 2048]  [2048, 2048]  4196352.0  4194304.0\n","5_model.BatchNorm1d_5          [2048]  [2048, 2048]     4096.0     2048.0\n","6_model.GELU_6                      -  [2048, 2048]        NaN        NaN\n","7_model.Dropout_7                   -  [2048, 2048]        NaN        NaN\n","8_model.Linear_8         [2048, 2048]  [2048, 2048]  4196352.0  4194304.0\n","9_model.BatchNorm1d_9          [2048]  [2048, 2048]     4096.0     2048.0\n","10_model.GELU_10                    -  [2048, 2048]        NaN        NaN\n","11_model.Dropout_11                 -  [2048, 2048]        NaN        NaN\n","12_model.Linear_12       [2048, 2048]  [2048, 2048]  4196352.0  4194304.0\n","13_model.BatchNorm1d_13        [2048]  [2048, 2048]     4096.0     2048.0\n","14_model.GELU_14                    -  [2048, 2048]        NaN        NaN\n","15_model.Dropout_15                 -  [2048, 2048]        NaN        NaN\n","16_model.Linear_16       [2048, 2048]  [2048, 2048]  4196352.0  4194304.0\n","17_model.BatchNorm1d_17        [2048]  [2048, 2048]     4096.0     2048.0\n","18_model.GELU_18                    -  [2048, 2048]        NaN        NaN\n","19_model.Dropout_19                 -  [2048, 2048]        NaN        NaN\n","20_model.Linear_20         [2048, 42]    [2048, 42]    86058.0    86016.0"],"text/html":["\n","  <div id=\"df-952d5448-3818-4931-9d89-043e737cc199\" class=\"colab-df-container\">\n","    <div>\n","<style scoped>\n","    .dataframe tbody tr th:only-of-type {\n","        vertical-align: middle;\n","    }\n","\n","    .dataframe tbody tr th {\n","        vertical-align: top;\n","    }\n","\n","    .dataframe thead th {\n","        text-align: right;\n","    }\n","</style>\n","<table border=\"1\" class=\"dataframe\">\n","  <thead>\n","    <tr style=\"text-align: right;\">\n","      <th></th>\n","      <th>Kernel Shape</th>\n","      <th>Output Shape</th>\n","      <th>Params</th>\n","      <th>Mult-Adds</th>\n","    </tr>\n","    <tr>\n","      <th>Layer</th>\n","      <th></th>\n","      <th></th>\n","      <th></th>\n","      <th></th>\n","    </tr>\n","  </thead>\n","  <tbody>\n","    <tr>\n","      <th>0_model.Linear_0</th>\n","      <td>[2727, 2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>5586944.0</td>\n","      <td>5584896.0</td>\n","    </tr>\n","    <tr>\n","      <th>1_model.BatchNorm1d_1</th>\n","      <td>[2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4096.0</td>\n","      <td>2048.0</td>\n","    </tr>\n","    <tr>\n","      <th>2_model.GELU_2</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>3_model.Dropout_3</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>4_model.Linear_4</th>\n","      <td>[2048, 2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4196352.0</td>\n","      <td>4194304.0</td>\n","    </tr>\n","    <tr>\n","      <th>5_model.BatchNorm1d_5</th>\n","      <td>[2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4096.0</td>\n","      <td>2048.0</td>\n","    </tr>\n","    <tr>\n","      <th>6_model.GELU_6</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>7_model.Dropout_7</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>8_model.Linear_8</th>\n","      <td>[2048, 2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4196352.0</td>\n","      <td>4194304.0</td>\n","    </tr>\n","    <tr>\n","      <th>9_model.BatchNorm1d_9</th>\n","      <td>[2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4096.0</td>\n","      <td>2048.0</td>\n","    </tr>\n","    <tr>\n","      <th>10_model.GELU_10</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>11_model.Dropout_11</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>12_model.Linear_12</th>\n","      <td>[2048, 2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4196352.0</td>\n","      <td>4194304.0</td>\n","    </tr>\n","    <tr>\n","      <th>13_model.BatchNorm1d_13</th>\n","      <td>[2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4096.0</td>\n","      <td>2048.0</td>\n","    </tr>\n","    <tr>\n","      <th>14_model.GELU_14</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>15_model.Dropout_15</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>16_model.Linear_16</th>\n","      <td>[2048, 2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4196352.0</td>\n","      <td>4194304.0</td>\n","    </tr>\n","    <tr>\n","      <th>17_model.BatchNorm1d_17</th>\n","      <td>[2048]</td>\n","      <td>[2048, 2048]</td>\n","      <td>4096.0</td>\n","      <td>2048.0</td>\n","    </tr>\n","    <tr>\n","      <th>18_model.GELU_18</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>19_model.Dropout_19</th>\n","      <td>-</td>\n","      <td>[2048, 2048]</td>\n","      <td>NaN</td>\n","      <td>NaN</td>\n","    </tr>\n","    <tr>\n","      <th>20_model.Linear_20</th>\n","      <td>[2048, 42]</td>\n","      <td>[2048, 42]</td>\n","      <td>86058.0</td>\n","      <td>86016.0</td>\n","    </tr>\n","  </tbody>\n","</table>\n","</div>\n","    <div class=\"colab-df-buttons\">\n","\n","  <div class=\"colab-df-container\">\n","    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-952d5448-3818-4931-9d89-043e737cc199')\"\n","            title=\"Convert this dataframe to an interactive table.\"\n","            style=\"display:none;\">\n","\n","  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n","    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n","  </svg>\n","    </button>\n","\n","  <style>\n","    .colab-df-container {\n","      display:flex;\n","      gap: 12px;\n","    }\n","\n","    .colab-df-convert {\n","      background-color: #E8F0FE;\n","      border: none;\n","      border-radius: 50%;\n","      cursor: pointer;\n","      display: none;\n","      fill: #1967D2;\n","      height: 32px;\n","      padding: 0 0 0 0;\n","      width: 32px;\n","    }\n","\n","    .colab-df-convert:hover {\n","      background-color: #E2EBFA;\n","      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n","      fill: #174EA6;\n","    }\n","\n","    .colab-df-buttons div {\n","      margin-bottom: 4px;\n","    }\n","\n","    [theme=dark] .colab-df-convert {\n","      background-color: #3B4455;\n","      fill: #D2E3FC;\n","    }\n","\n","    [theme=dark] .colab-df-convert:hover {\n","      background-color: #434B5C;\n","      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n","      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n","      fill: #FFFFFF;\n","    }\n","  </style>\n","\n","    <script>\n","      const buttonEl =\n","        document.querySelector('#df-952d5448-3818-4931-9d89-043e737cc199 button.colab-df-convert');\n","      buttonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","\n","      async function convertToInteractive(key) {\n","        const element = document.querySelector('#df-952d5448-3818-4931-9d89-043e737cc199');\n","        const dataTable =\n","          await google.colab.kernel.invokeFunction('convertToInteractive',\n","                                                    [key], {});\n","        if (!dataTable) return;\n","\n","        const docLinkHtml = 'Like what you see? Visit the ' +\n","          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n","          + ' to learn more about interactive tables.';\n","        element.innerHTML = '';\n","        dataTable['output_type'] = 'display_data';\n","        await google.colab.output.renderOutput(dataTable, element);\n","        const docLink = document.createElement('div');\n","        docLink.innerHTML = docLinkHtml;\n","        element.appendChild(docLink);\n","      }\n","    </script>\n","  </div>\n","\n","\n","<div id=\"df-0ca7d414-0499-482b-8f32-e44962af7689\">\n","  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-0ca7d414-0499-482b-8f32-e44962af7689')\"\n","            title=\"Suggest charts\"\n","            style=\"display:none;\">\n","\n","<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n","     width=\"24px\">\n","    <g>\n","        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n","    </g>\n","</svg>\n","  </button>\n","\n","<style>\n","  .colab-df-quickchart {\n","      --bg-color: #E8F0FE;\n","      --fill-color: #1967D2;\n","      --hover-bg-color: #E2EBFA;\n","      --hover-fill-color: #174EA6;\n","      --disabled-fill-color: #AAA;\n","      --disabled-bg-color: #DDD;\n","  }\n","\n","  [theme=dark] .colab-df-quickchart {\n","      --bg-color: #3B4455;\n","      --fill-color: #D2E3FC;\n","      --hover-bg-color: #434B5C;\n","      --hover-fill-color: #FFFFFF;\n","      --disabled-bg-color: #3B4455;\n","      --disabled-fill-color: #666;\n","  }\n","\n","  .colab-df-quickchart {\n","    background-color: var(--bg-color);\n","    border: none;\n","    border-radius: 50%;\n","    cursor: pointer;\n","    display: none;\n","    fill: var(--fill-color);\n","    height: 32px;\n","    padding: 0;\n","    width: 32px;\n","  }\n","\n","  .colab-df-quickchart:hover {\n","    background-color: var(--hover-bg-color);\n","    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n","    fill: var(--button-hover-fill-color);\n","  }\n","\n","  .colab-df-quickchart-complete:disabled,\n","  .colab-df-quickchart-complete:disabled:hover {\n","    background-color: var(--disabled-bg-color);\n","    fill: var(--disabled-fill-color);\n","    box-shadow: none;\n","  }\n","\n","  .colab-df-spinner {\n","    border: 2px solid var(--fill-color);\n","    border-color: transparent;\n","    border-bottom-color: var(--fill-color);\n","    animation:\n","      spin 1s steps(1) infinite;\n","  }\n","\n","  @keyframes spin {\n","    0% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","      border-left-color: var(--fill-color);\n","    }\n","    20% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    30% {\n","      border-color: transparent;\n","      border-left-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","      border-right-color: var(--fill-color);\n","    }\n","    40% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-top-color: var(--fill-color);\n","    }\n","    60% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","    }\n","    80% {\n","      border-color: transparent;\n","      border-right-color: var(--fill-color);\n","      border-bottom-color: var(--fill-color);\n","    }\n","    90% {\n","      border-color: transparent;\n","      border-bottom-color: var(--fill-color);\n","    }\n","  }\n","</style>\n","\n","  <script>\n","    async function quickchart(key) {\n","      const quickchartButtonEl =\n","        document.querySelector('#' + key + ' button');\n","      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n","      quickchartButtonEl.classList.add('colab-df-spinner');\n","      try {\n","        const charts = await google.colab.kernel.invokeFunction(\n","            'suggestCharts', [key], {});\n","      } catch (error) {\n","        console.error('Error during call to suggestCharts:', error);\n","      }\n","      quickchartButtonEl.classList.remove('colab-df-spinner');\n","      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n","    }\n","    (() => {\n","      let quickchartButtonEl =\n","        document.querySelector('#df-0ca7d414-0499-482b-8f32-e44962af7689 button');\n","      quickchartButtonEl.style.display =\n","        google.colab.kernel.accessAllowed ? 'block' : 'none';\n","    })();\n","  </script>\n","</div>\n","    </div>\n","  </div>\n"]},"metadata":{},"execution_count":16}],"source":["\n","INPUT_SIZE  = (2*config['context'] + 1) * 27 # Why is this the case?\n","model       = Network(INPUT_SIZE, len(train_data.phonemes)).to(device)\n","summary(model, frames.to(device))\n","# Check number of parameters of your network\n","# Remember, you are limited to 24 million parameters for HW1 (including ensembles)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"UROGEVJevKD-"},"outputs":[],"source":["criterion = torch.nn.CrossEntropyLoss() # Defining Loss function.\n","# We use CE because the task is multi-class classification\n","\n","optimizer = torch.optim.AdamW(model.parameters(), lr= config['init_lr'], weight_decay=0.1) #Defining Optimizer\n","# Recommended : Define Scheduler for Learning Rate,\n","# including but not limited to StepLR, MultiStepLR, CosineAnnealingLR, ReduceLROnPlateau, etc.\n","# You can refer to Pytorch documentation for more information on how to use them.\n","\n","scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, 'max', patience=1, threshold=1e-4, threshold_mode = 'rel')\n","# scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max = 60,eta_min = 1e-4)\n","# scheduler = torch.optim.lr_scheduler.ExponentialLR(optimizer, gamma = 0.5)\n","# Is your training time very high?\n","# Look into mixed precision training if your GPU (Tesla T4, V100, etc) can make use of it\n","# Refer - https://pytorch.org/docs/stable/notes/amp_examples.html\n","scaler = torch.cuda.amp.GradScaler()"]},{"cell_type":"markdown","metadata":{"id":"IBwunYpyugFg"},"source":["\n","\n","# Training and Validation Functions"]},{"cell_type":"markdown","metadata":{"id":"1JgeNhx4x2-P"},"source":["This section covers the training, and validation functions for each epoch of running your experiment with a given model architecture. The code has been provided to you, but we recommend going through the comments to understand the workflow to enable you to write these loops for future HWs."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"XblOHEVtKab2","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1707421228146,"user_tz":300,"elapsed":161,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"31a7bd72-a762-4f5c-f5f7-bf064ab0f1a0"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["36"]},"metadata":{},"execution_count":18}],"source":["torch.cuda.empty_cache()\n","gc.collect()"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"8wjPz7DHqKcL"},"outputs":[],"source":["def train(model, dataloader, optimizer, criterion):\n","\n","    model.train()\n","    tloss, tacc = 0, 0 # Monitoring loss and accuracy\n","    batch_bar   = tqdm(total=len(train_loader), dynamic_ncols=True, leave=False, position=0, desc='Train')\n","\n","    for i, (frames, phonemes) in enumerate(dataloader):\n","\n","        ### Initialize Gradients\n","        optimizer.zero_grad()\n","\n","        ### Move Data to Device (Ideally GPU)\n","        frames      = frames.to(device)\n","        phonemes    = phonemes.to(device)\n","        with torch.cuda.amp.autocast():\n","          ### Forward Propagation\n","          logits  = model(frames)\n","\n","          ### Loss Calculation\n","          loss    = criterion(logits, phonemes)\n","\n","        ### Backward Propagation\n","        scaler.scale(loss).backward()\n","\n","\n","        ### Gradient Descent\n","        scaler.step(optimizer)\n","\n","\n","        tloss   += loss.item()\n","        tacc    += torch.sum(torch.argmax(logits, dim= 1) == phonemes).item()/logits.shape[0]\n","\n","        batch_bar.set_postfix(loss=\"{:.04f}\".format(float(tloss / (i + 1))),\n","                              acc=\"{:.04f}%\".format(float(tacc*100 / (i + 1))))\n","\n","        scaler.update()\n","\n","        batch_bar.update()\n","\n","        ### Release memory\n","        del frames, phonemes, logits\n","        torch.cuda.empty_cache()\n","\n","    batch_bar.close()\n","    tloss   /= len(train_loader)\n","    tacc    /= len(train_loader)\n","\n","    return tloss, tacc"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"Q5npQNFH315V"},"outputs":[],"source":["def eval(model, dataloader):\n","\n","    model.eval() # set model in evaluation mode\n","    vloss, vacc = 0, 0 # Monitoring loss and accuracy\n","    batch_bar   = tqdm(total=len(val_loader), dynamic_ncols=True, position=0, leave=False, desc='Val')\n","\n","    for i, (frames, phonemes) in enumerate(dataloader):\n","\n","        ### Move data to device (ideally GPU)\n","        frames      = frames.to(device)\n","        phonemes    = phonemes.to(device)\n","\n","        # makes sure that there are no gradients computed as we are not training the model now\n","        with torch.inference_mode():\n","            ### Forward Propagation\n","            logits  = model(frames)\n","            ### Loss Calculation\n","            loss    = criterion(logits, phonemes)\n","\n","        vloss   += loss.item()\n","        vacc    += torch.sum(torch.argmax(logits, dim= 1) == phonemes).item()/logits.shape[0]\n","\n","        # Do you think we need loss.backward() and optimizer.step() here?\n","\n","        batch_bar.set_postfix(loss=\"{:.04f}\".format(float(vloss / (i + 1))),\n","                              acc=\"{:.04f}%\".format(float(vacc*100 / (i + 1))))\n","        batch_bar.update()\n","\n","        ### Release memory\n","        del frames, phonemes, logits\n","        torch.cuda.empty_cache()\n","\n","    batch_bar.close()\n","    vloss   /= len(val_loader)\n","    vacc    /= len(val_loader)\n","\n","    return vloss, vacc"]},{"cell_type":"markdown","metadata":{"id":"yMd_XxPku5qp"},"source":["# Weights and Biases Setup"]},{"cell_type":"markdown","metadata":{"id":"tjIbhR1wwbgI"},"source":["This section is to enable logging metrics and files with Weights and Biases. Please refer to wandb documentationa and recitation 0 that covers the use of weights and biases for logging, hyperparameter tuning and monitoring your runs for your homeworks. Using this tool makes it very easy to show results when submitting your code and models for homeworks, and also extremely useful for study groups to organize and run ablations under a single team in wandb.\n","\n","We have written code for you to make use of it out of the box, so that you start using wandb for all your HWs from the beginning."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"SCDYx5VEu6qI","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1707364376787,"user_tz":300,"elapsed":153,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"721d7d54-cac6-4bc3-f5b6-9d7bc7f4c536"},"outputs":[{"output_type":"stream","name":"stderr","text":["\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m Calling wandb.login() after wandb.init() has no effect.\n"]},{"output_type":"execute_result","data":{"text/plain":["True"]},"metadata":{},"execution_count":133}],"source":["wandb.login(key=\"efb5ba10afc754fad55f8408759058272e35301f\") #API Key is in your wandb account, under settings (wandb.ai/settings)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"xvUnYd3Bw2up","colab":{"base_uri":"https://localhost:8080/","height":140},"executionInfo":{"status":"ok","timestamp":1707421372668,"user_tz":300,"elapsed":2906,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"52d23945-be72-43cc-b194-0195ea312b87"},"outputs":[{"output_type":"stream","name":"stderr","text":["\u001b[34m\u001b[1mwandb\u001b[0m: Currently logged in as: \u001b[33mjianchengsunny\u001b[0m (\u001b[33mjiancheng12345\u001b[0m). Use \u001b[1m`wandb login --relogin`\u001b[0m to force relogin\n"]},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["wandb version 0.16.3 is available!  To upgrade, please run:\n"," $ pip install wandb --upgrade"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Tracking run with wandb version 0.16.2"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Run data is saved locally in <code>/content/wandb/run-20240208_194251-sl70opd0</code>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Syncing run <strong><a href='https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0' target=\"_blank\">8-run-dropout_0.2-ReduceLROnPlateau</a></strong> to <a href='https://wandb.ai/jiancheng12345/hw1p2' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":[" View project at <a href='https://wandb.ai/jiancheng12345/hw1p2' target=\"_blank\">https://wandb.ai/jiancheng12345/hw1p2</a>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":[" View run at <a href='https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0' target=\"_blank\">https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0</a>"]},"metadata":{}}],"source":["# Create your wandb run\n","run = wandb.init(\n","    name    = f\"8-run-dropout_{config['dropout']}-ReduceLROnPlateau\", ### Wandb creates random run names if you skip this field, we recommend you give useful names\n","    reinit  = True, ### Allows reinitalizing runs when you re-run this cell\n","    #id     = \"y28t31uz\", ### Insert specific run id here if you want to resume a previous run\n","    #resume = \"must\", ### You need this to resume previous runs, but comment out reinit = True when using this\n","    project = \"hw1p2\", ### Project should be created in your wandb account\n","    config  = config ### Wandb Config for your run\n",")"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"wft15E_IxYFi","colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"status":"ok","timestamp":1707421444383,"user_tz":300,"elapsed":254,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"ad9d8728-4765-4f48-cf40-fffbc5dd8f20"},"outputs":[{"output_type":"execute_result","data":{"text/plain":["['/content/wandb/run-20240208_194251-sl70opd0/files/8-run-dropout_0.2-ReduceLROnPlateau.txt']"]},"metadata":{},"execution_count":22}],"source":["### Save your model architecture as a string with str(model)\n","model_arch  = str(model)\n","\n","### Save it in a txt file\n","arch_file   = open(f\"{run.name}.txt\", \"w\")\n","file_write  = arch_file.write(model_arch)\n","arch_file.close()\n","\n","### log it in your wandb run with wandb.save()\n","wandb.save(f\"{run.name}.txt\")"]},{"cell_type":"markdown","metadata":{"id":"nclx_04fu7Dd"},"source":["# Experiment"]},{"cell_type":"markdown","metadata":{"id":"MdLMWfEpyGOB"},"source":["Now, it is time to finally run your ablations! Have fun!"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"MG4F77Nm0Am9"},"outputs":[],"source":["# Iterate over number of epochs to train and evaluate your model\n","torch.cuda.empty_cache()\n","gc.collect()\n","wandb.watch(model, log=\"all\")\n","best_acc = 0\n","\n","for epoch in range(config['epochs']):\n","\n","    print(\"\\nEpoch {}/{}\".format(epoch+1, config['epochs']))\n","\n","    curr_lr                 = float(optimizer.param_groups[0]['lr'])\n","    train_loss, train_acc   = train(model, train_loader, optimizer, criterion)\n","    val_loss, val_acc       = eval(model, val_loader)\n","\n","    scheduler.step(val_acc)\n","    print(\"\\tTrain Acc {:.04f}%\\tTrain Loss {:.04f}\\t Learning Rate {:.07f}\".format(train_acc*100, train_loss, curr_lr))\n","    print(\"\\tVal Acc {:.04f}%\\tVal Loss {:.04f}\".format(val_acc*100, val_loss))\n","\n","    ### Log metrics at each epoch in your run\n","    # Optionally, you can log at each batch inside train/eval functions\n","    # (explore wandb documentation/wandb recitation)\n","    wandb.log({'train_acc': train_acc*100, 'train_loss': train_loss,\n","               'val_acc': val_acc*100, 'valid_loss': val_loss, 'lr': curr_lr})\n","\n","    ### Highly Recommended: Save checkpoint in drive and/or wandb if accuracy is better than your current best\n","    # updating the model version\n","    if train_acc > best_acc:\n","      best_acc = train_acc\n","\n","      # saving the model and optimizer states\n","      torch.save({\n","            'model_state_dict'        : model.state_dict(),\n","            'optimizer_state_dict'    : optimizer.state_dict()\n","            }, \"Model\")\n","\n","      # creating WandB Artifact\n","      model_artifact = wandb.Artifact(config['architecture'], type='model')\n","\n","      # Adding model file to Artifact\n","      model_artifact.add_file(\"Model\")\n","\n","      # Saving Artifact to WandB\n","      run.log_artifact(model_artifact)\n","### Finish your wandb run\n","#run.finish()"]},{"cell_type":"markdown","metadata":{"id":"_kXwf5YUo_4A"},"source":["# Testing and submission to Kaggle"]},{"cell_type":"markdown","metadata":{"id":"WI1hSFYLpJvH"},"source":["Before we get to the following code, make sure to see the format of submission given in *sample_submission.csv*. Once you have done so, it is time to fill the following function to complete your inference on test data. Refer the eval function from previous cells to get an idea of how to go about completing this function."]},{"cell_type":"code","execution_count":null,"metadata":{"id":"R-SU9fZ3xHtk"},"outputs":[],"source":["def test(model, test_loader):\n","    ### What you call for model to perform inference?\n","    model.eval() # TODO train or eval?\n","\n","    ### List to store predicted phonemes of test data\n","    test_predictions = []\n","\n","    ### Which mode do you need to avoid gradients?\n","    with torch.inference_mode(): # TODO\n","\n","        for i, mfccs in enumerate(tqdm(test_loader)):\n","\n","            mfccs   = mfccs.to(device)\n","\n","            logits  = model(mfccs)\n","\n","            ### Get most likely predicted phoneme with argmax\n","            predicted_phonemes = torch.argmax(logits, dim=1)\n","\n","            ### How do you store predicted_phonemes with test_predictions? Hint, look at eval\n","            # TODO\n","            test_predictions.append(predicted_phonemes)\n","            #print(predicted_phonemes.shape)\n","            # print(test_predictions)\n","            del mfccs, logits\n","            #torch.cuda.empty_cache()\n","    all_predictions = torch.cat(test_predictions).cpu().numpy()\n","    print(all_predictions.shape)\n","    phoneme_predictions = [PHONEMES[i] for i in all_predictions]\n","    return phoneme_predictions"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"wG9v6Xmxu7wp","colab":{"base_uri":"https://localhost:8080/","height":67,"referenced_widgets":["0743085915524b3ab379a38735313d0e","fee79165f3ce4488a978120d3a732b53","0d6ef6c6a9a7456fbd327d68d622e1a0","80c383b19d8a4a86a00a8dd0d3930cc9","433954466de74b258576085156e35f27","7780b28d28944b89b79a6c2add4d6454","69db87212c6347c6946099efb275a53e","ca44b407930d4f14a3ae4ffe4446a82c","4d2ae829824746b8901cd695866895c7","8a66059da5b64302a46cce14a171eb60","b7899210a2074e6c887503a6c2533483"]},"executionInfo":{"status":"ok","timestamp":1707510177693,"user_tz":300,"elapsed":32058,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"25517e6b-843b-44e8-de97-469d668a3bbf"},"outputs":[{"output_type":"display_data","data":{"text/plain":["  0%|          | 0/945 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0743085915524b3ab379a38735313d0e"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["(1934138,)\n"]}],"source":["predictions = test(model, test_loader)"]},{"cell_type":"code","execution_count":null,"metadata":{"id":"ZE1hRnvf0bFz"},"outputs":[],"source":["### Create CSV file with predictions\n","with open(\"./submission.csv\", \"w+\") as f:\n","    f.write(\"id,label\\n\")\n","    for i in range(len(predictions)):\n","        f.write(\"{},{}\\n\".format(i, predictions[i]))"]},{"cell_type":"markdown","source":["The below section is used for restart the training part stored in wandb, assume you stop the training in the above blocks, you can restart it through wandb.\n"],"metadata":{"id":"xplmHNfjCpop"}},{"cell_type":"code","source":["# Create your wandb run\n","run = wandb.init(\n","    name    = f\"8-run-dropout_{config['dropout']}-ReduceLROnPlateau\", ### Wandb creates random run names if you skip this field, we recommend you give useful names\n","    #reinit  = True, ### Allows reinitalizing runs when you re-run this cell\n","    id     = \"sl70opd0\", ### Insert specific run id here if you want to resume a previous run\n","    resume = \"must\", ### You need this to resume previous runs, but comment out reinit = True when using this\n","    project = \"hw1p2\", ### Project should be created in your wandb account\n","    config  = config ### Wandb Config for your run\n",")\n","\n","## Retreiving the model\n","\n","# Getting the latest version of the artifact\n","artifact = run.use_artifact('{}:latest'.format(config['architecture']))\n","# Downloading the artifact\n","artifact_dir = artifact.download()\n","# Loading the model\n","model_dict = torch.load(os.path.join(artifact_dir, 'Model'))\n","\n","\n","# Loading weights\n","model.load_state_dict(model_dict['model_state_dict'])\n","# Loading optimizer state\n","# optimizer.load_state_dict(model_dict['optimizer_state_dict'])\n","# change the learning rate"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":784,"referenced_widgets":["8daa825304034efca0892af9f87f78a8","91eed9a54cf74f5995b8c7de8b5ef70a","e191e1688e744c4ab16e1f4faedb55b4","0816b2a329154ff7a654c1904003ab3d","7822ed23001d4f659a939e90f00dcca6","7dd69b9fb04f4e02b6a80eeb60ca7ebc","90fb60b87a6b4dada63b0534e0481e31","8a23e12cbe5d44cb9d80df943c0d8e3e","f50ee4d2f5c24937bbb97bc8da6e9403"]},"id":"ThPgmIEUyQYJ","executionInfo":{"status":"ok","timestamp":1707508254255,"user_tz":300,"elapsed":9389,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"9bf7cd7b-9d1f-48c8-9c95-a83fc84ce956"},"execution_count":null,"outputs":[{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Finishing last run (ID:sl70opd0) before initializing another..."]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["VBox(children=(Label(value='0.000 MB of 0.000 MB uploaded\\r'), FloatProgress(value=1.0, max=1.0)))"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"8daa825304034efca0892af9f87f78a8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["<style>\n","    table.wandb td:nth-child(1) { padding: 0 10px; text-align: left ; width: auto;} td:nth-child(2) {text-align: left ; width: 100%}\n","    .wandb-row { display: flex; flex-direction: row; flex-wrap: wrap; justify-content: flex-start; width: 100% }\n","    .wandb-col { display: flex; flex-direction: column; flex-basis: 100%; flex: 1; padding: 10px; }\n","    </style>\n","<div class=\"wandb-row\"><div class=\"wandb-col\"><h3>Run summary:</h3><br/><table class=\"wandb\"><tr><td>lr</td><td>0.0</td></tr><tr><td>train_acc</td><td>90.83673</td></tr><tr><td>train_loss</td><td>0.24587</td></tr><tr><td>val_acc</td><td>85.48343</td></tr><tr><td>valid_loss</td><td>0.48855</td></tr></table><br/></div></div>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":[" View run <strong style=\"color:#cdcd00\">8-run-dropout_0.2-ReduceLROnPlateau</strong> at: <a href='https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0' target=\"_blank\">https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0</a><br/>Synced 4 W&B file(s), 0 media file(s), 0 artifact file(s) and 0 other file(s)"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Find logs at: <code>./wandb/run-20240209_194525-sl70opd0/logs</code>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Successfully finished last run (ID:sl70opd0). Initializing new run:<br/>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["VBox(children=(Label(value='Waiting for wandb.init()...\\r'), FloatProgress(value=0.011113834811193454, max=1.0…"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"91eed9a54cf74f5995b8c7de8b5ef70a"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["wandb version 0.16.3 is available!  To upgrade, please run:\n"," $ pip install wandb --upgrade"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Tracking run with wandb version 0.16.2"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Run data is saved locally in <code>/content/wandb/run-20240209_195044-sl70opd0</code>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":["Resuming run <strong><a href='https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0' target=\"_blank\">8-run-dropout_0.2-ReduceLROnPlateau</a></strong> to <a href='https://wandb.ai/jiancheng12345/hw1p2' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/run' target=\"_blank\">docs</a>)<br/>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":[" View project at <a href='https://wandb.ai/jiancheng12345/hw1p2' target=\"_blank\">https://wandb.ai/jiancheng12345/hw1p2</a>"]},"metadata":{}},{"output_type":"display_data","data":{"text/plain":["<IPython.core.display.HTML object>"],"text/html":[" View run at <a href='https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0' target=\"_blank\">https://wandb.ai/jiancheng12345/hw1p2/runs/sl70opd0</a>"]},"metadata":{}},{"output_type":"stream","name":"stderr","text":["\u001b[34m\u001b[1mwandb\u001b[0m: Downloading large artifact a-model:latest, 257.35MB. 1 files... \n","\u001b[34m\u001b[1mwandb\u001b[0m:   1 of 1 files downloaded.  \n","Done. 0:0:0.6\n"]},{"output_type":"stream","name":"stdout","text":["AdamW (\n","Parameter Group 0\n","    amsgrad: False\n","    betas: (0.9, 0.999)\n","    capturable: False\n","    differentiable: False\n","    eps: 1e-08\n","    foreach: None\n","    fused: None\n","    lr: 1.0000000000000002e-06\n","    maximize: False\n","    weight_decay: 0.01\n",") <class 'torch.optim.adamw.AdamW'>\n"]}]},{"cell_type":"code","source":["# Iterate over number of epochs to train and evaluate your model\n","torch.cuda.empty_cache()\n","gc.collect()\n","wandb.watch(model, log=\"all\")\n","best_acc = 0\n","\n","for epoch in range(config['epochs']):\n","\n","    print(\"\\nEpoch {}/{}\".format(epoch+1, config['epochs']))\n","\n","    curr_lr                 = float(optimizer.param_groups[0]['lr'])\n","    print(f'current learning rate is: {curr_lr}')\n","    train_loss, train_acc   = train(model, train_loader, optimizer, criterion)\n","    val_loss, val_acc       = eval(model, val_loader)\n","\n","    scheduler.step(val_acc)\n","\n","    print(\"\\tTrain Acc {:.04f}%\\tTrain Loss {:.04f}\\t Learning Rate {:.07f}\".format(train_acc*100, train_loss, curr_lr))\n","    print(\"\\tVal Acc {:.04f}%\\tVal Loss {:.04f}\".format(val_acc*100, val_loss))\n","\n","    ### Log metrics at each epoch in your run\n","    # Optionally, you can log at each batch inside train/eval functions\n","    # (explore wandb documentation/wandb recitation)\n","    wandb.log({'train_acc': train_acc*100, 'train_loss': train_loss,\n","               'val_acc': val_acc*100, 'valid_loss': val_loss, 'lr': curr_lr})\n","\n","    ### Highly Recommended: Save checkpoint in drive and/or wandb if accuracy is better than your current best\n","    # updating the model version\n","    if train_acc > best_acc:\n","      best_acc = train_acc\n","\n","      # saving the model and optimizer states\n","      torch.save({\n","            'model_state_dict'        : model.state_dict(),\n","            'optimizer_state_dict'    : optimizer.state_dict()\n","            }, \"Model\")\n","\n","      # creating WandB Artifact\n","      model_artifact = wandb.Artifact(config['architecture'], type='model')\n","\n","      # Adding model file to Artifact\n","      model_artifact.add_file(\"Model\")\n","\n","      # Saving Artifact to WandB\n","      run.log_artifact(model_artifact)\n","### Finish your wandb run\n","#run.finish()"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":489,"referenced_widgets":["bc20f841c86548b5b77fde6342c047de","aa91efaaa8be489892429bfa09e264c8","76c4db9c6aeb43ca8349f4607d30f429","cfc9242425f14d8bbc7048bfd4f583ce","7fa86a47936a4ac98dc537108dfb0399","d62eeb6f22054836be97edf43290140d","9228f8e0af2a42388581f303a6679d1a","87bb2d903e2e4a08a2310fa0dff16cc5","c9ac4c2a5c26402c84ea35d23af7e8fa","cba7f9cb77ac4b0f8221003832689634","e21e1c00614848fa971c3b235eb44fbb"]},"id":"HC40Wlph1n0w","executionInfo":{"status":"error","timestamp":1707510131289,"user_tz":300,"elapsed":827060,"user":{"displayName":"Suwan Sun","userId":"12592220404673839974"}},"outputId":"49b1917d-79e5-4dd4-b4fd-636455294b46"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["\n","Epoch 1/30\n","current learning rate is: 5e-06\n"]},{"output_type":"display_data","data":{"text/plain":["Train:   0%|          | 0/17623 [00:00<?, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"bc20f841c86548b5b77fde6342c047de"}},"metadata":{}},{"output_type":"error","ename":"KeyboardInterrupt","evalue":"","traceback":["\u001b[0;31m---------------------------------------------------------------------------\u001b[0m","\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)","\u001b[0;32m<ipython-input-44-36f76bf042e1>\u001b[0m in \u001b[0;36m<cell line: 7>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     11\u001b[0m     \u001b[0mcurr_lr\u001b[0m                 \u001b[0;34m=\u001b[0m \u001b[0mfloat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0moptimizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mparam_groups\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m     \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mf'current learning rate is: {curr_lr}'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 13\u001b[0;31m     \u001b[0mtrain_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_acc\u001b[0m   \u001b[0;34m=\u001b[0m \u001b[0mtrain\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtrain_loader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moptimizer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     14\u001b[0m     \u001b[0mval_loss\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_acc\u001b[0m       \u001b[0;34m=\u001b[0m \u001b[0meval\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mval_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-19-0dc961261f6f>\u001b[0m in \u001b[0;36mtrain\u001b[0;34m(model, dataloader, optimizer, criterion)\u001b[0m\n\u001b[1;32m      5\u001b[0m     \u001b[0mbatch_bar\u001b[0m   \u001b[0;34m=\u001b[0m \u001b[0mtqdm\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtotal\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_loader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdynamic_ncols\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mleave\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mposition\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdesc\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'Train'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m     \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0mframes\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mphonemes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataloader\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0;31m### Initialize Gradients\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    628\u001b[0m                 \u001b[0;31m# TODO(https://github.com/pytorch/pytorch/issues/76750)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    629\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 630\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    631\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    672\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    673\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 674\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    675\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    676\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory_device\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m/usr/local/lib/python3.10/dist-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     49\u001b[0m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__getitems__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     50\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 51\u001b[0;31m                 \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     52\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;32m<ipython-input-7-728981ab1935>\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, ind)\u001b[0m\n\u001b[1;32m    104\u001b[0m         \u001b[0mframes\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mframes\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# TODO: Flatten to get 1d data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    105\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 106\u001b[0;31m         \u001b[0mframes\u001b[0m      \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mFloatTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mframes\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# Convert to tensors\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    107\u001b[0m         \u001b[0mphonemes\u001b[0m    \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtranscripts\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mind\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    108\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n","\u001b[0;31mKeyboardInterrupt\u001b[0m: "]}]}],"metadata":{"accelerator":"GPU","colab":{"provenance":[],"collapsed_sections":["z4vZbDmJvMp1","ZIi0Big7vPa9","Vuzce0_TdcaR"],"gpuType":"T4"},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.9.10"},"vscode":{"interpreter":{"hash":"b0fa6594d8f4cbf19f97940f81e996739fb7646882a419484c72d19e05852a7e"}},"widgets":{"application/vnd.jupyter.widget-state+json":{"0743085915524b3ab379a38735313d0e":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_fee79165f3ce4488a978120d3a732b53","IPY_MODEL_0d6ef6c6a9a7456fbd327d68d622e1a0","IPY_MODEL_80c383b19d8a4a86a00a8dd0d3930cc9"],"layout":"IPY_MODEL_433954466de74b258576085156e35f27"}},"fee79165f3ce4488a978120d3a732b53":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7780b28d28944b89b79a6c2add4d6454","placeholder":"​","style":"IPY_MODEL_69db87212c6347c6946099efb275a53e","value":"100%"}},"0d6ef6c6a9a7456fbd327d68d622e1a0":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_ca44b407930d4f14a3ae4ffe4446a82c","max":945,"min":0,"orientation":"horizontal","style":"IPY_MODEL_4d2ae829824746b8901cd695866895c7","value":945}},"80c383b19d8a4a86a00a8dd0d3930cc9":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8a66059da5b64302a46cce14a171eb60","placeholder":"​","style":"IPY_MODEL_b7899210a2074e6c887503a6c2533483","value":" 945/945 [00:31&lt;00:00, 30.61it/s]"}},"433954466de74b258576085156e35f27":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7780b28d28944b89b79a6c2add4d6454":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"69db87212c6347c6946099efb275a53e":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"ca44b407930d4f14a3ae4ffe4446a82c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"4d2ae829824746b8901cd695866895c7":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8a66059da5b64302a46cce14a171eb60":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"b7899210a2074e6c887503a6c2533483":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"91eed9a54cf74f5995b8c7de8b5ef70a":{"model_module":"@jupyter-widgets/controls","model_name":"VBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"VBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"VBoxView","box_style":"","children":["IPY_MODEL_e191e1688e744c4ab16e1f4faedb55b4","IPY_MODEL_0816b2a329154ff7a654c1904003ab3d"],"layout":"IPY_MODEL_7822ed23001d4f659a939e90f00dcca6"}},"e191e1688e744c4ab16e1f4faedb55b4":{"model_module":"@jupyter-widgets/controls","model_name":"LabelModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"LabelModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"LabelView","description":"","description_tooltip":null,"layout":"IPY_MODEL_7dd69b9fb04f4e02b6a80eeb60ca7ebc","placeholder":"​","style":"IPY_MODEL_90fb60b87a6b4dada63b0534e0481e31","value":"Waiting for wandb.init()...\r"}},"0816b2a329154ff7a654c1904003ab3d":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_8a23e12cbe5d44cb9d80df943c0d8e3e","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_f50ee4d2f5c24937bbb97bc8da6e9403","value":1}},"7822ed23001d4f659a939e90f00dcca6":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"7dd69b9fb04f4e02b6a80eeb60ca7ebc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"90fb60b87a6b4dada63b0534e0481e31":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"8a23e12cbe5d44cb9d80df943c0d8e3e":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"f50ee4d2f5c24937bbb97bc8da6e9403":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"bc20f841c86548b5b77fde6342c047de":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_aa91efaaa8be489892429bfa09e264c8","IPY_MODEL_76c4db9c6aeb43ca8349f4607d30f429","IPY_MODEL_cfc9242425f14d8bbc7048bfd4f583ce"],"layout":"IPY_MODEL_7fa86a47936a4ac98dc537108dfb0399"}},"aa91efaaa8be489892429bfa09e264c8":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_d62eeb6f22054836be97edf43290140d","placeholder":"​","style":"IPY_MODEL_9228f8e0af2a42388581f303a6679d1a","value":"Train:  46%"}},"76c4db9c6aeb43ca8349f4607d30f429":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"","description":"","description_tooltip":null,"layout":"IPY_MODEL_87bb2d903e2e4a08a2310fa0dff16cc5","max":17623,"min":0,"orientation":"horizontal","style":"IPY_MODEL_c9ac4c2a5c26402c84ea35d23af7e8fa","value":8149}},"cfc9242425f14d8bbc7048bfd4f583ce":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_cba7f9cb77ac4b0f8221003832689634","placeholder":"​","style":"IPY_MODEL_e21e1c00614848fa971c3b235eb44fbb","value":" 8149/17623 [13:56&lt;15:05, 10.46it/s, acc=90.8403%, loss=0.2459]"}},"7fa86a47936a4ac98dc537108dfb0399":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":"inline-flex","flex":null,"flex_flow":"row wrap","grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"100%"}},"d62eeb6f22054836be97edf43290140d":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"9228f8e0af2a42388581f303a6679d1a":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"87bb2d903e2e4a08a2310fa0dff16cc5":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":"2","flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"c9ac4c2a5c26402c84ea35d23af7e8fa":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"cba7f9cb77ac4b0f8221003832689634":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"e21e1c00614848fa971c3b235eb44fbb":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"nbformat":4,"nbformat_minor":0}