QOL improvements and updates for FFNN baseline

Author: rhgrossman

code/Step4_RemoveRedundancy/bert/model.py

@@ -17,7 +17,6 @@ def __init__(
         super(MatchHead, self).__init__()
         self.GRU_1 = nn.GRU(base_model_feature_size, rnn_dimension, bidirectional=False)
         self.GRU_2 = nn.GRU(base_model_feature_size, rnn_dimension, bidirectional=False)
-        self.linear_1 = nn.Linear(additional_feature_size, linear_1_dimension)
         self.linear_1 = nn.Linear(rnn_dimension * 2 + additional_feature_size, linear_1_dimension)
         self.linear_2 = nn.Linear(linear_1_dimension, num_classes)
 
@@ -47,6 +46,7 @@ def forward(self, data_1, data_2, additional_feats):
 
         return sigmoid_output
 
+
 class MatchArchitecture(nn.Module):
     "Transformer base model for matching."
     def __init__(
@@ -108,8 +108,54 @@ def forward(
         sequence_output_2 = outputs_2[0]
 
         match_classification = self.match_head(sequence_output_1, sequence_output_2, additional_feats)
-        #match_classification = self.match_head(None, None, additional_feats)
 
         return match_classification
 
 
+class FFMatchHead(nn.Module):
+    """Roberta Head for Matching."""
+    def __init__(
+        self,
+        additional_feature_size,
+        num_classes,
+        linear_1_dimension,
+    ):
+        """Model architecture definition for the capitalization model in torch."""
+        super(MatchHead, self).__init__()
+        self.linear_1 = nn.Linear(additional_feature_size, linear_1_dimension)
+        self.linear_2 = nn.Linear(linear_1_dimension, num_classes)
+
+    def forward(self, additional_feats):
+        """Forward pass"""
+
+        # batch second is faster
+        linear_input = additional_feats
+        linear_output = self.linear_1(linear_input)
+        activated_linear_output = F.relu(linear_output)
+        pre_sigmoid_output = self.linear_2(activated_linear_output)
+        sigmoid_output = F.sigmoid(pre_sigmoid_output)
+
+        return sigmoid_output
+
+
+class FFMatchArchitecture(nn.Module):
+    "Transformer base model for matching."
+    def __init__(
+        self,
+        additional_feature_size,
+        num_classes,
+        linear_1_dimension,
+    ):
+        super(FFMatchArchitecture, self).__init__()
+        self.match_head = FFMatchHead(
+            additional_feature_size, num_classes, linear_1_dimension
+        )
+
+    def forward(self, additional_feats):
+        """Forward pass"""
+
+        match_classification = self.match_head(additional_feats)
+        return match_classification
+
+
+

code/Step4_RemoveRedundancy/bert/train.py

@@ -17,10 +17,10 @@
 from model import MatchArchitecture
 from data_utils import MatchingDataset
 
-
+RANDOM_SEED = 117
 SEQ_LEN = 10
 RNN_DIM = 64
-LINEAR_DIM=64
+LINEAR_DIM = 64
 CLASSES = 1
 ROBERTA_FEAT_SIZE = 768
 ADDITIONAL_FEAT_SIZE = 0
@@ -53,7 +53,7 @@
         if "target" in additional_feats.columns:
             additional_feats.drop("target", axis=1, inplace=True)
         ADDITIONAL_FEAT_SIZE = additional_feats.shape[1]
-        kf = KFold(n_splits=5, random_state = 117, shuffle = True)
+        kf = KFold(n_splits=5, random_state = RANDOM_SEED, shuffle = True)
 
 
         # TODO:@Ray improve the fold selection
@@ -271,13 +271,8 @@ def _run_training_loop(model, train_config):
         else:
             temp_preds = sigmoid_output.cpu().detach().numpy()
             preds = np.concatenate([preds, temp_preds], axis=0)
-            temp_labels = y_batch.cpu().detach().numpy()
-            labels = np.concatenate([labels, temp_labels], axis =0)
 
-    assert(len(preds)==len(labels))
     oof_preds[val_inx, 0] = preds[:len(val_inx), 0]
-    oof_preds2[cur_oof_inx:cur_oof_inx + len(labels), 0] = preds[:len(val_inx), 0]
-    oof_labels[cur_oof_inx:cur_oof_inx + len(labels), 0] = labels[:len(val_inx), 0]
     cur_oof_inx += len(labels)
     del model
 
@@ -291,9 +286,3 @@ def _run_training_loop(model, train_config):
     print('Precision at {}: '.format(threshold), mt.precision_score(target, oof_preds > threshold))
 
 
-for threshold in thresholds:
-    print('F1 at {}: '.format(threshold), mt.f1_score(oof_labels, oof_preds2 > threshold))
-    print('Recall at {}: '.format(threshold), mt.recall_score(oof_labels, oof_preds2 > threshold))
-    print('Precision at {}: '.format(threshold), mt.precision_score(oof_labels, oof_preds2 > threshold))
-
-