test w2v-bert

evaluate.py

@@ -17,7 +17,7 @@ def load_metadata(file_path):
  lines = f.readlines()
  for line in lines:
  line = line.strip()
- label = line.split(" ")[5]
+ label = line.split(" ")[2]
  labels.append(label)
  return labels
 
@@ -40,8 +40,8 @@ def load_metadata_from_proto(meta_file_path, proto_file_path):
  lines = f.readlines()
  for line in lines:
  line = line.strip()
- file_name = line.split(" ")[1]
- label = line.split(" ")[5]
+ file_name = line.split(" ")[0]
+ label = line.split(" ")[2]
  if file_name in protos:
  index = protos.index(file_name)
  labels[index] = label
@@ -167,8 +167,8 @@ def calculate_EER(scores, labels):
  # and bonafide otherwise
 
  # create two lists: one for the labels and one for the predictions
- # labels = metadata
- labels = load_metadata_from_proto(args.metadata_file, args.protocol_file)
+ labels = metadata
+ # labels = load_metadata_from_proto(args.metadata_file, args.protocol_file)
  predictions = []
  for i, file_name in enumerate(proto):
  score = scores[i]
@@ -194,5 +194,5 @@ def calculate_EER(scores, labels):
  print(f"TN = {cm[1][1]}")
  print(f"FP = {cm[0][1]}")
  print(f"FN = {cm[1][0]}")
- 
+
  calculate_EER(scores, labels)

models/__init__.py

@@ -1,3 +1,4 @@
 # from .lcnn import *
 # from .senet import *
 # from .occm import *
+# from ..seamless_communication.src.seamless_communication.models.conformer_shaw import load_conformer_shaw_model

models/w2vassist.py

@@ -12,12 +12,8 @@
 from fairseq2.nn.padding import get_seqs_and_padding_mask
 from fairseq2.data import Collater
 from pathlib import Path
-from seamless_communication.src.seamless_communication.models.conformer_shaw import load_conformer_shaw_model
-
-
-
-
 
+from seamless_communication.src.seamless_communication.models.conformer_shaw import load_conformer_shaw_model
 
 
 ___author__ = "Long Nguyen-Vu"
@@ -596,7 +592,8 @@ def forward(self, x):
 
  return emb, output
 
-if __name__ == '__main__':
+
+def main():
  import librosa
 
  model = AModel(None,"cuda").to("cuda")
@@ -605,3 +602,6 @@ def forward(self, x):
  emb, out = model(torch.Tensor(audio_data).unsqueeze(0).to("cuda"))
  print(emb.shape)
  print(out.shape)
+
+if __name__ == '__main__':
+ main()

oc_classifier.py

@@ -1,27 +1,17 @@
 import os
 import argparse
-import librosa
 import torch
 import torch.nn.functional as F
 from torch.nn import DataParallel
 from torch.utils.data import Dataset
-from torch.utils.data import DataLoader, random_split
+from torch.utils.data import DataLoader
 
-import numpy as np
-from torchattacks import PGD
+from models.w2vassist import *
 
-
-from models.lcnn import *
-from models.senet import *
-from models.xlsr import *
-from models.sslassist import *
-
-from losses.custom_loss import compactness_loss, descriptiveness_loss, euclidean_distance_loss
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 
-# to be used with one-class classifier
-# input is now a raw audio file
+
 
 
 class ASVDataset(Dataset):
@@ -51,9 +41,6 @@ def __init__(self, protocol_file, dataset_dir, eval=False):
  self.file_list = []
  self.label_list = [] 
  self.eval = eval
- # file_list is now the second column of the protocol file
- # label list is now the fifth column of the protocol file
- # read the protocol file
 
  if self.eval:
  # collect all files
@@ -73,12 +60,37 @@ def __init__(self, protocol_file, dataset_dir, eval=False):
  for line in lines:
  line = line.strip()
  line = line.split(" ")
+ # self.file_list.append(line[0])
+ # self.label_list.append("bonafide") # bonafide only
  if line[4] == "bonafide":
  self.file_list.append(line[1])
  self.label_list.append(line[4]) # bonafide only
 
  self._length = len(self.file_list) 
+ def _preprocess_wav(self, audio_wav_path):
+ # Create a device and dtype 
+ dtype = torch.float32
+ device = torch.device("cuda")
+ audio_decoder = AudioDecoder(dtype=dtype, device=device)
+ fbank_converter = WaveformToFbankConverter(
+ num_mel_bins=80,
+ waveform_scale=2**15,
+ channel_last=True,
+ standardize=True,
+ device=device,
+ dtype=dtype,
+ )
+ collater = Collater(pad_value=1)
 
+
+ with Path(audio_wav_path).open("rb") as fb:
+ block = MemoryBlock(fb.read())
+ # print("Decoding audio...")
+ decoded_audio = audio_decoder(block)
+ src = collater(fbank_converter(decoded_audio))["fbank"]
+ seqs, padding_mask = get_seqs_and_padding_mask(src)
+ return seqs
+
  def __len__(self):
  return self._length
 
@@ -89,8 +101,7 @@ def __getitem__(self, idx):
  file_path = os.path.join(self.dataset_dir, audio_file + ".flac")
  if not os.path.exists(file_path):
  file_path = os.path.join(self.dataset_dir, audio_file + ".wav")
-
- feature, _ = librosa.load(file_path, sr=None)
+ feature = self._preprocess_wav(file_path)
  feature_tensors = torch.tensor(feature, dtype=torch.float32)
 
  if self.eval == False:
@@ -179,10 +190,10 @@ def create_reference_embedding2(model, dataloader, device):
  total_distances = []
 
  with torch.no_grad():
- for _, (data, target) in enumerate(dataloader):
+ for _, (data, _) in enumerate(dataloader):
  data = data.to(device)
- target = target.to(device)
- emb, out = model(data) # torch.Size([1, 160])
+ data.squeeze_(0)
+ emb, out = model(data)
  total_embeddings.append(emb)
 
  # reference embedding is the mean of all embeddings
@@ -255,6 +266,7 @@ def score_eval_set_1c2(model, dataloader, device, reference_embedding, threshold
  with torch.no_grad():
  for idx, (data, target) in enumerate(dataloader):
  data = data.to(device)
+ data.squeeze_(0)
  target = target.to(device)
  emb, out = model(data)
  print(f"Processing file counts: {idx} ...")
@@ -264,7 +276,6 @@ def score_eval_set_1c2(model, dataloader, device, reference_embedding, threshold
  else:
  f.write(f"{float(distance)}, 0 \n")
 
-
 def score_eval_set_2c1(extractor, encoder, dataloader, device):
  """TWO-CLASS APPROACH:
  Score the evaluation set and save the scores to a file
@@ -338,24 +349,17 @@ def score_eval_set_2c2(model, dataloader, device):
 
  # load pretrained weights
  aasist.load_state_dict(torch.load(args.pretrained_sslaasist))
- # ssl.load_state_dict(torch.load(args.pretrained_ssl))
- # senet.load_state_dict(torch.load(args.pretrained_senet))
  aasist = DataParallel(aasist)
- # senet = DataParallel(senet)
- # ssl = DataParallel(ssl)
  print("Pretrained weights loaded")
 
  # create a reference embedding & find a threshold
  train_dataset = ASVDataset(args.protocol_file, args.dataset_dir)
  train_dataloader = DataLoader(train_dataset, batch_size=1, shuffle=False, num_workers=0)
- # reference_embedding, threshold = create_reference_embedding(ssl, senet, train_dataloader, device)
  reference_embedding, threshold = create_reference_embedding2(aasist, train_dataloader, device)
 
  # score the evaluation set
  eval_dataset = ASVDataset(args.eval_protocol_file, args.eval_dataset_dir, eval=True)
  eval_dataloader = DataLoader(eval_dataset, batch_size=1, shuffle=False, num_workers=0)
- # score_eval_set(ssl, senet, eval_dataloader, device, reference_embedding, threshold)
  score_eval_set_1c2(aasist, eval_dataloader, device, reference_embedding, threshold)
- # score_eval_set_2c2(aasist, eval_dataloader, device)
 
  print(f"threshold = {threshold}")

oc_training.py

@@ -335,38 +335,22 @@ def collate_fn(self, batch):
  # Define the collate function
 
  train_dataset = PFDataset(args.train_protocol_file, dataset_dir=args.train_dataset_dir)
- # test_dataset = PFDataset(args.test_protocol_file, dataset_dir=args.test_dataset_dir)
 
  # Create dataloaders for training and validation
  batch_size = 1
 
  print("Creating dataloaders...")
- # train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0, collate_fn=train_dataset.collate_fn)
  train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, num_workers=0)
 
- # test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=0, collate_fn=test_dataset.collate_fn)
-
  print("Instantiating model...")
  device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
 
  aasist = AModel(None, device).to(device)
- # ssl = SSLModel(device)
- # senet34 = se_resnet34().to(device)
- # lcnn = lcnn_net(asoftmax=False).to(device)
+
  optimizer = optim.Adam(aasist.parameters(), lr=0.00001)
- # optimizer = optim.Adam(list(ssl.parameters()) + list(senet34.parameters()) + list(lcnn.parameters()), lr=0.0001)
- # optimizer = optim.Adam(list(ssl.parameters()) + list(senet34.parameters()), lr=0.00001, weight_decay=0.0005)
 
  aasist = DataParallel(aasist)
 
- # ssl = DataParallel(ssl)
- # senet34 = DataParallel(senet34)
- # lcnn = DataParallel(lcnn)
-
- # if args.model == "lcnn_net_asoftmax":
- # criterion = AngleLoss()
-
-
  # WandB – Initialize a new run
  wandb.init(project="oc_classifier-w2v", entity="longnv")
 
@@ -382,9 +366,7 @@ def collate_fn(self, batch):
 
  # Training phase
  aasist.train()
- # ssl.eval()
- # senet34.train()
- # lcnn.train()
+
 
  running_loss = 0.0
  running_closs = 0.0
@@ -422,7 +404,4 @@ def collate_fn(self, batch):
  wandb.log({"Epoch": epoch, "Train Loss": running_loss / (i+1), "Train Compactness Loss": running_closs / (i+1), "Train Descriptiveness Loss": running_dloss / (i+1)})
  # save the models after each epoch
  print("Saving the models...")
- # torch.save(ssl.module.state_dict(), f"ssl_vocoded_{epoch}.pt")
- # torch.save(senet34.module.state_dict(), f"senet34_vocoded_{epoch}.pt")
  torch.save(aasist.module.state_dict(), f"w2v_vocoded_{epoch}.pt")
- # torch.save(lcnn.module.state_dict(), f"lcnn_{epoch}.pt")