simplify

Author: OlaWod

Utils/JDC/model.py

@@ -134,7 +134,7 @@ def forward(self, x):
         # sizes: (b, 31, 722), (b, 31, 2)
         # classifier output consists of predicted pitch classes per frame
         # detector output consists of: (isvoice, notvoice) estimates per frame
-        return torch.abs(classifier_out.squeeze()), GAN_feature, poolblock_out
+        return torch.abs(classifier_out.squeeze(-1)), GAN_feature, poolblock_out
 
     @staticmethod
     def init_weights(m):

convert_mp.py

@@ -16,7 +16,6 @@
 from env import AttrDict
 from meldataset import mel_spectrogram, MAX_WAV_VALUE
 from models import Generator
-from stft import TorchSTFT
 from Utils.JDC.model import JDCNet
 from asv import compute_similarity2, compute_embedding, get_asv_models
 
@@ -37,8 +36,8 @@ def get_sim(y, emb_tgts, embedding_models, feature_extractor):
     return similarity
 
 
-def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search):
-    y = generator.infer(x, initial_f0, stft)
+def get_best_wav(x, initial_f0, wav_tgt, generator, embedding_models, feature_extractor, search):
+    y = generator.infer(x, initial_f0)
     if not search:
         return y
 
@@ -62,7 +61,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             lf0 = initial_lf0 + step * i
             f0 = torch.exp(lf0)
             f0 = torch.where(voiced, f0, initial_f0)
-            y = generator.infer(x, initial_f0, stft)
+            y = generator.infer(x, initial_f0)
 
             similarity = get_sim(y, emb_tgts, embedding_models, feature_extractor)
 
@@ -78,7 +77,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
     return best_wav
 
 
-def process_one(line, generator, stft, wavlm, embedding_models, feature_extractor, device, args, h, spk2id, f0_stats):
+def process_one(line, generator, wavlm, embedding_models, feature_extractor, device, args, h, spk2id, f0_stats):
     with torch.no_grad():
         title, src_wav, tgt_wav, tgt_spk, tgt_emb = line.strip().split("|")
 
@@ -90,6 +89,7 @@ def process_one(line, generator, stft, wavlm, embedding_models, feature_extracto
         spk_emb = torch.from_numpy(spk_emb).unsqueeze(0).to(device)
 
         f0_mean_tgt = f0_stats[tgt_spk]["mean"]
+        f0_mean_tgt = torch.FloatTensor([f0_mean_tgt]).unsqueeze(0).to(device)
 
         wav_tgt, sr = librosa.load(tgt_wav, sr=16000)
         wav_tgt = torch.FloatTensor(wav_tgt).to(device)
@@ -106,7 +106,7 @@ def process_one(line, generator, stft, wavlm, embedding_models, feature_extracto
         # cvt
         f0 = generator.get_f0(mel, f0_mean_tgt)
         x = generator.get_x(x, spk_emb, spk_id)
-        y = get_best_wav(x, f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search=args.search)
+        y = get_best_wav(x, f0, wav_tgt, generator, embedding_models, feature_extractor, search=args.search)
 
         audio = y.squeeze()
         audio = audio / torch.max(torch.abs(audio)) * 0.95
@@ -128,7 +128,6 @@ def process_batch(batch, args, h, spk2id, f0_stats):
     # load models
     F0_model = JDCNet(num_class=1, seq_len=192)
     generator = Generator(h, F0_model).to(device)
-    stft = TorchSTFT(filter_length=h.gen_istft_n_fft, hop_length=h.gen_istft_hop_size, win_length=h.gen_istft_n_fft).to(device)
 
     state_dict_g = torch.load(args.ptfile, map_location=device)
     generator.load_state_dict(state_dict_g['generator'], strict=True)
@@ -156,7 +155,7 @@ def process_batch(batch, args, h, spk2id, f0_stats):
     rank = rank[0] if len(rank) > 0 else 0
 
     for line in tqdm(batch, position=rank):
-        process_one(line, generator, stft, wavlm, embedding_models, feature_extractor, device, args, h, spk2id, f0_stats)
+        process_one(line, generator, wavlm, embedding_models, feature_extractor, device, args, h, spk2id, f0_stats)
 
 
 if __name__ == "__main__":

convert_sp.py

@@ -14,7 +14,6 @@
 from env import AttrDict
 from meldataset import mel_spectrogram, MAX_WAV_VALUE
 from models import Generator
-from stft import TorchSTFT
 from Utils.JDC.model import JDCNet
 from asv import compute_similarity2, compute_embedding, get_asv_models
 
@@ -35,8 +34,8 @@ def get_sim(y, emb_tgts, embedding_models, feature_extractor):
     return similarity
 
 
-def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search):
-    y = generator.infer(x, initial_f0, stft)
+def get_best_wav(x, initial_f0, wav_tgt, generator, embedding_models, feature_extractor, search):
+    y = generator.infer(x, initial_f0)
     if not search:
         return y
 
@@ -60,7 +59,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             lf0 = initial_lf0 + step * i
             f0 = torch.exp(lf0)
             f0 = torch.where(voiced, f0, initial_f0)
-            y = generator.infer(x, initial_f0, stft)
+            y = generator.infer(x, initial_f0)
 
             similarity = get_sim(y, emb_tgts, embedding_models, feature_extractor)
 
@@ -104,7 +103,6 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
     # load models
     F0_model = JDCNet(num_class=1, seq_len=192)
     generator = Generator(h, F0_model).to(device)
-    stft = TorchSTFT(filter_length=h.gen_istft_n_fft, hop_length=h.gen_istft_hop_size, win_length=h.gen_istft_n_fft).to(device)
 
     state_dict_g = torch.load(args.ptfile, map_location=device)
     generator.load_state_dict(state_dict_g['generator'], strict=True)
@@ -150,6 +148,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             spk_emb = torch.from_numpy(spk_emb).unsqueeze(0).to(device)
 
             f0_mean_tgt = f0_stats[tgt_spk]["mean"]
+            f0_mean_tgt = torch.FloatTensor([f0_mean_tgt]).unsqueeze(0).to(device)
 
             wav_tgt, sr = librosa.load(tgt_wav, sr=16000)
             wav_tgt = torch.FloatTensor(wav_tgt).to(device)
@@ -166,7 +165,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             # cvt
             f0 = generator.get_f0(mel, f0_mean_tgt)
             x = generator.get_x(x, spk_emb, spk_id)
-            y = get_best_wav(x, f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search=args.search)
+            y = get_best_wav(x, f0, wav_tgt, generator, embedding_models, feature_extractor, search=args.search)
 
             audio = y.squeeze()
             audio = audio / torch.max(torch.abs(audio)) * 0.95

metrics/macs/macs.py

@@ -14,18 +14,16 @@
 from env import AttrDict
 from meldataset import mel_spectrogram, MAX_WAV_VALUE
 from models import Generator
-from stft import TorchSTFT
 from Utils.JDC.model import JDCNet
 from asv import compute_similarity2, compute_embedding, get_asv_models
 
 from thop import profile, clever_format
 
 
 class Model(torch.nn.Module):
-    def __init__(self, generator, stft, wavlm):
+    def __init__(self, generator, wavlm):
         super().__init__()
         self.generator = generator
-        self.stft = stft
         self.wavlm = wavlm
 
     def forward(self, wav, mel, f0_mean_tgt, spk_emb, spk_id):
@@ -35,17 +33,25 @@ def forward(self, wav, mel, f0_mean_tgt, spk_emb, spk_id):
 
         f0 = self.generator.get_f0(mel, f0_mean_tgt)
         x = self.generator.get_x(x, spk_emb, spk_id)
-        y = self.generator.infer(x, f0, self.stft)
+        y = self.generator.infer(x, f0)
+
+
+class F0(torch.nn.Module):
+    def __init__(self, generator):
+        super().__init__()
+        self.generator = generator
+
+    def forward(self, mel, f0_mean_tgt):
+        f0 = self.generator.get_f0(mel, f0_mean_tgt)
 
 
 class Voc(torch.nn.Module):
-    def __init__(self, generator, stft):
+    def __init__(self, generator):
         super().__init__()
         self.generator = generator
-        self.stft = stft
 
     def forward(self, x, f0):
-        y = self.generator.infer(x, f0, self.stft)
+        y = self.generator.infer(x, f0)
 
 
 class Enc(torch.nn.Module):
@@ -104,7 +110,6 @@ def forward(self, spk_id, spk_emb):
     # load models
     F0_model = JDCNet(num_class=1, seq_len=192)
     generator = Generator(h, F0_model)#.to(device)
-    stft = TorchSTFT(filter_length=h.gen_istft_n_fft, hop_length=h.gen_istft_hop_size, win_length=h.gen_istft_n_fft)#.to(device)
 
     # state_dict_g = torch.load(args.ptfile, map_location=device)
     # generator.load_state_dict(state_dict_g['generator'], strict=True)
@@ -126,8 +131,9 @@ def forward(self, spk_id, spk_emb):
         lines = f.readlines()
 
     # define model & modules
-    model = Model(generator, stft, wavlm)#.to(device)
-    mvoc = Voc(generator, stft)#.to(device)
+    model = Model(generator, wavlm)#.to(device)
+    mf0 = F0(generator)#.to(device)
+    mvoc = Voc(generator)#.to(device)
     menc = Enc(generator)#.to(device)
     mdec = Dec(generator)#.to(device)
     mspk = Spk(generator)#.to(device)
@@ -145,6 +151,7 @@ def forward(self, spk_id, spk_emb):
         spk_emb = torch.from_numpy(spk_emb).unsqueeze(0)#.to(device)
 
         f0_mean_tgt = f0_stats[tgt_spk]["mean"]
+        f0_mean_tgt = torch.FloatTensor([f0_mean_tgt]).unsqueeze(0)#.to(device)
 
         wav_tgt, sr = librosa.load(tgt_wav, sr=16000)
         wav_tgt = torch.FloatTensor(wav_tgt)#.to(device)
@@ -160,6 +167,12 @@ def forward(self, spk_id, spk_emb):
         macs, params = clever_format([macs, params], "%.3f")
         print(macs, params)
 
+        # macs: f0
+        print("--- f0 ---")
+        macs, params = profile(mf0, inputs=(mel, f0_mean_tgt))
+        macs, params = clever_format([macs, params], "%.3f")
+        print(macs, params)
+
         # macs: wavlm
         print("--- wavlm ---")
         macs, params = profile(wavlm, inputs=(wav.unsqueeze(0),))
@@ -205,7 +218,7 @@ def forward(self, spk_id, spk_emb):
         # cvt
         # f0 = generator.get_f0(mel, f0_mean_tgt)
         # x = generator.get_x(x, spk_emb, spk_id)
-        # y = get_best_wav(x, f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search=args.search)
+        # y = get_best_wav(x, f0, wav_tgt, generator, embedding_models, feature_extractor, search=args.search)
 
         # audio = y.squeeze()
         # audio = audio / torch.max(torch.abs(audio)) * 0.95

metrics/rtf/rtf.py

@@ -15,7 +15,6 @@
 from env import AttrDict
 from meldataset import mel_spectrogram, MAX_WAV_VALUE
 from models import Generator
-from stft import TorchSTFT
 from Utils.JDC.model import JDCNet
 from asv import compute_similarity2, compute_embedding, get_asv_models
 
@@ -36,8 +35,8 @@ def get_sim(y, emb_tgts, embedding_models, feature_extractor):
     return similarity
 
 
-def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search):
-    y = generator.infer(x, initial_f0, stft)
+def get_best_wav(x, initial_f0, wav_tgt, generator, embedding_models, feature_extractor, search):
+    y = generator.infer(x, initial_f0)
     if not search:
         return y
 
@@ -61,7 +60,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             lf0 = initial_lf0 + step * i
             f0 = torch.exp(lf0)
             f0 = torch.where(voiced, f0, initial_f0)
-            y = generator.infer(x, initial_f0, stft)
+            y = generator.infer(x, initial_f0)
 
             similarity = get_sim(y, emb_tgts, embedding_models, feature_extractor)
 
@@ -105,7 +104,6 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
     # load models
     F0_model = JDCNet(num_class=1, seq_len=192)
     generator = Generator(h, F0_model).to(device)
-    stft = TorchSTFT(filter_length=h.gen_istft_n_fft, hop_length=h.gen_istft_hop_size, win_length=h.gen_istft_n_fft).to(device)
 
     state_dict_g = torch.load(args.ptfile, map_location=device)
     generator.load_state_dict(state_dict_g['generator'], strict=True)
@@ -153,6 +151,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             spk_emb = torch.from_numpy(spk_emb).unsqueeze(0).to(device)
 
             f0_mean_tgt = f0_stats[tgt_spk]["mean"]
+            f0_mean_tgt = torch.FloatTensor([f0_mean_tgt]).unsqueeze(0).to(device)
 
             wav_tgt, sr = librosa.load(tgt_wav, sr=16000)
             wav_tgt = torch.FloatTensor(wav_tgt).to(device)
@@ -172,7 +171,7 @@ def get_best_wav(x, initial_f0, wav_tgt, generator, stft, embedding_models, feat
             # cvt
             f0 = generator.get_f0(mel, f0_mean_tgt)
             x = generator.get_x(x, spk_emb, spk_id)
-            y = get_best_wav(x, f0, wav_tgt, generator, stft, embedding_models, feature_extractor, search=args.search)
+            y = get_best_wav(x, f0, wav_tgt, generator, embedding_models, feature_extractor, search=args.search)
 
             rtf = (time.time() - start) / length_y
             total_rtf += rtf

metrics/rtf/run.sh

@@ -1,7 +1,7 @@
 mv rtf.py ../..
 cd ../..
 
-CUDA_VISIBLE_DEVICES=0 python rtf.py --hpfile config_v1_16k.json --ptfile exp/default/g_00700000 --txtpath test/txts/u2s.txt 
+CUDA_VISIBLE_DEVICES=-1 python rtf.py --hpfile config_v1_16k.json --ptfile exp/default/g_00700000 --txtpath test/txts/s2s.txt 
 
 mv rtf.py metrics/rtf
 cd metrics/rtf