Retrain or resume the checkpoint you provided, and there is a screeching current sound

[Liuxuetao1219]

We retrained on our own dataset or resumed with the checkpoints you provided, but we got poor results, especially the harsh electrical sound. We haven't changed the rest of the code, can you let us know if this is normal in the early days, and how long your checkpoint has been trained, thanks

[alibabasglab]

Hi, please let us know which model are you currently training on? How did you create your dataset and whether you have verified your dataset using other models?

[Liuxuetao1219]

Hi，we are using the model of SE GAN16K.

First is the code we used to generate reverberant speech, and then a few speeches are provided for you.

We have trained FullsubNet before your job and can match theirs.

def convolve_with_rir(clean_signal, rir_signal):
# 对 clean_signal 和 rir_signal 进行归一化
clean_signal = librosa.util.normalize(clean_signal)
rir_signal = librosa.util.normalize(rir_signal)
""" 使用 RIR 卷积干净语音生成混响语音 """
# 进行卷积
reverberated_signal = convolve(clean_signal, rir_signal, mode='full')[:len(clean_signal)]
# 对结果进行归一化，避免过大或过小
return librosa.util.normalize(reverberated_signal)

def get_files(input_folder, file_extension=".wav"):
""" 返回文件的路径列表，完整路径可用于读取相应xls中T60值 """
files = []
for root, dirs, files_in_dir in os.walk(input_folder):
for file in files_in_dir:
if file.endswith(file_extension):
files.append(os.path.join(root, file))
return files

def generate_output_filename(speech_file, rir_file, noise_file=None, snr=None):
""" 根据文件路径生成混响语音的文件名 """
# 获取干净语音文件名部分
speech_parts = speech_file.split('/')
speech_name = speech_parts[-3] + "" + speech_parts[-2] + "" + speech_parts[-1].split('.')[0]
# 获取 RIR 文件名部分
rir_parts = rir_file.split('/')
rir_name = rir_parts[-3] + "" + rir_parts[-2] + "" + rir_parts[-1].split('.')[0]

if noise_file and snr is not None:
    noise_parts = noise_file.split('/')
    noise_name = noise_parts[-1].split('.')[0]
    return f"{speech_name}__{rir_name}__{noise_name}__{snr}dB.wav"
else:
    return f"{speech_name}__{rir_name}.wav"

def add_noise_to_signal(signal, noise, snr):
""" 根据给定的 SNR 将噪声添加到信号中 """
Average_Energy_signal = np.sum(signal ** 2) / len(signal)
Average_Energy_noise = np.sum(noise ** 2) / len(noise)
# 根据 SNR 计算噪声缩放因子
scaling_factor = np.sqrt(Average_Energy_signal / Average_Energy_noise / 10 ** (snr * 0.1))

# 添加噪声
noisy_signal = signal + scaling_factor * noise
return noisy_signal

def random_noise_segment(noise, signal_length):
""" 从噪声信号中随机选择一个片段，长度与信号一致 """
# 如果噪声长度小于信号长度，重复噪声信号直到长度足够长
if len(noise) < signal_length:
# 将噪声重复多次，使其长度足够长
noise = np.tile(noise, (signal_length // len(noise)) + 1)

# 随机选择噪声片段的起始位置，确保不会超出噪声信号的长度
start_idx = random.randint(0, len(noise) - signal_length)

# 返回选择的噪声片段，长度与目标信号一致
return noise[start_idx:start_idx + signal_length]

if name == "main":
args = parser.parse_args()
# set_seed(42)
# 确保输出文件夹存在
if not os.path.exists(args.output_folder):
os.makedirs(args.output_folder)

# 获取 RIR 文件路径列表
rir_files = get_files(args.input_rir_folder, ".wav")
# 输出 RIR 文件路径列表
print(f"找到 {len(rir_files)} 个 RIR 文件:")
for rir_file in rir_files:
    print(rir_file)
    break

# 获取噪音语音文件路径列表
noise_files = get_files(args.noise_folder, ".wav")
print(f"找到 {len(noise_files)} 个噪音文件:")
for noise_file in noise_files:
    print(noise_file)
    break  

# 获取干净语音文件路径列表
speech_files = get_files(args.input_speech_folder, ".wav")
print(f"找到 {len(speech_files)} 个干净语音文件:")
for speech_file in speech_files:
    print(speech_file)
    break  

# 打开一个文件来记录保存的混响语音文件路径
with open(os.path.join(args.output_folder, "saved_reverb_files.txt"), "w") as log_file:
    for speech_file in tqdm(speech_files):
        # 随机选择一个 RIR 文件
        rir_file = random.choice(rir_files)
        # 随机选择一个噪声文件
        noise_file = random.choice(noise_files)
        # 随机选择一个 SNR
        snr = random.choice(args.snr)

        # 读取语音文件和 RIR 文件
        clean_signal, _ = librosa.load(speech_file, sr=args.fs, mono=False)
        rir_signal, _ = librosa.load(rir_file, sr=args.fs, mono=False)
        noise_signal, _ = librosa.load(noise_file, sr=args.fs, mono=False)

        # 确保干净语音和 RIR 信号的通道数一致
        if clean_signal.ndim == 1:
            clean_signal = np.expand_dims(clean_signal, axis=1)
        if rir_signal.ndim == 1:
            rir_signal = np.expand_dims(rir_signal, axis=1)

        try:
            # 对每个通道进行卷积生成混响语音
            reverberated_signal = np.array([convolve_with_rir(clean_signal[:, chan], rir_signal[:, 0]) for chan in range(clean_signal.shape[1])]).T

            # 从噪声中随机选择一个片段，长度与混响语音一致
            noise_segment = random_noise_segment(noise_signal, reverberated_signal.shape[0])
            
            # 将噪声添加到混响语音中
            noisy_reverberated_signal = np.array([add_noise_to_signal(reverberated_signal[:, chan], noise_segment, snr) for chan in range(reverberated_signal.shape[1])]).T
            
            # 生成混响语音文件名
            output_filename = generate_output_filename(speech_file, rir_file, noise_file, snr)
            output_path = os.path.join(args.output_folder, output_filename)

            # 归一化带噪声的混响语音
            noisy_reverberated_signal = noisy_reverberated_signal / np.max(np.abs(noisy_reverberated_signal))
            
            # 保存混响语音文件
            sf.write(output_path, reverberated_signal, args.fs)

[alibabasglab]

Hi, we found the training may rely heavily on the validation set. To reduce the dependence, we add a new line 216 in https://github.com/modelscope/ClearerVoice-Studio/blob/main/train/speech_enhancement/solver.py . You may pull the new code using git or just insert the following new line below line 215:

self.best_val_loss = val_loss

[fengsky401]

Hi, we have pulled the newest code, and trained the model. The phenomenon of harsh electrical sound has been alleviated a lot, but it still exists if listening carefully. Do you have suggestions?

[alibabasglab]

Hi, sorry we cannot repeat your outputs, so it is hard to tell where the problem comes. From what you have obtained from the updated code, it seems training longer can reduce the electric sound. That also means it might produce quite obvious electrical sound at beginning. Please try other dataset such as VoiceBank+DMENAD for verification.