Merge tag 'rs-experiment' of kdm00:/mnt/syno128/volume1/fujimotos/git/lhotse

Experiemntal version for ReazonSpeech

Author: Chen

lhotse/bin/modes/recipes/__init__.py

@@ -62,6 +62,7 @@
 from .peoples_speech import *
 from .primewords import *
 from .rir_noise import *
+from .reazonspeech import *
 from .speechcommands import *
 from .spgispeech import *
 from .stcmds import *

lhotse/bin/modes/recipes/reazonspeech.py

@@ -0,0 +1,15 @@
+import click
+
+from lhotse.bin.modes import prepare
+from lhotse.recipes.reazonspeech import prepare_reazonspeech
+from lhotse.utils import Pathlike
+
+__all__ = ["reazonspeech"]
+
+
+@prepare.command(context_settings=dict(show_default=True))
+@click.argument("corpus_dir", type=click.Path(exists=True, dir_okay=True))
+@click.argument("output_dir", type=click.Path())
+def reazonspeech(corpus_dir: Pathlike, output_dir: Pathlike):
+    """ReazonSpeech data preparation."""
+    prepare_reazonspeech(corpus_dir, output_dir=output_dir)

lhotse/recipes/__init__.py

@@ -63,6 +63,7 @@
 from .nsc import prepare_nsc
 from .peoples_speech import prepare_peoples_speech
 from .rir_noise import download_rir_noise, prepare_rir_noise
+from .reazonspeech import prepare_reazonspeech
 from .speechcommands import download_speechcommands, prepare_speechcommands
 from .spgispeech import download_spgispeech, prepare_spgispeech
 from .stcmds import download_stcmds, prepare_stcmds

lhotse/recipes/reazonspeech.py

@@ -0,0 +1,65 @@
+import json
+import logging
+from collections import defaultdict
+from pathlib import Path
+from typing import Dict, Optional, Union
+
+from lhotse import validate_recordings_and_supervisions
+from lhotse.audio import Recording, RecordingSet
+from lhotse.qa import fix_manifests
+from lhotse.supervision import SupervisionSegment, SupervisionSet
+from lhotse.utils import Pathlike
+
+
+def prepare_reazonspeech(
+    corpus_dir: Pathlike,
+    output_dir: Optional[Pathlike] = None,
+) -> Dict[str, Dict[str, Union[RecordingSet, SupervisionSet]]]:
+
+    corpus_dir = Path(corpus_dir)
+    assert corpus_dir.is_dir(), f"No such directory: {corpus_dir}"
+    manifests = defaultdict(dict)
+
+    if output_dir is not None:
+        output_dir = Path(output_dir)
+        output_dir.mkdir(parents=True, exist_ok=True)
+
+    idx = 0
+    for part in ["train", "valid", "test"]:
+        recordings = []
+        supervisions = []
+        with open("%s/%s.json" % (corpus_dir, part)) as fp:
+            for line in fp:
+                line = line.strip()
+                if not line:
+                    continue
+                item = json.loads(line)
+                recordings.append(
+                    Recording.from_file(item['audio_filepath'], recording_id=str(idx))
+                )
+                supervisions.append(SupervisionSegment(
+                    id=str(idx),
+                    recording_id=str(idx),
+                    start=0.0,
+                    duration=item['duration'],
+                    channel=0,
+                    language="Japanese",
+                    speaker=str(idx),
+                    text=item['text']
+                ))
+                idx += 1
+
+        recording_set = RecordingSet.from_recordings(recordings)
+        supervision_set = SupervisionSet.from_segments(supervisions)
+        recording_set, supervision_set = fix_manifests(recording_set, supervision_set)
+        validate_recordings_and_supervisions(recording_set, supervision_set)
+
+        if output_dir is not None:
+            supervision_set.to_file(
+                output_dir / f"reazonspeech_supervisions_{part}.jsonl.gz"
+            )
+            recording_set.to_file(output_dir / f"reazonspeech_recordings_{part}.jsonl.gz")
+
+        manifests[part] = {"recordings": recording_set, "supervisions": supervision_set}
+
+    return manifests