add cycle gan

requiremets.txt

@@ -0,0 +1,13 @@
+numpy
+cupy
+chainer
+librosa
+pysptk
+pyworld
+fastdtw
+matplotlib
+chainerui
+tensorflow
+pillow
+git+https://github.com/neka-nat/tensorboard-chainer
+git+https://github.com/Hiroshiba/become-yukarin

train.py

@@ -13,12 +13,13 @@
 from chainer.iterators import MultiprocessIterator
 from chainer.training import extensions
 from chainerui.utils import save_args
+from tb_chainer import SummaryWriter
 
+from utility.chainer_utility import TensorBoardReport
 from yukarin.config import create_from_json
 from yukarin.dataset import create as create_dataset
 from yukarin.model import create
 from yukarin.updater import Updater
-from yukarin.utility.chainer_utility import TensorBoardReport
 
 parser = argparse.ArgumentParser()
 parser.add_argument('config_json_path', type=Path)
@@ -84,6 +85,7 @@ def create_optimizer(model):
 trigger_stop = (config.train.stop_iteration, 'iteration') if config.train.stop_iteration is not None else None
 
 trainer = training.Trainer(updater, stop_trigger=trigger_stop, out=arguments.output)
+tb_writer = SummaryWriter(Path(arguments.output))
 
 ext = extensions.Evaluator(test_iter, models, converter, device=config.train.gpu, eval_func=updater.forward)
 trainer.extend(ext, name='test', trigger=trigger_log)
@@ -96,7 +98,7 @@ def create_optimizer(model):
 trainer.extend(ext, trigger=trigger_snapshot)
 
 trainer.extend(extensions.LogReport(trigger=trigger_log))
-trainer.extend(TensorBoardReport(), trigger=trigger_log)
+trainer.extend(TensorBoardReport(writer=tb_writer), trigger=trigger_log)
 
 if trigger_stop is not None:
     trainer.extend(extensions.ProgressBar(trigger_stop))

train_cg.py

@@ -0,0 +1,108 @@
+import argparse
+from copy import copy
+from functools import partial
+from pathlib import Path
+from typing import Any
+from typing import Dict
+
+from chainer import cuda
+from chainer import optimizers
+from chainer import training
+from chainer.dataset import convert
+from chainer.iterators import MultiprocessIterator
+from chainer.training import extensions
+from chainerui.utils import save_args
+
+from utility.chainer_utility import TensorBoardReport
+from yukarin.config import create_from_json
+from yukarin.dataset import create_cg as create_dataset
+from yukarin.model import create
+from yukarin.updater import CGUpdater
+
+parser = argparse.ArgumentParser()
+parser.add_argument('config_json_path', type=Path)
+parser.add_argument('output', type=Path)
+arguments = parser.parse_args()
+
+config = create_from_json(arguments.config_json_path)
+arguments.output.mkdir(exist_ok=True)
+config.save_as_json((arguments.output / 'config.json').absolute())
+
+# model
+if config.train.gpu >= 0:
+    cuda.get_device_from_id(config.train.gpu).use()
+predictor_xy, discriminator_x = create(config.model)
+predictor_yx, discriminator_y = create(config.model)
+models = {
+    'predictor_xy': predictor_xy,
+    'predictor_yx': predictor_yx,
+    'discriminator_x': discriminator_x,
+    'discriminator_y': discriminator_y,
+}
+
+if config.train.pretrained_model is not None:
+    raise ValueError('cannot set pretrained model')
+
+# dataset
+dataset = create_dataset(config.dataset)
+train_iter = MultiprocessIterator(dataset['train'], config.train.batchsize)
+test_iter = MultiprocessIterator(dataset['test'], config.train.batchsize, repeat=False, shuffle=False)
+train_eval_iter = MultiprocessIterator(dataset['train_eval'], config.train.batchsize, repeat=False, shuffle=False)
+
+# optimizer
+def create_optimizer(model):
+    cp: Dict[str, Any] = copy(config.train.optimizer)
+    n = cp.pop('name').lower()
+
+    if n == 'adam':
+        optimizer = optimizers.Adam(**cp)
+    elif n == 'sgd':
+        optimizer = optimizers.SGD(**cp)
+    else:
+        raise ValueError(n)
+
+    optimizer.setup(model)
+    return optimizer
+
+
+opts = {key: create_optimizer(model) for key, model in models.items()}
+
+# updater
+converter = partial(convert.concat_examples, padding=0)
+updater = CGUpdater(
+    loss_config=config.loss,
+    predictor_xy=predictor_xy,
+    predictor_yx=predictor_yx,
+    discriminator_x=discriminator_x,
+    discriminator_y=discriminator_y,
+    device=config.train.gpu,
+    iterator=train_iter,
+    optimizer=opts,
+    converter=converter,
+)
+
+# trainer
+trigger_log = (config.train.log_iteration, 'iteration')
+trigger_snapshot = (config.train.snapshot_iteration, 'iteration')
+trigger_stop = (config.train.stop_iteration, 'iteration') if config.train.stop_iteration is not None else None
+
+trainer = training.Trainer(updater, stop_trigger=trigger_stop, out=arguments.output)
+
+ext = extensions.Evaluator(test_iter, models, converter, device=config.train.gpu, eval_func=updater.forward)
+trainer.extend(ext, name='test', trigger=trigger_log)
+ext = extensions.Evaluator(train_eval_iter, models, converter, device=config.train.gpu, eval_func=updater.forward)
+trainer.extend(ext, name='train', trigger=trigger_log)
+
+trainer.extend(extensions.dump_graph('predictor_xy/loss'))
+
+ext = extensions.snapshot_object(predictor_xy, filename='predictor_{.updater.iteration}.npz')
+trainer.extend(ext, trigger=trigger_snapshot)
+
+trainer.extend(extensions.LogReport(trigger=trigger_log))
+trainer.extend(TensorBoardReport(), trigger=trigger_log)
+
+if trigger_stop is not None:
+    trainer.extend(extensions.ProgressBar(trigger_stop))
+
+save_args(arguments, arguments.output)
+trainer.run()

yukarin/utility/chainer_utility.py → utility/chainer_utility.py

@@ -5,12 +5,12 @@
 
 
 class TensorBoardReport(chainer.training.Extension):
-    def __init__(self):
-        self.writer = None
+    def __init__(self, writer=None):
+        self.writer = writer
 
     def __call__(self, trainer: chainer.training.Trainer):
         if self.writer is None:
-            self.writer = SummaryWriter(Path(trainer.out) / Path(trainer.out).name)
+            self.writer = SummaryWriter(Path(trainer.out))
 
         observations = trainer.observation
         n_iter = trainer.updater.iteration

utility/extension_utility.py

@@ -0,0 +1,23 @@
+from pathlib import Path
+
+import chainer
+from tb_chainer import SummaryWriter
+
+
+class TensorBoardReport(chainer.training.Extension):
+    def __init__(self, writer=None):
+        self.writer = writer
+
+    def __call__(self, trainer: chainer.training.Trainer):
+        if self.writer is None:
+            self.writer = SummaryWriter(Path(trainer.out))
+
+        observations = trainer.observation
+        n_iter = trainer.updater.iteration
+        for n, v in observations.items():
+            if isinstance(v, chainer.Variable):
+                v = v.data
+            if isinstance(v, chainer.cuda.cupy.ndarray):
+                v = chainer.cuda.to_cpu(v)
+
+            self.writer.add_scalar(n, v, n_iter)

yukarin/dataset.py

@@ -119,7 +119,7 @@ def get_example(self, i):
         mask = random_crop(mask, seed=seed, crop_size=self.config.train_crop_size)
 
         if train:
-            input = add_noise(input, p_global=self.config.target_global_noise, p_local=self.config.target_local_noise)
+            input = add_noise(input, p_global=self.config.input_global_noise, p_local=self.config.input_local_noise)
             target = add_noise(target, p_global=self.config.target_global_noise, p_local=self.config.target_local_noise)
 
         return dict(
@@ -129,6 +129,60 @@ def get_example(self, i):
         )
 
 
+class CGDataset(chainer.dataset.DatasetMixin):
+    def __init__(self, x_paths: List[Path], y_paths: List[Path], config: DatasetConfig) -> None:
+        self.x_paths = x_paths
+        self.y_paths = y_paths
+        self.config = config
+
+    def __len__(self):
+        return min(len(self.x_paths), len(self.y_paths))
+
+    def get_example(self, i):
+        train = chainer.config.train
+
+        p_x = self.x_paths[numpy.random.randint(len(self.x_paths))]
+        p_y = self.y_paths[numpy.random.randint(len(self.y_paths))]
+
+        f_x = AcousticFeature.load(p_x)
+        x = encode_feature(f_x, targets=self.config.features)
+
+        f_y = AcousticFeature.load(p_y)
+        y = encode_feature(f_y, targets=self.config.features)
+
+        mask_x = encode_feature(make_mask(f_x), targets=self.config.features)
+        mask_y = encode_feature(make_mask(f_y), targets=self.config.features)
+
+        # padding
+        seed = numpy.random.randint(2 ** 32)
+        x = random_pad(x, seed=seed, min_size=self.config.train_crop_size)
+        mask_x = random_pad(mask_x, seed=seed, min_size=self.config.train_crop_size)
+
+        seed = numpy.random.randint(2 ** 32)
+        y = random_pad(y, seed=seed, min_size=self.config.train_crop_size)
+        mask_y = random_pad(mask_y, seed=seed, min_size=self.config.train_crop_size)
+
+        # crop
+        seed = numpy.random.randint(2 ** 32)
+        x = random_crop(x, seed=seed, crop_size=self.config.train_crop_size)
+        mask_x = random_crop(mask_x, seed=seed, crop_size=self.config.train_crop_size)
+
+        seed = numpy.random.randint(2 ** 32)
+        y = random_crop(y, seed=seed, crop_size=self.config.train_crop_size)
+        mask_y = random_crop(mask_y, seed=seed, crop_size=self.config.train_crop_size)
+
+        if train:
+            x = add_noise(x, p_global=self.config.input_global_noise, p_local=self.config.input_local_noise)
+            y = add_noise(y, p_global=self.config.target_global_noise, p_local=self.config.target_local_noise)
+
+        return dict(
+            x=x,
+            y=y,
+            mask_x=mask_x,
+            mask_y=mask_y,
+        )
+
+
 def create(config: DatasetConfig):
     input_paths = {Path(p).stem: Path(p) for p in glob.glob(str(config.input_glob))}
     target_paths = {Path(p).stem: Path(p) for p in glob.glob(str(config.target_glob))}
@@ -157,3 +211,26 @@ def create(config: DatasetConfig):
         'test': Dataset(test_paths, config=config),
         'train_eval': Dataset(train_for_evaluate_paths, config=config),
     }
+
+
+def create_cg(config: DatasetConfig):
+    x_paths = [Path(p) for p in glob.glob(str(config.input_glob))]
+    y_paths = [Path(p) for p in glob.glob(str(config.target_glob))]
+
+    num_test = config.num_test
+    numpy.random.RandomState(config.seed).shuffle(x_paths)
+    numpy.random.RandomState(config.seed).shuffle(y_paths)
+
+    train_x_paths = x_paths[num_test:]
+    test_x_paths = x_paths[:num_test]
+    train_for_evaluate_x_paths = x_paths[:num_test]
+
+    train_y_paths = y_paths[num_test:]
+    test_y_paths = y_paths[:num_test]
+    train_for_evaluate_y_paths = y_paths[:num_test]
+
+    return {
+        'train': CGDataset(train_x_paths, train_y_paths, config=config),
+        'test': CGDataset(test_x_paths, test_y_paths, config=config),
+        'train_eval': CGDataset(train_for_evaluate_x_paths, train_for_evaluate_y_paths, config=config),
+    }

yukarin/model.py

@@ -154,13 +154,15 @@ def __init__(self, in_ch, out_ch, base=32, extensive_layers=5, is_weak=False) ->
             else:
                 self.c4 = Convolution1D(base * 16, 1, 1, 1, 0, initialW=w)
 
-    def __call__(self, x_0, x_1):
-        h = F.concat([self.c0_0(x_0), self.c0_1(x_1)])
+    def __call__(self, x_0, x_1=None):
+        if x_1 is not None:
+            h = F.concat([self.c0_0(x_0), self.c0_1(x_1)])
+        else:
+            h = F.concat([self.c0_0(x_0), self.c0_1(x_0)])
         h = self.c1(h)
         h = self.c2(h)
         h = self.c3(h)
         h = self.c4(h)
-        # h = F.average_pooling_2d(h, h.data.shape[2], 1, 0)
         return h