SA_audio_classification.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import tqdm as tqdm\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy.io import wavfile\n",
    "from python_speech_features import mfcc, logfbank\n",
    "import librosa"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keras.layers import Activation, Dense, Dropout, Conv2D, Flatten, MaxPooling2D, GlobalMaxPooling2D, GlobalAveragePooling1D, AveragePooling2D, Input, Add\n",
    "from keras.models import Sequential\n",
    "from keras.utils import to_categorical\n",
    "from sklearn.utils.class_weight import compute_class_weight"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_mfccs (mfccs):\n",
    "    fig,axes = plt.subplots (nrows=2, ncols=5, sharex=False, sharey=True, figsize=(20,5))\n",
    "    \n",
    "    fig.suptitle('Mel Frequency Cepstrum Coefficients', size=16) \n",
    "    i=0\n",
    "    \n",
    "    for x in range(1):\n",
    "        for y in range(2):\n",
    "            axes[x,y].set_title(list (mfccs.keys())[i]) \n",
    "            axes[x,y].imshow(list(mfccs.values()) [i],cmap='hot', interpolation='nearest') \n",
    "            axes[x,y].get_xaxis().set_visible(False)\n",
    "            axes [x,y].get_yaxis().set_visible(False) \n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_fbank (fbank):\n",
    "\n",
    "    fig, axes = plt.subplots (nrows=2, ncols=5, sharex=False,\n",
    "    \n",
    "    sharey=True, figsize=(20,5))\n",
    "    \n",
    "    fig.suptitle('Filter Bank Coefficients', size=16) \n",
    "    i = 0\n",
    "    \n",
    "    for x in range(1):\n",
    "    \n",
    "        for y in range(2):\n",
    "    \n",
    "            axes[x,y].set_title(list(fbank.keys())[i]) \n",
    "            axes[x,y].imshow(list(fbank.values()) [i], cmap='hot', interpolation='nearest') \n",
    "            axes[x,y].get_xaxis().set_visible(False)\n",
    "            axes[x,y].get_yaxis().set_visible(False)\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_signals (signals):\n",
    "\n",
    "    fig, axes = plt.subplots (nrows=2, ncols=5, sharex=False,sharey=True, figsize=(20,5)) \n",
    "    fig.suptitle('Time Series', size=16) \n",
    "    i = 0\n",
    "    \n",
    "    for x in range(1):\n",
    "        for y in range(2):\n",
    "            axes[x,y].set_title(list(signals.keys())[i]) \n",
    "            axes[x,y].plot(list(signals.values())[i]) \n",
    "            axes[x,y].get_xaxis().set_visible(False) \n",
    "            axes[x,y].get_yaxis().set_visible(False)\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_fft(fft):\n",
    "\n",
    "    fig, axes = plt.subplots (nrows=2, ncols=5, sharex=False, sharey=True, figsize=(20,5))\n",
    "    fig.suptitle('Fourier Transforms', size=16) \n",
    "    i = 0\n",
    "    for x in range(1):\n",
    "        for y in range (2):\n",
    "            data = list (fft.values())[i] \n",
    "            Y, freq = data[0], data[1]\n",
    "            axes[x,y].set_title(list(fft.keys()) [i]) \n",
    "            axes[x,y].plot(freq, Y)\n",
    "            axes[x,y].get_xaxis().set_visible(False) \n",
    "            axes[x,y].get_yaxis().set_visible(False)\n",
    "            i += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df=pd.read_csv('dataset_name.csv')\n",
    "df.set_index('filename',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for f in df.index:\n",
    "    rate,signal = wavfile.read('b_cry/all/'+f)\n",
    "    df.at[f,'length']= signal.shape[0]/rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "classes = list(np.unique(df.label))\n",
    "class_dist = df.groupby(['label'])['length'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fix, ax = plt.subplots()\n",
    "ax.set_title('cd',y=1.08)\n",
    "ax.pie(class_dist,labels=class_dist.index, autopct='%1.1f%%')\n",
    "ax.axis('equal')\n",
    "plt.show()\n",
    "df.reset_index(inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plot_signals(signals)\n",
    "plot_fft(fft)\n",
    "plot_fbank(fbank)\n",
    "plot_mfccs(mfccs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calc_fft(y, rate):\n",
    "    n=len(y)\n",
    "    freq=np.fft.rfftfreq(n, d=1/rate)\n",
    "    Y= abs(np.fft.rfft(y)/n)\n",
    "    return (Y, freq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def envelope(y, rate, threshold):\n",
    "    mask = []\n",
    "    y = pd.Series(y).apply(np.abs)\n",
    "    y_mean = y.rolling(window=int(rate/10),\n",
    "                       min_periods=1,\n",
    "                       center=True).mean()\n",
    "    for mean in y_mean:\n",
    "        if mean > threshold:\n",
    "            mask.append(True)\n",
    "        else:\n",
    "            mask.append(False)\n",
    "    return mask, y_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "signals={}\n",
    "fft={}\n",
    "fbank={}\n",
    "mfccs={}\n",
    "for c in classes:\n",
    "    wav_file= df[df.label == c].iloc[0,0]\n",
    "    signal,rate = librosa.load('b_cry/all/'+wav_file,sr=22050)\n",
    "    mask, y_mean = envelope(signal, rate, 0.0005)\n",
    "    signal = signal[mask]\n",
    "    signals[c]= signal\n",
    "    fft[c]= calc_fft(signal, rate)\n",
    "    \n",
    "    bank=logfbank(signal[:rate], rate, nfilt=26, nfft=552).T\n",
    "    fbank[c]=bank\n",
    "    mel= mfcc(signal[:rate],rate, numcep=13, nfilt=26, nfft=552).T\n",
    "    mfccs[c]=mel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# if len(os.listdir('clean'))==0:\n",
    "#     for f in df['filename']:\n",
    "#         signal,rate = librosa.load('b_cry/all/'+f,sr=16000)\n",
    "#         mask=envelope(signal, rate, 0.0005)\n",
    "#         wavfile.write(filename='clean/'+f,rate=rate,data=signal[mask])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_samples = 2*int(df['length'].sum()/0.1)\n",
    "prob_dist = class_dist/class_dist.sum()\n",
    "choices = np.random.choice(class_dist.index, p=prob_dist)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "prob_dist"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Conv1D(N_CLASSES=2, SR=16000, DT=1.0):\n",
    "    input_shape = (int(SR*DT), 1)\n",
    "    i = get_melspectrogram_layer(input_shape=input_shape,\n",
    "                                 n_mels=128,\n",
    "                                 pad_end=True,\n",
    "                                 n_fft=512,\n",
    "                                 win_length=400,\n",
    "                                 hop_length=160,\n",
    "                                 sample_rate=SR,\n",
    "                                 return_decibel=True,\n",
    "                                 input_data_format='channels_last',\n",
    "                                 output_data_format='channels_last')\n",
    "    x = LayerNormalization(axis=2, name='batch_norm')(i.output)\n",
    "    x = TimeDistributed(layers.Conv1D(8, kernel_size=(4), activation='tanh'), name='td_conv_1d_tanh')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), name='max_pool_2d_1')(x)\n",
    "    x = TimeDistributed(layers.Conv1D(16, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_1')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), name='max_pool_2d_2')(x)\n",
    "    x = TimeDistributed(layers.Conv1D(32, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_2')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), name='max_pool_2d_3')(x)\n",
    "    x = TimeDistributed(layers.Conv1D(64, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_3')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), name='max_pool_2d_4')(x)\n",
    "    x = TimeDistributed(layers.Conv1D(128, kernel_size=(4), activation='relu'), name='td_conv_1d_relu_4')(x)\n",
    "    x = layers.GlobalMaxPooling2D(name='global_max_pooling_2d')(x)\n",
    "    x = layers.Dropout(rate=0.1, name='dropout')(x)\n",
    "    x = layers.Dense(64, activation='relu', activity_regularizer=l2(0.001), name='dense')(x)\n",
    "    o = layers.Dense(N_CLASSES, activation='softmax', name='softmax')(x)\n",
    "    model = Model(inputs=i.input, outputs=o, name='1d_convolution')\n",
    "    model.compile(optimizer='adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Conv2D(N_CLASSES=2, SR=16000, DT=1.0):\n",
    "    input_shape = (int(SR*DT), 1)\n",
    "    i = get_melspectrogram_layer(input_shape=input_shape,\n",
    "                                 n_mels=128,\n",
    "                                 pad_end=True,\n",
    "                                 n_fft=512,\n",
    "                                 win_length=400,\n",
    "                                 hop_length=160,\n",
    "                                 sample_rate=SR,\n",
    "                                 return_decibel=True,\n",
    "                                 input_data_format='channels_last',\n",
    "                                 output_data_format='channels_last')\n",
    "    x = LayerNormalization(axis=2, name='batch_norm')(i.output)\n",
    "    x = layers.Conv2D(8, kernel_size=(7,7), activation='tanh', padding='same', name='conv2d_tanh')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), padding='same', name='max_pool_2d_1')(x)\n",
    "    x = layers.Conv2D(16, kernel_size=(5,5), activation='relu', padding='same', name='conv2d_relu_1')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), padding='same', name='max_pool_2d_2')(x)\n",
    "    x = layers.Conv2D(16, kernel_size=(3,3), activation='relu', padding='same', name='conv2d_relu_2')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), padding='same', name='max_pool_2d_3')(x)\n",
    "    x = layers.Conv2D(32, kernel_size=(3,3), activation='relu', padding='same', name='conv2d_relu_3')(x)\n",
    "    x = layers.MaxPooling2D(pool_size=(2,2), padding='same', name='max_pool_2d_4')(x)\n",
    "    x = layers.Conv2D(32, kernel_size=(3,3), activation='relu', padding='same', name='conv2d_relu_4')(x)\n",
    "    x = layers.Flatten(name='flatten')(x)\n",
    "    x = layers.Dropout(rate=0.2, name='dropout')(x)\n",
    "    x = layers.Dense(64, activation='relu', activity_regularizer=l2(0.001), name='dense')(x)\n",
    "    o = layers.Dense(N_CLASSES, activation='softmax', name='softmax')(x)\n",
    "    model = Model(inputs=i.input, outputs=o, name='2d_convolution')\n",
    "    model.compile(optimizer='adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def LSTM(N_CLASSES=2, SR=16000, DT=1.0):\n",
    "    input_shape = (int(SR*DT), 1)\n",
    "    i = get_melspectrogram_layer(input_shape=input_shape,\n",
    "                                     n_mels=128,\n",
    "                                     pad_end=True,\n",
    "                                     n_fft=512,\n",
    "                                     win_length=400,\n",
    "                                     hop_length=160,\n",
    "                                     sample_rate=SR,\n",
    "                                     return_decibel=True,\n",
    "                                     input_data_format='channels_last',\n",
    "                                     output_data_format='channels_last',\n",
    "                                     name='2d_convolution')\n",
    "    x = LayerNormalization(axis=2, name='batch_norm')(i.output)\n",
    "    x = TimeDistributed(layers.Reshape((-1,)), name='reshape')(x)\n",
    "    s = TimeDistributed(layers.Dense(64, activation='tanh'),\n",
    "                        name='td_dense_tanh')(x)\n",
    "    x = layers.Bidirectional(layers.LSTM(32, return_sequences=True),\n",
    "                             name='bidirectional_lstm')(s)\n",
    "    x = layers.concatenate([s, x], axis=2, name='skip_connection')\n",
    "    x = layers.Dense(64, activation='relu', name='dense_1_relu')(x)\n",
    "    x = layers.MaxPooling1D(name='max_pool_1d')(x)\n",
    "    x = layers.Dense(32, activation='relu', name='dense_2_relu')(x)\n",
    "    x = layers.Flatten(name='flatten')(x)\n",
    "    x = layers.Dropout(rate=0.2, name='dropout')(x)\n",
    "    x = layers.Dense(32, activation='relu',\n",
    "                         activity_regularizer=l2(0.001),\n",
    "                         name='dense_3_relu')(x)\n",
    "    o = layers.Dense(N_CLASSES, activation='softmax', name='softmax')(x)\n",
    "    model = Model(inputs=i.input, outputs=o, name='long_short_term_memory')\n",
    "    model.compile(optimizer='adam',\n",
    "                  loss='categorical_crossentropy',\n",
    "                  metrics=['accuracy'])\n",
    "\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DataGenerator(tf.keras.utils.Sequence):\n",
    "    def __init__(self, wav_paths, labels, sr, dt, n_classes,\n",
    "                 batch_size=32, shuffle=True):\n",
    "        self.wav_paths = wav_paths\n",
    "        self.labels = labels\n",
    "        self.sr = sr\n",
    "        self.dt = dt\n",
    "        self.n_classes = n_classes\n",
    "        self.batch_size = batch_size\n",
    "        self.shuffle = True\n",
    "        self.on_epoch_end()\n",
    "\n",
    "\n",
    "    def __len__(self):\n",
    "        return int(np.floor(len(self.wav_paths) / self.batch_size))\n",
    "\n",
    "\n",
    "    def __getitem__(self, index):\n",
    "        indexes = self.indexes[index*self.batch_size:(index+1)*self.batch_size]\n",
    "\n",
    "        wav_paths = [self.wav_paths[k] for k in indexes]\n",
    "        labels = [self.labels[k] for k in indexes]\n",
    "\n",
    "        # generate a batch of time data\n",
    "        X = np.empty((self.batch_size, int(self.sr*self.dt), 1), dtype=np.float32)\n",
    "        Y = np.empty((self.batch_size, self.n_classes), dtype=np.float32)\n",
    "\n",
    "        for i, (path, label) in enumerate(zip(wav_paths, labels)):\n",
    "            rate, wav = wavfile.read(path)\n",
    "            X[i,] = wav.reshape(-1, 1)\n",
    "            Y[i,] = to_categorical(label, num_classes=self.n_classes)\n",
    "\n",
    "        return X, Y\n",
    "\n",
    "\n",
    "    def on_epoch_end(self):\n",
    "        self.indexes = np.arange(len(self.wav_paths))\n",
    "        if self.shuffle:\n",
    "            np.random.shuffle(self.indexes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train(args):\n",
    "    src_root = args.src_root\n",
    "    sr = args.sample_rate\n",
    "    dt = args.delta_time\n",
    "    batch_size = args.batch_size\n",
    "    model_type = args.model_type\n",
    "    params = {'N_CLASSES':len(os.listdir(args.src_root)),\n",
    "              'SR':sr,\n",
    "              'DT':dt}\n",
    "    models = {'conv1d':Conv1D(**params),\n",
    "              'conv2d':Conv2D(**params),\n",
    "              'lstm':  LSTM(**params)}\n",
    "    assert model_type in models.keys(), '{} not an available model'.format(model_type)\n",
    "    csv_path = os.path.join('logs', '{}_history.csv'.format(model_type))\n",
    "\n",
    "    wav_paths = glob('{}/**'.format(src_root), recursive=True)\n",
    "    wav_paths = [x.replace(os.sep, '/') for x in wav_paths if '.wav' in x]\n",
    "    classes = sorted(os.listdir(args.src_root))\n",
    "    le = LabelEncoder()\n",
    "    le.fit(classes)\n",
    "    labels = [os.path.split(x)[0].split('/')[-1] for x in wav_paths]\n",
    "    labels = le.transform(labels)\n",
    "    wav_train, wav_val, label_train, label_val = train_test_split(wav_paths,\n",
    "                                                                  labels,\n",
    "                                                                  test_size=0.1,\n",
    "                                                                  random_state=0)\n",
    "\n",
    "    assert len(label_train) >= args.batch_size, 'Number of train samples must be >= batch_size'\n",
    "    if len(set(label_train)) != params['N_CLASSES']:\n",
    "        warnings.warn('Found {}/{} classes in training data. Increase data size or change random_state.'.format(len(set(label_train)), params['N_CLASSES']))\n",
    "    if len(set(label_val)) != params['N_CLASSES']:\n",
    "        warnings.warn('Found {}/{} classes in validation data. Increase data size or change random_state.'.format(len(set(label_val)), params['N_CLASSES']))\n",
    "\n",
    "    tg = DataGenerator(wav_train, label_train, sr, dt,\n",
    "                       params['N_CLASSES'], batch_size=batch_size)\n",
    "    vg = DataGenerator(wav_val, label_val, sr, dt,\n",
    "                       params['N_CLASSES'], batch_size=batch_size)\n",
    "    model = models[model_type]\n",
    "    cp = ModelCheckpoint('models/{}.h5'.format(model_type), monitor='val_loss',\n",
    "                         save_best_only=True, save_weights_only=False,\n",
    "                         mode='auto', save_freq='epoch', verbose=1)\n",
    "    csv_logger = CSVLogger(csv_path, append=False)\n",
    "    model.fit(tg, validation_data=vg,\n",
    "              epochs=30, verbose=1,\n",
    "              callbacks=[csv_logger, cp])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_prediction(args):\n",
    "\n",
    "    model = load_model(args.model_fn,\n",
    "        custom_objects={'STFT':STFT,\n",
    "                        'Magnitude':Magnitude,\n",
    "                        'ApplyFilterbank':ApplyFilterbank,\n",
    "                        'MagnitudeToDecibel':MagnitudeToDecibel})\n",
    "    wav_paths = glob('{}/**'.format(args.src_dir), recursive=True)\n",
    "    wav_paths = sorted([x.replace(os.sep, '/') for x in wav_paths if '.wav' in x])\n",
    "    classes = sorted(os.listdir(args.src_dir))\n",
    "    labels = [os.path.split(x)[0].split('/')[-1] for x in wav_paths]\n",
    "    le = LabelEncoder()\n",
    "    y_true = le.fit_transform(labels)\n",
    "    results = []\n",
    "\n",
    "    for z, wav_fn in tqdm(enumerate(wav_paths), total=len(wav_paths)):\n",
    "        rate, wav = downsample_mono(wav_fn, args.sr)\n",
    "        mask, env = envelope(wav, rate, threshold=args.threshold)\n",
    "        clean_wav = wav[mask]\n",
    "        step = int(args.sr*args.dt)\n",
    "        batch = []\n",
    "\n",
    "        for i in range(0, clean_wav.shape[0], step):\n",
    "            sample = clean_wav[i:i+step]\n",
    "            sample = sample.reshape(-1, 1)\n",
    "            if sample.shape[0] < step:\n",
    "                tmp = np.zeros(shape=(step, 1), dtype=np.float32)\n",
    "                tmp[:sample.shape[0],:] = sample.flatten().reshape(-1, 1)\n",
    "                sample = tmp\n",
    "            batch.append(sample)\n",
    "        X_batch = np.array(batch, dtype=np.float32)\n",
    "        y_pred = model.predict(X_batch)\n",
    "        y_mean = np.mean(y_pred, axis=0)\n",
    "        y_pred = np.argmax(y_mean)\n",
    "        real_class = os.path.dirname(wav_fn).split('/')[-1]\n",
    "        print('Actual class: {}, Predicted class: {}'.format(real_class, classes[y_pred]))\n",
    "        results.append(y_mean)\n",
    "\n",
    "    np.save(os.path.join('logs', args.pred_fn), np.array(results))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}