diploma.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import librosa\n",
    "import librosa.display\n",
    "import mir_eval\n",
    "import pretty_midi\n",
    "pretty_midi.pretty_midi.MAX_TICK = 1e10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import scipy.io.wavfile"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "import time\n",
    "import math"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from midiutil import MIDIFile\n",
    "from midi2audio import FluidSynth"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "config = tf.compat.v1.ConfigProto()\n",
    "config.gpu_options.per_process_gpu_memory_fraction = 0.7\n",
    "config.gpu_options.allow_growth = True\n",
    "session = tf.compat.v1.Session(config=config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.config.list_physical_devices()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.test.is_built_with_cuda()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Analiza"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "note_times_train = np.array([0.0 for i in range(88)])\n",
    "note_times_test = np.array([0.0 for i in range(88)])\n",
    "note_appearences_train = np.array([0 for i in range(88)])\n",
    "note_appearences_test = np.array([0 for i in range(88)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "inputpath = \"C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS\"\n",
    "\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS\"+dirpath[len(inputpath):]\n",
    "        print(pwd_read)\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-4] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        \n",
    "        if \"ENSTDkAm\" in dirpath or \"ENSTDkCl\" in dirpath:\n",
    "            for name in names:\n",
    "                if \"_lower\" in name or \"_higher\" in name:\n",
    "                    continue\n",
    "                with open(os.path.join(pwd_read, name)+\".txt\") as f:\n",
    "                    next(f)\n",
    "                    try:\n",
    "                        for line in f:\n",
    "                            if line == '\\n':\n",
    "                                continue\n",
    "                            args = line.rstrip().split(\"\\t\")\n",
    "                            note = int(args[2])-21\n",
    "                            note_appearences_test[note] += 1\n",
    "                            note_times_test[note] += float(args[1])-float(args[0])\n",
    "                    except:\n",
    "                        print(os.path.join(pwd_read, name)+\".txt\")\n",
    "                        continue\n",
    "        else:\n",
    "            for name in names:\n",
    "                if \"_lower\" in name or \"_higher\" in name:\n",
    "                    continue\n",
    "                with open(os.path.join(pwd_read, name)+\".txt\") as f:\n",
    "                    next(f)\n",
    "                    try:\n",
    "                        for line in f:\n",
    "                            if line == '\\n':\n",
    "                                continue\n",
    "                            args = line.rstrip().split(\"\\t\")\n",
    "                            note = int(args[2])-21\n",
    "                            note_appearences_train[note] += 1\n",
    "                            note_times_train[note] += float(args[1])-float(args[0])\n",
    "                    except:\n",
    "                        print(os.path.join(pwd_read, name)+\".txt\")\n",
    "                        continue\n",
    "\n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar([i+21 for i in range(88)], note_appearences_train)\n",
    "plt.xlabel(\"MIDI številka\")\n",
    "plt.ylabel(\"Število pojavitev\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = sum([(i+21)*note_appearences_train[i] for i in range(88)])\n",
    "n = sum(note_appearences_train)\n",
    "avg = data/n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "librosa.note_to_hz(librosa.midi_to_note(avg))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sd = math.sqrt(sum([note_appearences_train[i]*(i+21-avg)**2 for i in range(88)])/n)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar([i for i in range(88)], note_appearences_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "notes_shifted_train = note_appearences_train + np.roll(note_appearences_train, 30) + np.roll(note_appearences_train, -30)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.bar([i+21 for i in range(88)], notes_shifted_train)\n",
    "plt.xlabel(\"MIDI številka\")\n",
    "plt.ylabel(\"Število pojavitev\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generiranje transponiranih skladb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputpath = \"C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS\"\n",
    "fs = FluidSynth()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS\"+dirpath[len(inputpath):]\n",
    "        pwd_write = \"MAPS\"+dirpath[len(inputpath):]\n",
    "        \n",
    "        print(pwd_read)\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-4] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        names = {n for n in names if \"_lower\" not in n or \"_higher\" not in n}\n",
    "        lst = list(names)\n",
    "        \n",
    "        # process those files and save them\n",
    "        for name in names:\n",
    "            print(name)\n",
    "            \n",
    "            midi_lower = pretty_midi.PrettyMIDI(os.path.join(pwd_read, name)+\".mid\")\n",
    "            for instrument in midi_lower.instruments:\n",
    "                for note in instrument.notes:\n",
    "                    note.pitch = ((note.pitch-21)+58)%88+21\n",
    "            name_lower = os.path.join(pwd_read, name)+\"_lower\"\n",
    "            midi_lower.write(name_lower+\".mid\")\n",
    "            fs.midi_to_audio(name_lower+\".mid\", name_lower+\".wav\")\n",
    "\n",
    "            midi_higher = pretty_midi.PrettyMIDI(os.path.join(pwd_read, name)+\".mid\")\n",
    "            for instrument in midi_higher.instruments:\n",
    "                for note in instrument.notes:\n",
    "                    note.pitch = ((note.pitch-21)+30)%88+21\n",
    "            name_higher = os.path.join(pwd_read, name)+\"_higher\"\n",
    "            midi_higher.write(name_higher+\".mid\")\n",
    "            fs.midi_to_audio(name_higher+\".mid\", name_higher+\".wav\")\n",
    "            \n",
    "\n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rate, data = scipy.io.wavfile.read(\"wav_outputs/output_MUSlh.wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "d = data[0:rate*10,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scipy.io.wavfile.write(\"wav_outputs/transposed_10.wav\", rate, d)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Ime datoteke"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"MAPS/AkPnBcht/ISOL/TR1/MAPS_ISOL_TR1_F_S0_M53_AkPnBcht\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Branje .wav datotek"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rate, data = scipy.io.wavfile.read(file_name+\".wav\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "length = data.shape[0] / rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "length"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "t = np.linspace(0., length, data.shape[0])\n",
    "plt.plot(t, data[:, 0], label=\"Levi kanal\")\n",
    "plt.plot(t, data[:, 1], label=\"Desni kanal\")\n",
    "plt.legend()\n",
    "plt.xlabel(\"Čas [s]\")\n",
    "plt.ylabel(\"Amplituda\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "s, e = 88200, 88641 #90405\n",
    "t = np.linspace(s/rate, e/rate, e-s)\n",
    "plt.plot(t, data[s:e, 0], label=\"Levi kanal\")\n",
    "plt.plot(t, data[s:e, 1], label=\"Desni kanal\")\n",
    "plt.legend()\n",
    "plt.xlabel(\"Čas [s]\")\n",
    "plt.ylabel(\"Amplituda\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# To Mono"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_f = data.astype(float)\n",
    "data_mono = librosa.to_mono(data_f.T)\n",
    "#data_mono = librosa.to_mono(data_resampled)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mono.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mono.shape[0] / rate"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Fourier transform"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stft_data_1 = np.abs(librosa.stft(data_mono,\n",
    "                        n_fft = 2048,\n",
    "                        hop_length = 1024,\n",
    "                        pad_mode=\"wrap\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stft_data_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = stft_data_1[:256, :]\n",
    "S_db = librosa.amplitude_to_db(tmp, ref=np.max)\n",
    "librosa.display.specshow(S_db, x_axis='time', y_axis='linear', sr=rate*(256/1025), hop_length=1024/4)\n",
    "plt.colorbar(format=\"%+2.f dB\")\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"Frekvenca [Hz]\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Constant Q Transform"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "n_bins = 7 * bins_per_octave -> 8 ce zelis tudi visoke frekvence\n",
    "\n",
    "cq_data.shape * hop_length = data.shape\n",
    "\n",
    "hop_length / rate -> koliko [s] zavzame en hop/frame v cq_data\n",
    "128 -> 2.902 ms\n",
    "256 -> 5.805 ms\n",
    "384 -> 8.707 ms\n",
    "512 -> 11.610 ms\n",
    "1024 -> 23.219 ms\n",
    "(brez downsamplinga)\n",
    "\n",
    "length [s] = data.shape [frames] / rate [frame/s]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOP_LENGTH = 1024 #1024 # 512\n",
    "BINS_PER_OCTAVE = 12 * 2 # 60\n",
    "N_BINS = BINS_PER_OCTAVE * 8 # 60 * 8\n",
    "\n",
    "FRAMES_PER_BIN = HOP_LENGTH / rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_data_1 = np.abs(librosa.cqt(data_mono,\n",
    "                        sr = rate,\n",
    "                        hop_length = HOP_LENGTH, \n",
    "                        fmin = librosa.note_to_hz(librosa.midi_to_note(21)),\n",
    "                        bins_per_octave = BINS_PER_OCTAVE,\n",
    "                        n_bins = N_BINS,\n",
    "                        pad_mode = \"wrap\"))\n",
    "# https://dsp.stackexchange.com/questions/71874/understanding-cqt-constant-q-transformation-parameters-for-piano-amr-automati"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_data_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "S_db = librosa.amplitude_to_db(cq_data_1, ref=np.max)\n",
    "librosa.display.specshow(S_db, x_axis='time', y_axis='cqt_hz', sr=rate, fmin=librosa.note_to_hz(librosa.midi_to_note(21)),\n",
    "                         hop_length=HOP_LENGTH, bins_per_octave=BINS_PER_OCTAVE)\n",
    "plt.colorbar(format=\"%+2.0f dB\")\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"Frekvenca [Hz]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# highest piano note in hz\n",
    "librosa.note_to_hz(librosa.midi_to_note(54))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_data_1.max()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_data_1 = np.float64(cq_data_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_data_1.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def minMaxNorm(d):\n",
    "    M = d.max()\n",
    "    m = d.min()\n",
    "    return (d-m)/(M-m)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize - log"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_log = np.log(cq_data_1+1e-10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(cq_log, aspect='auto', origin='lower')\n",
    "plt.xlabel(\"Time\")\n",
    "plt.ylabel(\"Pitch\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "S_db = librosa.amplitude_to_db(cq_log, ref=np.max)\n",
    "librosa.display.specshow(S_db, x_axis='time', y_axis='cqt_hz', sr=rate/2, bins_per_octave=BINS_PER_OCTAVE)\n",
    "plt.colorbar(format=\"%+2.f dB\")\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"Frekvenca [Hz]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_log_Mm = minMaxNorm(cq_log)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(cq_log_Mm, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_log_Mm.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_log_Mm.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Normalize - z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "avg = cq_data_1.mean(axis=1)\n",
    "sd = cq_data_1.std(axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_z = ((cq_data_1.T-avg)/sd).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(cq_z, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_z_Mm = minMaxNorm(cq_z)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(cq_z_Mm, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# txt to \"spectrogram\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cq_data_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "of_data_1 = np.zeros((96, cq_data_1.shape[1])).astype(np.bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FRAMES_PER_BIN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"MAPS/AkPnBcht/ISOL/TR1/MAPS_ISOL_TR1_F_S0_M53_AkPnBcht.txt\") as f:\n",
    "    next(f) # skip header\n",
    "    \n",
    "    for line in f:\n",
    "        if line == '\\n':\n",
    "            continue\n",
    "        args = line.rstrip().split(\"\\t\")\n",
    "        # print(args)\n",
    "        onset_bin = int(float(args[0])//FRAMES_PER_BIN)\n",
    "        ofset_bin = int(float(args[1])//FRAMES_PER_BIN)\n",
    "        note = int(args[2])-21\n",
    "        of_data_1[note, onset_bin:ofset_bin+1] = True\n",
    "        # print(note, onset_bin, ofset_bin+1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "of_data_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "plt.imshow(of_data_1, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "plt.matshow(of_data_1, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MIDI to \"spectrogram\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "of_data_1_midi = np.zeros((88, cq_data_1.shape[1])).astype(np.bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "midi_data = pretty_midi.PrettyMIDI(file_name+\".mid\")\n",
    "for instrument in midi_data.instruments: # imamo samo 1 instrument\n",
    "    for note in instrument.notes:\n",
    "        print(note)\n",
    "        onset_bin = int(float(note.start)//FRAMES_PER_BIN)\n",
    "        ofset_bin = int(float(note.end)//FRAMES_PER_BIN)\n",
    "        note = int(note.pitch)-21\n",
    "        of_data_1_midi[note, onset_bin:ofset_bin+1] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "of_data_1_midi.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(of_data_1_midi, aspect=0.08, origin='lower', extent=[0, length, 21, 108])\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"MIDI številka\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "of_data_1_midi = np.zeros((96, cq_data_1.shape[1])).astype(np.bool)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "midi_data = pretty_midi.PrettyMIDI(\"MAPS/AkPnBcht/MUS/MAPS_MUS-alb_se3_AkPnBcht.mid\")\n",
    "for instrument in midi_data.instruments: # imamo samo 1 instrument\n",
    "    for note in instrument.notes:\n",
    "        onset_bin = int(float(note.start)//FRAMES_PER_BIN)\n",
    "        ofset_bin = int(float(note.end)//FRAMES_PER_BIN)\n",
    "        note = int(note.pitch)-21\n",
    "        of_data_1_midi[note, onset_bin:ofset_bin+1] = True"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(of_data_1_midi, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save CQT and \"spectrogram\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"MAPS_processed/AkPnBcht/ISOL/CH/MAPS_ISOL_CH0.1_F_AkPnBcht_wav\", cq_data_1) # save CQT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.save(\"MAPS_processed/AkPnBcht/ISOL/CH/MAPS_ISOL_CH0.1_F_AkPnBcht_mid\", of_data_1) # save spectrogram"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tmp = scipy.sparse.csc_matrix(of_data_1)\n",
    "scipy.sparse.save_npz(\"MAPS_processed/AkPnBcht/ISOL/CH/MAPS_ISOL_CH0.1_F_AkPnBcht_mid\", tmp) # save spectrogram as sparse"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# creates same folder structure as MAPS\n",
    "\"\"\"\n",
    "inputpath = 'C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS'\n",
    "outputpath = 'C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS_z'\n",
    "\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    structure = outputpath+dirpath[len(inputpath):]\n",
    "    #print(structure)\n",
    "    if not os.path.isdir(structure):\n",
    "        os.mkdir(structure)\n",
    "    else:\n",
    "        print(\"Folder does already exits!\")\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Process all files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputpath = \"C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def minMaxNorm(d):\n",
    "    M = d.max()\n",
    "    m = d.min()\n",
    "    return (d-m)/(M-m)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOP_LENGTH = 1024 # 128*n\n",
    "BINS_PER_OCTAVE = 24 # 12*m\n",
    "N_BINS = BINS_PER_OCTAVE*8\n",
    "FMIN = librosa.note_to_hz(librosa.midi_to_note(21))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS\"+dirpath[len(inputpath):]\n",
    "        pwd_write = \"MAPS_processed\"+dirpath[len(inputpath):]\n",
    "        \n",
    "        # check if folder has been processed\n",
    "        #if len(os.listdir(pwd_write)) == (len(filenames)//3)*2:\n",
    "        #    continue\n",
    "        print(pwd_read)\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-4] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        \n",
    "        # names = {n for n in names if \"_lower\" not in n or \"_higher\" not in n} # _lower in _higher nimajo .txt\n",
    "        \n",
    "        lst = list(names)\n",
    "        \n",
    "        # process those files and save them\n",
    "        for name in names:\n",
    "            #if os.path.exists(os.path.join(pwd_write, name)+\"_wav.npy\"): # file already exists\n",
    "            #    continue\n",
    "\n",
    "            print(os.path.join(pwd_read, name)+\" --> \"+os.path.join(pwd_write, name))\n",
    "            \n",
    "            # process .wav file\n",
    "            rate, data = scipy.io.wavfile.read(os.path.join(pwd_read, name)+\".wav\")\n",
    "            data = np.float32(data)\n",
    "            \n",
    "            #data = librosa.resample(data.T, rate, rate/2, res_type = \"kaiser_fast\")\n",
    "            #rate = rate/2\n",
    "\n",
    "            data_mono = librosa.to_mono(data.T)\n",
    "            cq_data = np.abs(librosa.cqt(data_mono, sr = rate, hop_length = HOP_LENGTH,\n",
    "                                           fmin = FMIN, bins_per_octave = BINS_PER_OCTAVE,\n",
    "                                           n_bins = N_BINS, pad_mode = \"wrap\"))\n",
    "            \n",
    "            \"\"\"avg = cq_data.mean(axis=1)\n",
    "            sd = cq_data.std(axis=1)\n",
    "            cq_data = ((cq_data.T-avg)/sd).T\"\"\"\n",
    "            \n",
    "            #cq_data = np.log(cq_data+1e-10)\n",
    "            \n",
    "            #cq_data = minMaxNorm(cq_data)\n",
    "            cq_data = np.float32(cq_data)\n",
    "            \n",
    "            frames_per_bin = HOP_LENGTH/rate\n",
    "            # process .txt file\n",
    "            \"\"\"of_data = np.zeros((96, cq_data.shape[1])).astype(np.bool)\n",
    "            with open(os.path.join(pwd_read, name)+\".txt\") as f:\n",
    "                next(f)\n",
    "                try:\n",
    "                    for line in f:\n",
    "                        if line == '\\n': # ignore last line\n",
    "                            continue\n",
    "                        args = line.rstrip().split(\"\\t\")\n",
    "                        # print(args)\n",
    "                        onset_bin = int(float(args[0])//frames_per_bin)\n",
    "                        ofset_bin = int(float(args[1])//frames_per_bin)\n",
    "                        note = int(args[2])-21\n",
    "                        of_data[note, onset_bin:ofset_bin+1] = True\n",
    "                except:\n",
    "                    print(os.path.join(pwd_read, name)+\".txt\")\n",
    "                    continue\"\"\"\n",
    "            \n",
    "            # process .mid file\n",
    "            of_data = np.zeros((96, cq_data.shape[1])).astype(np.bool)\n",
    "            midi_data = pretty_midi.PrettyMIDI(os.path.join(pwd_read, name)+\".mid\")\n",
    "            for instrument in midi_data.instruments:\n",
    "                for note in instrument.notes:\n",
    "                    onset_bin = int(float(note.start)//frames_per_bin)\n",
    "                    ofset_bin = int(float(note.end)//frames_per_bin)\n",
    "                    note = int(note.pitch)-21\n",
    "                    of_data[note, onset_bin:ofset_bin+1] = True\n",
    "\n",
    "            # save CQT and spectrogram\n",
    "            np.save(os.path.join(pwd_write, name)+\"_wav\", cq_data) # save CQT\n",
    "            tmp = scipy.sparse.csc_matrix(of_data)\n",
    "            scipy.sparse.save_npz(os.path.join(pwd_write, name)+\"_mid\", tmp) # save as sparse\n",
    "\n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "downsample in HOP_SIZE(256) data ===== samo HOP_SIZE(512) {hitreje}\n",
    "\n",
    "HOP_LENGTH = 256, BINS_PER_OCTAVE = 24\n",
    "1.23GB -> 476MB\n",
    "144[s]\n",
    "BINS_PER_OCTAVE nima vpliva na cas, poveca pa velikost linearno"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "n_files = 0\n",
    "sum_start = 0\n",
    "avg_start = 0\n",
    "min_start = 1\n",
    "max_start = 0\n",
    "\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2:\n",
    "        names = {file[:-4] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        lst = list(names)\n",
    "        \n",
    "        pwd_read = \"MAPS\"+dirpath[len(inputpath):]\n",
    "        for file in lst:\n",
    "            with open(os.path.join(pwd_read, file)+\".txt\") as f:\n",
    "                next(f)\n",
    "                for line in f:\n",
    "                    try:\n",
    "                        n_files += 1\n",
    "                        start = float(line.rstrip().split(\"\\t\")[0])\n",
    "                        sum_start += start\n",
    "                        if start > max_start:\n",
    "                            max_start = start\n",
    "                        if start < min_start:\n",
    "                            min_start = start\n",
    "                    except:\n",
    "                        print(os.path.join(pwd_read, file)+\".txt\")\n",
    "                    break\n",
    "avg_start = sum_start/n_files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(n_files, min_start, avg_start, max_start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Branje podatkov nazaj"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"MAPS_log/AkPnBcht/ISOL/TR1/MAPS_ISOL_TR1_F_S0_M53_AkPnBcht\"\n",
    "data_wav = np.load(file_name+\"_wav.npy\")\n",
    "data_mid = scipy.sparse.load_npz(file_name+\"_mid.npz\").toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "S_db = librosa.amplitude_to_db(data_wav, ref=np.max)\n",
    "librosa.display.specshow(S_db, x_axis='time', y_axis='cqt_hz', sr=rate, fmin=librosa.note_to_hz(librosa.midi_to_note(21)),\n",
    "                         hop_length=HOP_LENGTH, bins_per_octave=BINS_PER_OCTAVE)\n",
    "plt.colorbar(format=\"%+2.f dB\")\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"Frekvenca [Hz]\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(data_mid, aspect='auto', origin='lower')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_wav.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_wav.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_wav.sum(axis=1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.mean(data_wav, axis=1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# convert mean vector into matrix\n",
    "np.array([np.mean(data_wav, axis=1),]*data_wav.shape[1]).T.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "np.zeros((192,)).shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MAPS_z"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputpath = \"C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS_processed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOP_LENGTH = 1024 # 128*n\n",
    "BINS_PER_OCTAVE = 24 # 12*m\n",
    "N_BINS = BINS_PER_OCTAVE*8\n",
    "FMIN = librosa.note_to_hz(librosa.midi_to_note(21))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "note_average = np.zeros((192,))\n",
    "note_N = 0\n",
    "\n",
    "# get mean note values\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS_processed\"+dirpath[len(inputpath):]\n",
    "        \n",
    "        print(pwd_read)\n",
    "        \n",
    "        if \"ENSTDkAm\" in dirpath or \"ENSTDkCl\" in dirpath:\n",
    "            continue\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-8] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        \n",
    "        lst = list(names)\n",
    "        \n",
    "        # process those files and save them\n",
    "        for name in names:\n",
    "            #print(os.path.join(pwd_read, name))\n",
    "            \n",
    "            data_wav = np.load(os.path.join(pwd_read, name)+\"_wav.npy\")\n",
    "            \n",
    "            note_average += data_wav.sum(axis=1)\n",
    "            note_N += data_wav.shape[1]\n",
    "            \n",
    "note_average /= note_N\n",
    "            \n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "note_average"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "note_std = np.zeros((192,))\n",
    "\n",
    "# get standard deveiation of notes\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS_processed\"+dirpath[len(inputpath):]\n",
    "        \n",
    "        print(pwd_read)\n",
    "        \n",
    "        if \"ENSTDkAm\" in dirpath or \"ENSTDkCl\" in dirpath:\n",
    "            continue\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-8] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        \n",
    "        lst = list(names)\n",
    "        \n",
    "        # process those files and save them\n",
    "        for name in names:\n",
    "            #print(os.path.join(pwd_read, name))\n",
    "            \n",
    "            data_wav = np.load(os.path.join(pwd_read, name)+\"_wav.npy\")\n",
    "            \n",
    "            tmp = np.array([note_average,]*data_wav.shape[1]).T\n",
    "            note_std += np.power(data_wav-tmp, 2).sum(axis=1)\n",
    "\n",
    "note_std /= note_N\n",
    "note_std = np.sqrt(note_std)\n",
    "\n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "note_std"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "# get standard deveiation of notes\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS_processed\"+dirpath[len(inputpath):]\n",
    "        pwd_write = \"MAPS_z\"+dirpath[len(inputpath):]\n",
    "        \n",
    "        print(pwd_read)\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-8] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        \n",
    "        lst = list(names)\n",
    "        \n",
    "        # process those files and save them\n",
    "        for name in names:\n",
    "            #print(os.path.join(pwd_read, name)+\" --> \"+os.path.join(pwd_write, name))\n",
    "            \n",
    "            data_wav = np.load(os.path.join(pwd_read, name)+\"_wav.npy\")\n",
    "            avg = np.array([note_average,]*data_wav.shape[1]).T\n",
    "            std = np.array([note_std,]*data_wav.shape[1]).T\n",
    "            out = (data_wav-avg)/std\n",
    "            np.save(os.path.join(pwd_write, name)+\"_wav\", np.float32(out))\n",
    "            \n",
    "            data_mid = scipy.sparse.load_npz(os.path.join(pwd_read, name)+\"_mid.npz\")\n",
    "            scipy.sparse.save_npz(os.path.join(pwd_write, name)+\"_mid\", data_mid)\n",
    "\n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train / Test oblika"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_process(data_x, data_y, window):\n",
    "    shape_x = data_x.shape\n",
    "    shape_y = data_y.shape\n",
    "    size = shape_x[1]-window+1\n",
    "    pad = int((window-1)/2)\n",
    "    output_x = np.zeros((size, 384, window))\n",
    "    output_y = np.zeros((size, 88))\n",
    "    \n",
    "    for i in range(size):\n",
    "        output_x[i, :, :] = data_x[:, i:i+window]\n",
    "        output_y[i, :] = data_y[:88, i+pad]*1\n",
    "    \n",
    "    output_x = np.float32(output_x)\n",
    "    output_y = np.int8(output_y)\n",
    "    return output_x, output_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x, train_y = data_process(data_wav, data_mid, 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_x.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_y.dtype"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "inputpath = \"C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS_processed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) > 2 and \"MUS\" in dirpath: # .mid, .txt, .wav\n",
    "        \n",
    "        # get working direcotry\n",
    "        pwd_read = \"MAPS_processed\"+dirpath[len(inputpath):]\n",
    "        pwd_write = \"MAPS_traintest\"+dirpath[len(inputpath):]\n",
    "        \n",
    "        # check if folder has been processed\n",
    "        # if len(os.listdir(pwd_write)) == (len(filenames)//3)*2:\n",
    "        #   continue\n",
    "        print(pwd_read)\n",
    "        \n",
    "        # get unique file names\n",
    "        names = {file[:-4] for file in filenames} # vcasih 4, vcasih 8...???\n",
    "        names.discard(\"desktop\")\n",
    "        lst = list(names)\n",
    "        \n",
    "        # process those files and save them\n",
    "        for name in names:\n",
    "            data_wav = np.load(os.path.join(pwd_read, name)+\"_wav.npy\")\n",
    "            data_mid = scipy.sparse.load_npz(os.path.join(pwd_read, name)+\"_mid.npz\").toarray()\n",
    "            \n",
    "            train_x, train_y = data_process(data_wav, data_mid, 7)\n",
    "\n",
    "            np.save(os.path.join(pwd_write, name)+\"_wav\", train_x)\n",
    "            tmp = scipy.sparse.csr_matrix(train_y)\n",
    "            scipy.sparse.save_npz(os.path.join(pwd_write, name)+\"_mid\", tmp)\n",
    "\n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Priprava na fit_generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_train = dict()\n",
    "data_test = dict()\n",
    "window = 7\n",
    "inputpath = \"C:/Users/dmoho/Documents/FRI/DIPLOMA/MAPS_processed\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "start = time.time()\n",
    "for dirpath, dirnames, filenames in os.walk(inputpath):\n",
    "    if len(filenames) >= 2 and \"MUS\" in dirpath: # .npy, .npz\n",
    "        pwd_read = \"MAPS_processed\"+dirpath[len(inputpath):]\n",
    "        print(pwd_read)\n",
    "\n",
    "        names = {file[:-8] for file in filenames}\n",
    "        names.discard(\"desktop\")\n",
    "        lst = list(names)\n",
    "        \n",
    "        if \"ENSTDkAm\" in dirpath or \"ENSTDkCl\" in dirpath: # test data\n",
    "            for name in names:\n",
    "                if \"_higher\" in name or \"_lower\" in name: # skip those for testing\n",
    "                    continue\n",
    "                n = os.path.join(pwd_read, name)\n",
    "                data_wav = np.load(n+\"_wav.npy\")\n",
    "                n = n.replace(\"\\\\\", \"/\")\n",
    "                data_test[n] = data_wav.shape[1]-window+1\n",
    "        else: # train data\n",
    "            for name in names:\n",
    "                n = os.path.join(pwd_read, name)\n",
    "                data_wav = np.load(n+\"_wav.npy\")\n",
    "                n = n.replace(\"\\\\\", \"/\")\n",
    "                data_train[n] = data_wav.shape[1]-window+1\n",
    "        \n",
    "end = time.time()\n",
    "print(end-start)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(data_train.keys())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sum([x for x in data_train.values()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sys.getsizeof(data_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# save dict\n",
    "#pickle.dump(data_train, open(\"tt_files/train_p\"+str(window)+\".pickle\", \"wb\"))\n",
    "#pickle.dump(data_test, open(\"tt_files/test_p\"+str(window)+\".pickle\", \"wb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# read dict\n",
    "data_train = pickle.load(open(\"tt_files/MUSlh_train_log.pickle\", \"rb\"))\n",
    "data_test = pickle.load(open(\"tt_files/MUSlh_test_log.pickle\", \"rb\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepareForGenerator(data):\n",
    "    names = np.array([k for k in data.keys()])\n",
    "    values = np.cumsum([v for v in data.values()])\n",
    "    return names, values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "names_train, values_train = prepareForGenerator(data_train)\n",
    "names_test, values_test = prepareForGenerator(data_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "names_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "values_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# find the file based on index...\n",
    "names_train[np.argmin(values_train <= 38092)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "index = 0\n",
    "idx = np.argmin(values_train <= index*128)\n",
    "if idx == 0:\n",
    "    sample = index*128\n",
    "else:\n",
    "    sample = index*128-values_train[idx-1]\n",
    "    if sample+128+7 > values_train[idx]-values_train[idx-1]:\n",
    "        sample = values_train[idx]-values_train[idx-1]-(128+7)\n",
    "sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "values_train[idx]-values_train[idx-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = names_train[np.argmin(values_train <= index*128)]\n",
    "data_wav = np.load(filename+\"_wav.npy\")\n",
    "data_mid = scipy.sparse.load_npz(filename+\"_mid.npz\").toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "pad = int((7-1)/2)\n",
    "output_x = np.zeros((128, 384, 7, 1))\n",
    "output_y = np.zeros((128, 88))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(128):\n",
    "    output_x[i, :, :, :] = np.reshape(data_wav[:, sample+i:sample+i+7], (384, 7, 1))\n",
    "    output_y[i, :] = data_mid[:88, sample+i+pad]*1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Funkcije za Generator"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getRandomSample(data_x, data_y, window, size = 32):\n",
    "    pad = int((window-1)/2)\n",
    "    \n",
    "    output_x = np.zeros((size, 384, window, 1))\n",
    "    output_y = np.zeros((size, 88))\n",
    "    \n",
    "    sample = np.random.randint(pad, data_x.shape[1]-pad, size)\n",
    "    \n",
    "    i = 0\n",
    "    for j in sample:\n",
    "        output_x[i, :, :, :] = np.reshape(data_x[:, j:j+window], (384, window, 1))\n",
    "        output_y[i, :] = data_y[:88, j+pad]*1\n",
    "        i += 1\n",
    "    \n",
    "    output_x = np.float32(output_x)\n",
    "    output_y = np.int8(output_y)\n",
    "    return output_x, output_y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_name = \"MAPS_processed/ENSTDkAm/MUS/MAPS_MUS-schub_d760_3_ENSTDkAm\"\n",
    "data_wav = np.load(file_name+\"_wav.npy\")\n",
    "data_mid = scipy.sparse.load_npz(file_name+\"_mid.npz\").toarray()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = getRandomSample(data_wav, data_mid, 7, 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generator za keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class CustomGenerator(tf.keras.utils.Sequence):\n",
    "    def __init__(self, names, values, batch_size = 32, window = 7):\n",
    "        self.names = names\n",
    "        self.values = values\n",
    "        self.batch_size = batch_size\n",
    "        self.window = window\n",
    "        self.samples = self.values[-1]\n",
    "    \n",
    "    # number of batches per epoch\n",
    "    def __len__(self):\n",
    "        return self.samples // self.batch_size\n",
    "    \n",
    "    def getRandomSample(self, data_x, data_y):\n",
    "        pad = int((self.window-1)/2)\n",
    "\n",
    "        output_x = np.zeros((self.batch_size, 384, self.window, 1))\n",
    "        output_y = np.zeros((self.batch_size, 88))\n",
    "\n",
    "        sample = np.random.randint(0, data_x.shape[1]-self.window-1, self.batch_size)\n",
    "\n",
    "        i = 0\n",
    "        for j in sample:\n",
    "            output_x[i, :, :, :] = np.reshape(data_x[:, j:j+self.window], (384, self.window, 1))\n",
    "            output_y[i, :] = data_y[:88, j+pad]*1\n",
    "            i += 1\n",
    "\n",
    "        output_x = np.float32(output_x)\n",
    "        output_y = np.int8(output_y)\n",
    "        return output_x, output_y\n",
    "    \n",
    "    def __getitem__(self, index):\n",
    "        filename = self.names[np.argmin(self.values <= index*self.batch_size)]\n",
    "        \n",
    "        # load data\n",
    "        data_wav = np.load(filename+\"_wav.npy\")\n",
    "        data_mid = scipy.sparse.load_npz(filename+\"_mid.npz\").toarray()\n",
    "        \n",
    "        X, y = self.getRandomSample(data_wav, data_mid)\n",
    "        \n",
    "        return X, y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Binary Crossentropy loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "bce = tf.keras.losses.BinaryCrossentropy(from_logits=True, reduction=tf.keras.losses.Reduction.NONE)\n",
    "def custom_binaryCrossentropy(y_true, y_pred):\n",
    "    out = bce(y_true, y_pred).numpy()\n",
    "    return out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def custom_bce(y_true, y_pred):\n",
    "    tmp = y_pred-y_pred*y_true + tf.math.log(1+tf.math.exp((-1)*tf.math.abs(y_pred)))\n",
    "    weights = y_true*0.99+(1-y_true)*0.01\n",
    "    return tf.convert_to_tensor(tmp*weights)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_true = np.array([[1, 0], [1, 1], [0, 1]])\n",
    "y_pred = np.random.rand(3, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "custom_bce(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.Sequential()\n",
    "model.add(tf.keras.layers.Conv2D(32, (48, 3), activation=\"relu\", input_shape=(384, 7, 1), data_format=\"channels_last\", padding=\"valid\"))\n",
    "model.add(tf.keras.layers.MaxPooling2D((2, 1)))\n",
    "model.add(tf.keras.layers.Conv2D(32, (24, 3), activation=\"relu\", padding=\"valid\"))\n",
    "model.add(tf.keras.layers.MaxPooling2D((2, 1)))\n",
    "model.add(tf.keras.layers.Flatten())\n",
    "model.add(tf.keras.layers.Dense(88, activation=\"relu\"))\n",
    "model.add(tf.keras.layers.Dense(88, activation=\"sigmoid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.compile(optimizer=\"adam\", loss=tf.keras.losses.CategoricalCrossentropy(from_logits=True),\n",
    "              metrics=[tf.keras.metrics.Precision(name='precision'), tf.keras.metrics.Recall(name='recall')])\n",
    "#metrics=[\"precision\", \"recall\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#model.compile(optimizer=\"adam\", loss=loss_function, metrics=[\"crossentropy\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = \"cnnModels/conv32_48_3_mp2_1_conv64_24_3_mp2_1_flat_den132_den88\"\n",
    "checkpoint = tf.keras.callbacks.ModelCheckpoint(model_name, monitor=\"val_recall\", verbose=1,\n",
    "                                                save_best_only=True, mode=\"max\", save_weights_only=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "generator_train = CustomGenerator(names_train, values_train, 128, 7)\n",
    "generator_test = CustomGenerator(names_test, values_test, 128, 7)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "history = model.fit(generator_train, validation_data=generator_test, epochs=100, verbose=1, callbacks=[checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save(model_name)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# CNN load model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = tf.keras.models.load_model(\"hteModels/l_43_3\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def data_test(data_x, window):\n",
    "    shape_x = data_x.shape\n",
    "    size = shape_x[1]-window+1\n",
    "    pad = int((window-1)/2)\n",
    "    output_x = np.zeros((size, 192, window, 1))\n",
    "    \n",
    "    for i in range(size):\n",
    "        output_x[i, :, :] = np.reshape(data_x[:, i:i+window], (192, window, 1))\n",
    "    \n",
    "    output_x = np.float32(output_x)\n",
    "    return output_x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# MAPS_log/ENSTDkAm/ISOL/CH/MAPS_ISOL_CH0.3_F_ENSTDkAm\n",
    "# MAPS_log/ENSTDkCL/MUS/MAPS_MUS-bk_xmas4_ENSTDkCl\n",
    "# MAPS_log/AkPnBsdf/MUS/MAPS_MUS-alb_se4_AkPnBsdf\n",
    "#file_name = \"MAPS_log/ENSTDkAm/MUS/MAPS_MUS-schub_d760_3_ENSTDkAm\"\n",
    "file_name = \"MAPS_log/AkPnBcht/ISOL/TR1/MAPS_ISOL_TR1_F_S0_M53_AkPnBcht\"\n",
    "data_wav = np.load(file_name+\"_wav.npy\")\n",
    "data_mid = scipy.sparse.load_npz(file_name+\"_mid.npz\").toarray()[:88,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_wav.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mid.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x = data_test(data_wav, 5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_y = model.predict(test_x, verbose = 1).T"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(test_y, aspect='auto', origin='lower', extent=[0, 281*0.04307/2, 21, 108])\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"MIDI številka\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.savefig(\"img/output_example.png\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(test_y>0.5, aspect='auto', origin='lower', extent=[0, 281*0.04307/2, 21, 108])\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"MIDI številka\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.savefig(\"img/output_threshold_example.png\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.matshow(data_mid, aspect='auto', origin='lower', extent=[0, 281*0.04307/2, 21, 108])\n",
    "plt.xlabel(\"Čas [t]\")\n",
    "plt.ylabel(\"MIDI številka\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.savefig(\"img/output_true_example.png\", bbox_inches=\"tight\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(test_y.shape)\n",
    "print(test_y.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_mid = np.asarray(data_mid, dtype=np.byte)\n",
    "print(data_mid.shape)\n",
    "print(data_mid.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "THRESHOLD = 0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_y = np.zeros(data_mid.shape)\n",
    "data_y[:, 2:data_y.shape[1]-2] = test_y > THRESHOLD\n",
    "data_y = np.asarray(data_y, dtype=np.byte)\n",
    "print(data_y.shape)\n",
    "print(data_y.dtype)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "best_i = 0\n",
    "lowest = np.Inf\n",
    "arr = np.zeros(100)\n",
    "r = np.arange(0, 1.0, 0.01)\n",
    "for i in range(len(r)):\n",
    "    data_y[:, 2:data_y.shape[1]-2] = test_y >= r[i]\n",
    "    diff = np.sum(np.abs(data_mid-data_y))\n",
    "    arr[i] = diff\n",
    "    if diff < lowest:\n",
    "        lowest = diff\n",
    "        best_i = i\n",
    "print(r[best_i], lowest)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def evaluateResult(y_true, y_pred):\n",
    "    METRICS = [tf.keras.metrics.TruePositives(name='tp'),\n",
    "               tf.keras.metrics.FalsePositives(name='fp'),\n",
    "               tf.keras.metrics.TrueNegatives(name='tn'),\n",
    "               tf.keras.metrics.FalseNegatives(name='fn'),\n",
    "               tf.keras.metrics.BinaryAccuracy(name='accuracy'),\n",
    "               tf.keras.metrics.Precision(name='precision'),\n",
    "               tf.keras.metrics.Recall(name='recall')] #, tf.keras.metrics.AUC(name='auc')\n",
    "        \n",
    "    NAMES = [\"tp\", \"fp\", \"tn\", \"fn\", \"acc\", \"prec\", \"rec\"] #, \"auc\"\n",
    "    \n",
    "    for m, n in zip(METRICS, NAMES):\n",
    "        m.update_state(y_true, y_pred)\n",
    "        print(\"[%s]: %.3f\" % (n, m.result().numpy()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "evaluateResult(data_mid, data_y)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Output to events"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "HOP_LENGTH = 1024 # 512\n",
    "BINS_PER_OCTAVE = 12 * 2 # 60\n",
    "N_BINS = BINS_PER_OCTAVE * 8\n",
    "WINDOW = 5\n",
    "\n",
    "FRAMES_PER_BIN = HOP_LENGTH / 44100 #rate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def output_to_events(data_y):\n",
    "    tracing = np.array([False for i in range(88)])\n",
    "    onsets = np.array([0.0 for i in range(88)])\n",
    "    \n",
    "    intervals = list()\n",
    "    pitch = list()\n",
    "\n",
    "    for frame in range(data_y.shape[1]):\n",
    "        for note in range(88):\n",
    "            if data_y[note, frame]:\n",
    "                if not tracing[note]:\n",
    "                    tracing[note] = True\n",
    "                    onsets[note] = (frame+(WINDOW-1)/2)*FRAMES_PER_BIN\n",
    "            elif tracing[note]:\n",
    "                tracing[note] = False\n",
    "                intervals.append(np.array([onsets[note], (frame+(WINDOW-1)/2)*FRAMES_PER_BIN]))\n",
    "                pitch.append(note+21)\n",
    "                #print(onsets[note], frame*FRAMES_PER_BIN, note+21)\n",
    "    return np.array(intervals), np.array(pitch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def clean_events(intervals, pitch):\n",
    "    intervals_true = list()\n",
    "    pitch_true = list()\n",
    "    \n",
    "    for i in range(len(pitch)):\n",
    "        interval = intervals[i]\n",
    "        if interval[1]-interval[0] > 0.05:\n",
    "            intervals_true.append(interval)\n",
    "            pitch_true.append(pitch[i])\n",
    "    return np.array(intervals_true), np.array(pitch_true)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onsets, pitch = output_to_events(data_y)\n",
    "#onsets, pitch = clean_events(onsets, pitch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onsets.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"MAPS/ENSTDkAm/MUS/MAPS_MUS-schub_d760_3_ENSTDkAm.txt\") as f:\n",
    "    next(f) # skip header\n",
    "    \n",
    "    intervals = list()\n",
    "    notes = list()\n",
    "    \n",
    "    for line in f:\n",
    "        if line == '\\n':\n",
    "            continue\n",
    "        args = line.rstrip().split(\"\\t\")\n",
    "        intervals.append(np.array([float(args[0]), float(args[1])]))\n",
    "        notes.append(int(args[2]))\n",
    "    onsets_real = np.array(intervals)\n",
    "    pitch_real = np.array(notes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onsets_real.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scores = mir_eval.transcription.evaluate(onsets_real, pitch_real, onsets, pitch)\n",
    "for key, val in scores.items():\n",
    "    print(\"[%s]: %f\" % (key, val))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# From MIDI to WAV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tempo = 1/(min(onsets[:, 1]-onsets[:, 0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tempo = 14.33"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "track = 0\n",
    "channel = 0\n",
    "time = 0\n",
    "duration = 2\n",
    "volume = 90"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "midi = MIDIFile()\n",
    "midi.addTempo(track, time, tempo*50)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for e, p in zip(onsets, pitch):\n",
    "    l = tempo*(e[1]-e[0])\n",
    "    t = tempo*e[0]\n",
    "    midi.addNote(track, channel, p, t, l, volume)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"test.mid\", \"wb\") as output_file:\n",
    "    midi.writeFile(output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "FluidSynth().midi_to_audio(\"test.mid\", \"output_MUSlh.wav\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Replicate other models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ConvNet = tf.keras.models.Sequential()\n",
    "ConvNet.add(tf.keras.layers.Conv2D(50, (25, 5), activation=\"tanh\", input_shape=(252, 7, 1), data_format=\"channels_last\", padding=\"valid\"))\n",
    "ConvNet.add(tf.keras.layers.Dropout(0.5))\n",
    "ConvNet.add(tf.keras.layers.MaxPooling2D((3, 1)))\n",
    "ConvNet.add(tf.keras.layers.Conv2D(50, (5, 3), activation=\"tanh\", padding=\"valid\"))\n",
    "ConvNet.add(tf.keras.layers.Dropout(0.5))\n",
    "ConvNet.add(tf.keras.layers.MaxPooling2D((3, 1)))\n",
    "ConvNet.add(tf.keras.layers.Flatten())\n",
    "ConvNet.add(tf.keras.layers.Dense(1000, activation=\"sigmoid\"))\n",
    "ConvNet.add(tf.keras.layers.Dropout(0.5))\n",
    "ConvNet.add(tf.keras.layers.Dense(500, activation=\"sigmoid\"))\n",
    "ConvNet.add(tf.keras.layers.Dropout(0.5))\n",
    "ConvNet.add(tf.keras.layers.Dense(88, activation=\"sigmoid\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "ConvNet.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "stack = tf.keras.models.Sequential()\n",
    "stack.add(tf.keras.layers.Conv2D(32, (3, 3), input_shape=(229, 7, 1), data_format=\"channels_last\", padding=\"valid\"))\n",
    "stack.add(tf.keras.layers.Conv2D(32, (3, 3)))\n",
    "stack.add(tf.keras.layers.MaxPooling2D((2, 1)))\n",
    "stack.add(tf.keras.layers.Dropout(0.25))\n",
    "stack.add(tf.keras.layers.Conv2D(64, (3, 3)))\n",
    "stack.add(tf.keras.layers.MaxPooling2D((2, 1)))\n",
    "stack.add(tf.keras.layers.Dropout(0.25))\n",
    "stack.add(tf.keras.layers.Dense(512))\n",
    "stack.add(tf.keras.layers.Dropout(0.5))\n",
    "stack.add(tf.keras.layers.Dense(88))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "onset = tf.keras.models.Sequential()\n",
    "onset.add(tf.keras.layers.InputLayer(input_shape=(88,1)))\n",
    "onset.add(tf.keras.layers.LSTM(128, go_backwards=True))\n",
    "onset.add(tf.keras.layers.Dense(88))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames = tf.keras.models.Sequential()\n",
    "frames.add(tf.keras.layers.InputLayer(input_shape=(88,)))\n",
    "frames.add(tf.keras.layers.Dense(88))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames_ = tf.keras.models.Sequential()\n",
    "frames_.add(tf.keras.layers.InputLayer(input_shape=(176,1)))\n",
    "frames_.add(tf.keras.layers.LSTM(128, go_backwards=True))\n",
    "frames_.add(tf.keras.layers.Dense(88))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "frames_.summary()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}