diff --git a/requirements.txt b/requirements.txt index 43eab0d..e6d5a8d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,4 @@ farasapy tqdm numpy black +pandas \ No newline at end of file diff --git a/tasks/demo.ipynb b/tasks/demo.ipynb index b6ec14c..bc52387 100644 --- a/tasks/demo.ipynb +++ b/tasks/demo.ipynb @@ -9,11 +9,12 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 38, "metadata": {}, "outputs": [], "source": [ - "#!pip3 install tkseem" + "%%capture\n", + "!pip3 install -e ../" ] }, { @@ -41,7 +42,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 2, "metadata": {}, "outputs": [ { @@ -59,7 +60,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { @@ -71,7 +72,7 @@ } ], "source": [ - "print(tokenizer)" + "print(tokenizer.name)" ] }, { @@ -83,7 +84,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { @@ -92,7 +93,7 @@ "['السلام', 'عليكم']" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -101,6 +102,26 @@ "tokenizer.tokenize(\"السلام عليكم\")" ] }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['السلام']" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer._tokenize_word(\"السلام\")" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -117,7 +138,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "[557, 798]\n" + "[560, 801]\n" ] } ], @@ -169,6 +190,48 @@ "print(detokenized)" ] }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[3, 560, 4, 0, 0, 0, 0, 0, 0, 0], [3, 801, 4, 0, 0, 0, 0, 0, 0, 0]]" + ] + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encodings = tokenizer.encode_sentences([\"السلام\", \"عليكم\"], add_boundry=True, out_length=10)\n", + "encodings" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['السلام', 'عليكم']" + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "decodings = tokenizer.decode_sentences(encodings)\n", + "decodings" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -185,7 +248,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 11, "metadata": {}, "outputs": [ { @@ -210,7 +273,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -219,7 +282,7 @@ "['▁صباح', '▁الخير', '▁يا', '▁أص', 'د', 'قاء']" ] }, - "execution_count": 10, + "execution_count": 12, "metadata": {}, "output_type": "execute_result" } @@ -237,14 +300,14 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[1799, 2741]\n" + "[1801, 2743]\n" ] } ], @@ -262,7 +325,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -280,14 +343,14 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 15, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " السلام عليكم\n" + "السلام عليكم\n" ] } ], @@ -296,6 +359,48 @@ "print(detokenized)" ] }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[3, 1801, 2743, 4, 0, 0, 0, 0, 0, 0], [3, 3024, 325, 3042, 4, 0, 0, 0, 0, 0]]" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encodings = tokenizer.encode_sentences([\"السلام عليكم\", \"أهلا وسهلا\"], add_boundry=True, out_length=10)\n", + "encodings" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['السلام عليكم', 'أهلا وسهلا']" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "decodings = tokenizer.decode_sentences(encodings)\n", + "decodings" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -312,7 +417,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -337,7 +442,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 20, "metadata": {}, "outputs": [ { @@ -346,7 +451,7 @@ "['ال', '##سلام', 'علي', '##كم']" ] }, - "execution_count": 15, + "execution_count": 20, "metadata": {}, "output_type": "execute_result" } @@ -364,14 +469,14 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "[2, 367, 764, 184]\n" + "[5, 370, 767, 187]\n" ] } ], @@ -389,7 +494,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 22, "metadata": {}, "outputs": [ { @@ -405,6 +510,47 @@ "print(decoded)" ] }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[[3, 5, 370, 4, 0, 0, 0, 0, 0, 0], [3, 767, 187, 4, 0, 0, 0, 0, 0, 0]]" + ] + }, + "execution_count": 23, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "encodings = tokenizer.encode_sentences([\"السلام\", \"عليكم\"], add_boundry=True, out_length=10)\n", + "encodings" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['السلام', 'عليكم']" + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "tokenizer.decode_sentences(encodings)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -414,7 +560,7 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 25, "metadata": {}, "outputs": [ { @@ -426,22 +572,22 @@ } ], "source": [ - "tokenizer = tk.B\n", + "tokenizer = tk.RandomTokenizer()\n", "tokenizer.train('samples/data.txt')" ] }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['السل', '##ام', 'علي', '##كم', 'أي', '##ها', 'الأص', '##دقا', '##ء']" + "['السلا', '##م', 'علي', '##كم', 'أي', '##ها', 'الأص', '##دقا', '##ء']" ] }, - "execution_count": 19, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -459,7 +605,7 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -477,7 +623,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -501,7 +647,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -519,7 +665,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 30, "metadata": {}, "outputs": [ { @@ -528,7 +674,7 @@ "['ا', '##ل', '##س', '##ل', '##ا', '##م', 'ع', '##ل', '##ي', '##ك', '##م']" ] }, - "execution_count": 23, + "execution_count": 30, "metadata": {}, "output_type": "execute_result" } @@ -546,7 +692,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 31, "metadata": {}, "outputs": [ { @@ -564,16 +710,16 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 34, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "['ال', '##سل', '##ام', 'عل', '##يك', '##م']" + "['السلام', 'عليكم']" ] }, - "execution_count": 3, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } @@ -591,7 +737,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 35, "metadata": {}, "outputs": [], "source": [ @@ -600,7 +746,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 36, "metadata": {}, "outputs": [ { @@ -616,7 +762,7 @@ "0.15384615384615385" ] }, - "execution_count": 2, + "execution_count": 36, "metadata": {}, "output_type": "execute_result" } @@ -630,7 +776,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 39, "metadata": {}, "outputs": [ { @@ -647,7 +793,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVzUlEQVR4nO3dfbRddX3n8ffHYIRVVFRuOw4BQ2nQRmGw3MFRHMEKawU7QjtiV7LUiqPNdE3jw8g4g6MLKc6silbtjIZq7FCpDw1IO86lphOpiO3woLmUEAgIptFKsDNc8Km2Cga/88feVzaHm3vPJefeS3bfr7Wysvdv/+4+330ePud39j57n1QVkqQD3+OWugBJ0mgY6JLUEwa6JPWEgS5JPWGgS1JPHLRUN3z44YfXypUrl+rmJemAdOONN95bVWMzLVuyQF+5ciWTk5NLdfOSdEBK8jf7WuYuF0nqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6YqhAT7ImyR1JdiU5b4blRyX5QpKbkuxI8tLRlypJms2cgZ5kGbAROANYDaxLsnqg2zuAy6vqucBa4OJRFypJmt0wI/STgF1VtbuqHgA2A2cN9CngSe30k4Fvjq5ESdIwhjlT9Ajgrs78HuB5A30uAD6X5A3ATwGnjaQ6SYvmQ+deudQlzMuG971sqUt4zBnVQdF1wMeqagXwUuDjSR6x7iTrk0wmmZyamhrRTUuSYLhAvxs4sjO/om3reh1wOUBVXQ8cDBw+uKKq2lRV41U1PjY247VlJEmP0jCBvg1YleToJMtpDnpODPT5BvASgCQ/TxPoDsElaRHNGehVtRfYAGwFbqf5NsvOJBcmObPtdi7w60luBv4IOKf89WlJWlRDXT63qrYAWwbazu9M3wacPNrSJEnz4ZmiktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUE0MFepI1Se5IsivJeTMs/0CS7e2/O5N8Z/SlSpJmM+dP0CVZBmwETgf2ANuSTLQ/OwdAVf37Tv83AM9dgFolSbMYZoR+ErCrqnZX1QPAZuCsWfqvo/mhaEnSIhom0I8A7urM72nbHiHJM4Cjgav3sXx9kskkk1NTU/OtVZI0i1EfFF0LXFFVD860sKo2VdV4VY2PjY2N+KYl6R+3YQL9buDIzvyKtm0ma3F3iyQtiWECfRuwKsnRSZbThPbEYKckzwKeAlw/2hIlScOYM9Crai+wAdgK3A5cXlU7k1yY5MxO17XA5qqqhSlVkjSbOb+2CFBVW4AtA23nD8xfMLqyJEnz5ZmiktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPXEUF9blKQD3X991dlLXcK8vP0TV8z7bxyhS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUE0MFepI1Se5IsivJefvo86tJbkuyM8mnRlumJGkuc17LJckyYCNwOrAH2JZkoqpu6/RZBbwNOLmqvp3kpxeqYEnSzIYZoZ8E7Kqq3VX1ALAZOGugz68DG6vq2wBVdc9oy5QkzWWYQD8CuKszv6dt6zoWODbJtUluSLJmphUlWZ9kMsnk1NTUo6tYkjSjUR0UPQhYBZwKrAM+muSwwU5VtamqxqtqfGxsbEQ3LUmC4QL9buDIzvyKtq1rDzBRVT+qqq8Bd9IEvCRpkQwT6NuAVUmOTrIcWAtMDPT5DM3onCSH0+yC2T3COiVJc5gz0KtqL7AB2ArcDlxeVTuTXJjkzLbbVuC+JLcBXwDeWlX3LVTRkqRHGuon6KpqC7BloO38znQBb2n/SZKWgGeKSlJPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9cRQ30NfTCe+9Q+XuoR5ufG9v7bUJUgS4AhdknrDQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4w0CWpJwx0SeoJA12SemKoQE+yJskdSXYlOW+G5eckmUqyvf33+tGXKkmazZyn/idZBmwETgf2ANuSTFTVbQNdL6uqDQtQoyRpCMOM0E8CdlXV7qp6ANgMnLWwZUmS5muYQD8CuKszv6dtG/TyJDuSXJHkyJlWlGR9kskkk1NTU4+iXEnSvozqoOiVwMqqOh64Crh0pk5VtamqxqtqfGxsbEQ3LUmC4QL9bqA74l7Rtv1EVd1XVfe3s78PnDia8iRJwxom0LcBq5IcnWQ5sBaY6HZI8vTO7JnA7aMrUZI0jDm/5VJVe5NsALYCy4BLqmpnkguByaqaAN6Y5ExgL/At4JwFrFmSNIOhfrGoqrYAWwbazu9Mvw1422hLkyTNh2eKSlJPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPDHW1RUnwxRedstQlzNspf/HFpS5Bi8gRuiT1hIEuST1hoEtSTwwV6EnWJLkjya4k583S7+VJKsn46EqUJA1jzkBPsgzYCJwBrAbWJVk9Q78nAm8CvjTqIiVJcxtmhH4SsKuqdlfVA8Bm4KwZ+r0LuAj44QjrkyQNaZhAPwK4qzO/p237iSS/ABxZVZ+dbUVJ1ieZTDI5NTU172IlSfu23wdFkzwOeD9w7lx9q2pTVY1X1fjY2Nj+3rQkqWOYQL8bOLIzv6Jtm/ZE4DnANUm+DvwLYMIDo5K0uIYJ9G3AqiRHJ1kOrAUmphdW1Xer6vCqWllVK4EbgDOranJBKpYkzWjOQK+qvcAGYCtwO3B5Ve1McmGSMxe6QEnScIa6lktVbQG2DLSdv4++p+5/WZKk+fJMUUnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wp+g08ic/MGTl7qEebv2DdcudQnSyDhCl6SeMNAlqScMdEnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4YKtCTrElyR5JdSc6bYflvJLklyfYk/yfJ6tGXKkmazZyBnmQZsBE4A1gNrJshsD9VVcdV1QnAe4D3j7xSSdKshhmhnwTsqqrdVfUAsBk4q9uhqr7Xmf0poEZXoiRpGMNcnOsI4K7O/B7geYOdkvwm8BZgOfCLM60oyXpgPcBRRx0131olSbMY2UHRqtpYVccA/wl4xz76bKqq8aoaHxsbG9VNS5IYboR+N3BkZ35F27Yvm4Hf25+i+uobFx631CXM21Hn37LUJUga0jAj9G3AqiRHJ1kOrAUmuh2SrOrM/hLw1dGVKEkaxpwj9Kram2QDsBVYBlxSVTuTXAhMVtUEsCHJacCPgG8Dr1nIoiVJjzTULxZV1RZgy0Db+Z3pN424LknSPHmmqCT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9cRQgZ5kTZI7kuxKct4My9+S5LYkO5J8PskzRl+qJGk2cwZ6kmXARuAMYDWwLsnqgW43AeNVdTxwBfCeURcqSZrdMCP0k4BdVbW7qh4ANgNndTtU1Req6h/a2RuAFaMtU5I0l2EC/Qjgrs78nrZtX14H/NlMC5KsTzKZZHJqamr4KiVJcxrpQdEkrwLGgffOtLyqNlXVeFWNj42NjfKmJekfvYOG6HM3cGRnfkXb9jBJTgPeDpxSVfePpjxJ0rCGGaFvA1YlOTrJcmAtMNHtkOS5wEeAM6vqntGXKUmay5yBXlV7gQ3AVuB24PKq2pnkwiRntt3eCxwKfDrJ9iQT+1idJGmBDLPLharaAmwZaDu/M33aiOuSJM2TZ4pKUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPDBXoSdYkuSPJriTnzbD8RUn+KsneJGePvkxJ0lzmDPQky4CNwBnAamBdktUD3b4BnAN8atQFSpKGM8xvip4E7Kqq3QBJNgNnAbdNd6iqr7fLfrwANUqShjDMLpcjgLs683vatnlLsj7JZJLJqampR7MKSdI+LOpB0araVFXjVTU+Nja2mDctSb03TKDfDRzZmV/RtkmSHkOGCfRtwKokRydZDqwFJha2LEnSfM0Z6FW1F9gAbAVuBy6vqp1JLkxyJkCSf55kD/AK4CNJdi5k0ZKkRxrmWy5U1RZgy0Db+Z3pbTS7YiRJS8QzRSWpJwx0SeoJA12SesJAl6SeMNAlqScMdEnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4w0CWpJwx0SeoJA12SesJAl6SeMNAlqSeGCvQka5LckWRXkvNmWP6EJJe1y7+UZOWoC5UkzW7OQE+yDNgInAGsBtYlWT3Q7XXAt6vq54APABeNulBJ0uyGGaGfBOyqqt1V9QCwGThroM9ZwKXt9BXAS5JkdGVKkuaSqpq9Q3I2sKaqXt/Ovxp4XlVt6PS5te2zp53/67bPvQPrWg+sb2efCdwxqg0ZwuHAvXP2OnC5fQeuPm8buH2j9oyqGptpwUGLWARVtQnYtJi3OS3JZFWNL8VtLwa378DV520Dt28xDbPL5W7gyM78irZtxj5JDgKeDNw3igIlScMZJtC3AauSHJ1kObAWmBjoMwG8pp0+G7i65tqXI0kaqTl3uVTV3iQbgK3AMuCSqtqZ5EJgsqomgP8BfDzJLuBbNKH/WLMku3oWkdt34OrztoHbt2jmPCgqSToweKaoJPWEgS5JPXFABnqSDyR5c2d+a5Lf78y/L8lbHsV6T03yp6Oqc6EkeXuSnUl2JNme5HlJrmkvz3BzkmuTPHOJa3ywre3WJFcmOWxE6z0nyYdGsa791dnGne39fm6Sx7XLxpP891n+9p8muWKO9c+6jrbPYUn+3aPbgoet558k2Zzkr5PcmGRLkvWL/XpI8p8X8/Y6t1tJPtGZPyjJ1Ki2P8nK9nydBXVABjpwLfACgPYFdDjw7M7yFwDXzbWS9rIGB5Qkzwf+FfALVXU8cBpwV7v4lVX1z2jO2n3vEpU47QdVdUJVPYfmQPlvLnE9C2F6G58NnE5zeYx3AlTVZFW9cV9/WFXfrKqzZ1v5XOtoHQbsV6C3Z3X/T+Caqjqmqk4E3gb8zH6u99Gc5zLvQB/R6/jvgeckOaSdP51Hfj17rjoW9byemRyogX4d8Px2+tnArcDfJXlKkicAPw88OclNSW5JcknbTpKvJ7koyV8Br2gvPPaVdv5fL8XGzNPTgXur6n6Aqrq3qr450OcvgJ9b9Mr27XrgCIAkJyW5vn1srpv+JNGOvP8kyf9O8tUk75n+4ySvTXJnki8DJ3faVya5uv2k8vkkR7XtH0vye0luSLK7/eR1SZLbk3xsITawqu6hOQt6Qxo/+bSX5JR2JL+93e4ndkdsSQ5O8gftc/WmJC9u27vruKDdhmvabZoO+ncDx7TrfrRv4i8GflRVH+5sz83AXwKHJrmifY18sg1/kpyfZFv7CWxTp/2aJL+bZBJ4U5KXpblg301J/jzJz7T9Du1s844kL0/ybuCQdls+2fZ7VZIvt20fmQ7vJN9P80n8Zh7Kgv21Bfildnod8EfTC5I8Ncln2lpvSHJ8235Bko8nuZbmm37nJPlf7f3w1STv7Kx/WZKPpvlE97nOm8foVNUB+Q/4GnAU8G+B3wDeBbyU5gW/jWbUemzb9w+BN7fTXwf+Yzt9cNtvFRDgcuBPl3rb5tjuQ4HtwJ3AxcApbfs1wHg7/VbgsiWu8/vt/8uAT9NcGgLgScBB7fRpwB+30+cAu2lOSjsY+Buak9WeDnwDGAOW03w6+1D7N1cCr2mn/w3wmXb6YzTXHArNdYa+BxxHM4C5EThhlNs40PYdmpHtqdPPpbbOkzuP30HASuDWtu1cmq8DAzyr3d6DB9ZxAc1A5gk0n0jvAx7fXc9+bMcbgQ/M0H4q8F2akwkfR/PG/MJ22VM7/T4OvKzzPLy4s+wpPPRtutcD72unLwJ+t9tv8D6lGZhdCTy+nb8Y+LV2uoBfHeXzFTie5lpUB9O8xrr3/weBd7bTvwhs7zwuNwKHdJ7Hfws8DTiEZrA53j5Oe6efezRZ86pRv+4O1BE6NE/uF7T/rm//Tc/vAb5WVXe2fS8FXtT528va/5/V9vtqNffyJ3iMq6rvAyfSjAangMuSnNMu/mSS7TRvav9haSr8iUPaWv4vTcBd1bY/Gfh0Ozr9AA/fVfb5qvpuVf0QuA14BvA8ml0BU9VcHO6yTv/nA59qpz8OvLCz7Mr2Mb0F+H9VdUtV/RjYSfPiWkzXAu9vR9WHVdXegeUvpH3uVdVXaN7Mjp1hPZ+tqvuruUbSPeznLpEhfbmq9rT33XYeuu9e3I68b6EJuO7j2H2MVgBb235v7fQ7jeYqrgBU1bdnuO2X0DzXt7XPpZcAP9suexD44/3ZsEFVtYNm+9bRjNa7XkjzHKOqrgaeluRJ7bKJqvpBp+9VVXVf2/YnPPS8/FpVbW+nb2QBnocHcqBP70c/juZd8AaaF/gLaEYJs/n7Ba1sgVXVg1V1TVW9E9gAvLxd9Mpq9un+clXdNcsqFsMPquoEmlAOD+1DfxfwhWr2rb+MZjQ07f7O9IPs37WGptf144H1/ng/17tPSX6Wpu57uu1V9W6a0ekhwLVJnvUob2KU90/XTprgHOo2kxxMM1o+u6qOAz7Kwx/H7uvrgzSfqI6j+TTd7TeXAJe2z+kTquqZVXVBu+yHVfXgPNY1rAngd+jsbhnCYJ4MntwzPb9Qj99PHMiBfh3NwcFvtQH3LZoDRM+needemWR6P/KrgS/OsI6vtP2OaefXLXDN+y3JM5Os6jSdQDOie0yqqn+g+Uh/bh66zs/0waZzhljFl4BTkjwtyeOBV3SWXcdDZyW/kmaf75JIMgZ8mCa8amDZMe0nhItodgcOBvpf0tRPkmNpdiUOeyXSvwOeuD+1A1cDT0hzNdTpmo8H/uU++k+H8r1JDqW53Me+dB/v13Tar6JzoDzJU9rJH7WPM8DngbOT/HTb56lJnjHE9uyPS4DfqqpbBtq7j9GpNMexvrePdZze1noI8Ms0g89FcSAH+i00+xJvGGj7bjWX8X0tzUf7W2hGZR8eXEH70X498Nk0B0XvGezzGHQocGmS25LsoPnRkQuWtqTZVdVNwA6aN8z3AL+d5CaGu/TE39Js3/U0L4zbO4vfALy2vR9eDbxptJXPafoA3k7gz4HPAb81Q783twcPdwA/Av6sbZ8O/ouBx7XP1cuAc6o96D2XqrqPZtR/66M9KNq+Af0KcFqary3uBH6bZnfZTP2/QzMqv5XmkiDbZln9BTSvwxt5+CVm/wvwlLbum2kOzEJzGv2OJJ+sqtuAdwCfa++7q2iOqSyYdvfSTF8VvQA4sa3j3Tz8zWnQl2kGlTtojhFNjrzQffDUf2kJJDkReH9VnbLUtWh02uNZ49X5vYjFdCCP0KUDUpJxmn20/22pa1G/OEKXpJ5whC5JPWGgS1JPGOiS1BMGuiT1hIEuST3x/wHNObM9FdbcdQAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAVzUlEQVR4nO3dfbRddX3n8ffHYIRVVFRuOw4BQ2nQRmGw3MFRHMEKawU7QjtiV7LUiqPNdE3jw8g4g6MLKc6silbtjIZq7FCpDw1IO86lphOpiO3woLmUEAgIptFKsDNc8Km2Cga/88feVzaHm3vPJefeS3bfr7Wysvdv/+4+330ePud39j57n1QVkqQD3+OWugBJ0mgY6JLUEwa6JPWEgS5JPWGgS1JPHLRUN3z44YfXypUrl+rmJemAdOONN95bVWMzLVuyQF+5ciWTk5NLdfOSdEBK8jf7WuYuF0nqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6YqhAT7ImyR1JdiU5b4blRyX5QpKbkuxI8tLRlypJms2cgZ5kGbAROANYDaxLsnqg2zuAy6vqucBa4OJRFypJmt0wI/STgF1VtbuqHgA2A2cN9CngSe30k4Fvjq5ESdIwhjlT9Ajgrs78HuB5A30uAD6X5A3ATwGnjaQ6SYvmQ+deudQlzMuG971sqUt4zBnVQdF1wMeqagXwUuDjSR6x7iTrk0wmmZyamhrRTUuSYLhAvxs4sjO/om3reh1wOUBVXQ8cDBw+uKKq2lRV41U1PjY247VlJEmP0jCBvg1YleToJMtpDnpODPT5BvASgCQ/TxPoDsElaRHNGehVtRfYAGwFbqf5NsvOJBcmObPtdi7w60luBv4IOKf89WlJWlRDXT63qrYAWwbazu9M3wacPNrSJEnz4ZmiktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUE0MFepI1Se5IsivJeTMs/0CS7e2/O5N8Z/SlSpJmM+dP0CVZBmwETgf2ANuSTLQ/OwdAVf37Tv83AM9dgFolSbMYZoR+ErCrqnZX1QPAZuCsWfqvo/mhaEnSIhom0I8A7urM72nbHiHJM4Cjgav3sXx9kskkk1NTU/OtVZI0i1EfFF0LXFFVD860sKo2VdV4VY2PjY2N+KYl6R+3YQL9buDIzvyKtm0ma3F3iyQtiWECfRuwKsnRSZbThPbEYKckzwKeAlw/2hIlScOYM9Crai+wAdgK3A5cXlU7k1yY5MxO17XA5qqqhSlVkjSbOb+2CFBVW4AtA23nD8xfMLqyJEnz5ZmiktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPXEUF9blKQD3X991dlLXcK8vP0TV8z7bxyhS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUE0MFepI1Se5IsivJefvo86tJbkuyM8mnRlumJGkuc17LJckyYCNwOrAH2JZkoqpu6/RZBbwNOLmqvp3kpxeqYEnSzIYZoZ8E7Kqq3VX1ALAZOGugz68DG6vq2wBVdc9oy5QkzWWYQD8CuKszv6dt6zoWODbJtUluSLJmphUlWZ9kMsnk1NTUo6tYkjSjUR0UPQhYBZwKrAM+muSwwU5VtamqxqtqfGxsbEQ3LUmC4QL9buDIzvyKtq1rDzBRVT+qqq8Bd9IEvCRpkQwT6NuAVUmOTrIcWAtMDPT5DM3onCSH0+yC2T3COiVJc5gz0KtqL7AB2ArcDlxeVTuTXJjkzLbbVuC+JLcBXwDeWlX3LVTRkqRHGuon6KpqC7BloO38znQBb2n/SZKWgGeKSlJPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9cRQ30NfTCe+9Q+XuoR5ufG9v7bUJUgS4AhdknrDQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4w0CWpJwx0SeoJA12SemKoQE+yJskdSXYlOW+G5eckmUqyvf33+tGXKkmazZyn/idZBmwETgf2ANuSTFTVbQNdL6uqDQtQoyRpCMOM0E8CdlXV7qp6ANgMnLWwZUmS5muYQD8CuKszv6dtG/TyJDuSXJHkyJlWlGR9kskkk1NTU4+iXEnSvozqoOiVwMqqOh64Crh0pk5VtamqxqtqfGxsbEQ3LUmC4QL9bqA74l7Rtv1EVd1XVfe3s78PnDia8iRJwxom0LcBq5IcnWQ5sBaY6HZI8vTO7JnA7aMrUZI0jDm/5VJVe5NsALYCy4BLqmpnkguByaqaAN6Y5ExgL/At4JwFrFmSNIOhfrGoqrYAWwbazu9Mvw1422hLkyTNh2eKSlJPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPDHW1RUnwxRedstQlzNspf/HFpS5Bi8gRuiT1hIEuST1hoEtSTwwV6EnWJLkjya4k583S7+VJKsn46EqUJA1jzkBPsgzYCJwBrAbWJVk9Q78nAm8CvjTqIiVJcxtmhH4SsKuqdlfVA8Bm4KwZ+r0LuAj44QjrkyQNaZhAPwK4qzO/p237iSS/ABxZVZ+dbUVJ1ieZTDI5NTU172IlSfu23wdFkzwOeD9w7lx9q2pTVY1X1fjY2Nj+3rQkqWOYQL8bOLIzv6Jtm/ZE4DnANUm+DvwLYMIDo5K0uIYJ9G3AqiRHJ1kOrAUmphdW1Xer6vCqWllVK4EbgDOranJBKpYkzWjOQK+qvcAGYCtwO3B5Ve1McmGSMxe6QEnScIa6lktVbQG2DLSdv4++p+5/WZKk+fJMUUnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ7wN0U1Mid/8OSlLmHern3DtUtdgjQyjtAlqScMdEnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4YKtCTrElyR5JdSc6bYflvJLklyfYk/yfJ6tGXKkmazZyBnmQZsBE4A1gNrJshsD9VVcdV1QnAe4D3j7xSSdKshhmhnwTsqqrdVfUAsBk4q9uhqr7Xmf0poEZXoiRpGMNcy+UI4K7O/B7geYOdkvwm8BZgOfCLM60oyXpgPcBRRx0131olSbMY2UHRqtpYVccA/wl4xz76bKqq8aoaHxsbG9VNS5IYboR+N3BkZ35F27Yvm4Hf25+i+uobFx631CXM21Hn37LUJUga0jAj9G3AqiRHJ1kOrAUmuh2SrOrM/hLw1dGVKEkaxpwj9Kram2QDsBVYBlxSVTuTXAhMVtUEsCHJacCPgG8Dr1nIoiVJjzTUD1xU1RZgy0Db+Z3pN424LknSPHmmqCT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9cRQgZ5kTZI7kuxKct4My9+S5LYkO5J8PskzRl+qJGk2cwZ6kmXARuAMYDWwLsnqgW43AeNVdTxwBfCeURcqSZrdMCP0k4BdVbW7qh4ANgNndTtU1Req6h/a2RuAFaMtU5I0l2EC/Qjgrs78nrZtX14H/NlMC5KsTzKZZHJqamr4KiVJcxrpQdEkrwLGgffOtLyqNlXVeFWNj42NjfKmJekfvYOG6HM3cGRnfkXb9jBJTgPeDpxSVfePpjxJ0rCGGaFvA1YlOTrJcmAtMNHtkOS5wEeAM6vqntGXKUmay5yBXlV7gQ3AVuB24PKq2pnkwiRntt3eCxwKfDrJ9iQT+1idJGmBDLPLharaAmwZaDu/M33aiOuSJM2TZ4pKUk8Y6JLUEwa6JPWEgS5JPWGgS1JPGOiS1BMGuiT1hIEuST1hoEtSTxjoktQTBrok9YSBLkk9YaBLUk8Y6JLUEwa6JPWEgS5JPWGgS1JPDBXoSdYkuSPJriTnzbD8RUn+KsneJGePvkxJ0lzmDPQky4CNwBnAamBdktUD3b4BnAN8atQFSpKGM8xvip4E7Kqq3QBJNgNnAbdNd6iqr7fLfrwANUqShjDMLpcjgLs683vatnlLsj7JZJLJqampR7MKSdI+LOpB0araVFXjVTU+Nja2mDctSb03TKDfDRzZmV/RtkmSHkOGCfRtwKokRydZDqwFJha2LEnSfM0Z6FW1F9gAbAVuBy6vqp1JLkxyJkCSf55kD/AK4CNJdi5k0ZKkRxrmWy5U1RZgy0Db+Z3pbTS7YiRJS8QzRSWpJwx0SeoJA12SesJAl6SeMNAlqScMdEnqCQNdknrCQJeknjDQJaknDHRJ6gkDXZJ6wkCXpJ4w0CWpJwx0SeoJA12SesJAl6SeMNAlqSeGCvQka5LckWRXkvNmWP6EJJe1y7+UZOWoC5UkzW7OQE+yDNgInAGsBtYlWT3Q7XXAt6vq54APABeNulBJ0uyGGaGfBOyqqt1V9QCwGThroM9ZwKXt9BXAS5JkdGVKkuaSqpq9Q3I2sKaqXt/Ovxp4XlVt6PS5te2zp53/67bPvQPrWg+sb2efCdwxqg0ZwuHAvXP2OnC5fQeuPm8buH2j9oyqGptpwUGLWARVtQnYtJi3OS3JZFWNL8VtLwa378DV520Dt28xDbPL5W7gyM78irZtxj5JDgKeDNw3igIlScMZJtC3AauSHJ1kObAWmBjoMwG8pp0+G7i65tqXI0kaqTl3uVTV3iQbgK3AMuCSqtqZ5EJgsqomgP8BfDzJLuBbNKH/WLMku3oWkdt34OrztoHbt2jmPCgqSToweKaoJPWEgS5JPXFABnqSDyR5c2d+a5Lf78y/L8lbHsV6T03yp6Oqc6EkeXuSnUl2JNme5HlJrmkvz3BzkmuTPHOJa3ywre3WJFcmOWxE6z0nyYdGsa791dnGne39fm6Sx7XLxpP891n+9p8muWKO9c+6jrbPYUn+3aPbgoet558k2Zzkr5PcmGRLkvWL/XpI8p8X8/Y6t1tJPtGZPyjJ1Ki2P8nK9nydBXVABjpwLfACgPYFdDjw7M7yFwDXzbWS9rIGB5Qkzwf+FfALVXU8cBpwV7v4lVX1z2jO2n3vEpU47QdVdUJVPYfmQPlvLnE9C2F6G58NnE5zeYx3AlTVZFW9cV9/WFXfrKqzZ1v5XOtoHQbsV6C3Z3X/T+Caqjqmqk4E3gb8zH6u99Gc5zLvQB/R6/jvgeckOaSdP51Hfj17rjoW9byemRyogX4d8Px2+tnArcDfJXlKkicAPw88OclNSW5JcknbTpKvJ7koyV8Br2gvPPaVdv5fL8XGzNPTgXur6n6Aqrq3qr450OcvgJ9b9Mr27XrgCIAkJyW5vn1srpv+JNGOvP8kyf9O8tUk75n+4ySvTXJnki8DJ3faVya5uv2k8vkkR7XtH0vye0luSLK7/eR1SZLbk3xsITawqu6hOQt6Qxo/+bSX5JR2JL+93e4ndkdsSQ5O8gftc/WmJC9u27vruKDdhmvabZoO+ncDx7TrfrRv4i8GflRVH+5sz83AXwKHJrmifY18sg1/kpyfZFv7CWxTp/2aJL+bZBJ4U5KXpblg301J/jzJz7T9Du1s844kL0/ybuCQdls+2fZ7VZIvt20fmQ7vJN9P80n8Zh7Kgv21Bfildnod8EfTC5I8Ncln2lpvSHJ8235Bko8nuZbmm37nJPlf7f3w1STv7Kx/WZKPpvlE97nOm8foVNUB+Q/4GnAU8G+B3wDeBbyU5gW/jWbUemzb9w+BN7fTXwf+Yzt9cNtvFRDgcuBPl3rb5tjuQ4HtwJ3AxcApbfs1wHg7/VbgsiWu8/vt/8uAT9NcGgLgScBB7fRpwB+30+cAu2lOSjsY+Buak9WeDnwDGAOW03w6+1D7N1cCr2mn/w3wmXb6YzTXHArNdYa+BxxHM4C5EThhlNs40PYdmpHtqdPPpbbOkzuP30HASuDWtu1cmq8DAzyr3d6DB9ZxAc1A5gk0n0jvAx7fXc9+bMcbgQ/M0H4q8F2akwkfR/PG/MJ22VM7/T4OvKzzPLy4s+wpPPRtutcD72unLwJ+t9tv8D6lGZhdCTy+nb8Y+LV2uoBfHeXzFTie5lpUB9O8xrr3/weBd7bTvwhs7zwuNwKHdJ7Hfws8DTiEZrA53j5Oe6efezRZ86pRv+4O1BE6NE/uF7T/rm//Tc/vAb5WVXe2fS8FXtT528va/5/V9vtqNffyJ3iMq6rvAyfSjAangMuSnNMu/mSS7TRvav9haSr8iUPaWv4vTcBd1bY/Gfh0Ozr9AA/fVfb5qvpuVf0QuA14BvA8ml0BU9VcHO6yTv/nA59qpz8OvLCz7Mr2Mb0F+H9VdUtV/RjYSfPiWkzXAu9vR9WHVdXegeUvpH3uVdVXaN7Mjp1hPZ+tqvuruUbSPeznLpEhfbmq9rT33XYeuu9e3I68b6EJuO7j2H2MVgBb235v7fQ7jeYqrgBU1bdnuO2X0DzXt7XPpZcAP9suexD44/3ZsEFVtYNm+9bRjNa7XkjzHKOqrgaeluRJ7bKJqvpBp+9VVXVf2/YnPPS8/FpVbW+nb2QBnocHcqBP70c/juZd8AaaF/gLaEYJs/n7Ba1sgVXVg1V1TVW9E9gAvLxd9Mpq9un+clXdNcsqFsMPquoEmlAOD+1DfxfwhWr2rb+MZjQ07f7O9IPs37WGptf144H1/ng/17tPSX6Wpu57uu1V9W6a0ekhwLVJnvUob2KU90/XTprgHOo2kxxMM1o+u6qOAz7Kwx/H7uvrgzSfqI6j+TTd7TeXAJe2z+kTquqZVXVBu+yHVfXgPNY1rAngd+jsbhnCYJ4MntwzPb9Qj99PHMiBfh3NwcFvtQH3LZoDRM+needemWR6P/KrgS/OsI6vtP2OaefXLXDN+y3JM5Os6jSdQDOie0yqqn+g+Uh/bh66zs/0waZzhljFl4BTkjwtyeOBV3SWXcdDZyW/kmaf75JIMgZ8mCa8amDZMe0nhItodgcOBvpf0tRPkmNpdiUOeyXSvwOeuD+1A1cDT0hzNdTpmo8H/uU++k+H8r1JDqW53Me+dB/v13Tar6JzoDzJU9rJH7WPM8DngbOT/HTb56lJnjHE9uyPS4DfqqpbBtq7j9GpNMexvrePdZze1noI8Ms0g89FcSAH+i00+xJvGGj7bjWX8X0tzUf7W2hGZR8eXEH70X498Nk0B0XvGezzGHQocGmS25LsoPnRkQuWtqTZVdVNwA6aN8z3AL+d5CaGu/TE39Js3/U0L4zbO4vfALy2vR9eDbxptJXPafoA3k7gz4HPAb81Q783twcPdwA/Av6sbZ8O/ouBx7XP1cuAc6o96D2XqrqPZtR/66M9KNq+Af0KcFqary3uBH6bZnfZTP2/QzMqv5XmkiDbZln9BTSvwxt5+CVm/wvwlLbum2kOzEJzGv2OJJ+sqtuAdwCfa++7q2iOqSyYdvfSTF8VvQA4sa3j3Tz8zWnQl2kGlTtojhFNjrzQffDUf2kJJDkReH9VnbLUtWh02uNZ49X5vYjFdCCP0KUDUpJxmn20/22pa1G/OEKXpJ5whC5JPWGgS1JPGOiS1BMGuiT1hIEuST3x/wFNmLM99QJ90AAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -662,6 +808,7 @@ "import seaborn as sns\n", "import pandas as pd\n", "import time \n", + "import tkseem as tk\n", "\n", "def calc_comp(fun):\n", " tokenizer = fun()\n", @@ -700,22 +847,21 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 41, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Training WordTokenizer ...\n", - "Saving as pickle file ...\n" + "Training WordTokenizer ...\n" ] } ], "source": [ "tokenizer = tk.WordTokenizer()\n", "tokenizer.train('samples/data.txt')\n", - "tokenizer.save_model('freq.pl')" + "tokenizer.save(file_path = \".\", name = 'm')" ] }, { @@ -727,25 +873,17 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 44, "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Loading as pickle file ...\n" - ] - } - ], + "outputs": [], "source": [ "tokenizer = tk.WordTokenizer()\n", - "tokenizer.load_model('freq.pl')" + "tokenizer.load(file_path = \".\", name ='m')" ] }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 45, "metadata": {}, "outputs": [ { @@ -754,7 +892,7 @@ "['السلام', 'عليكم']" ] }, - "execution_count": 26, + "execution_count": 45, "metadata": {}, "output_type": "execute_result" } @@ -798,7 +936,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASP0lEQVR4nO3df5BlZX3n8fdHBoUKKiCdWVaCYxQ1GpQsXRiEDSi6xcYYSUKoUGoGgzXZ2pBohTVrkq2Am6QCSRRdXddFJU6URAhqAPNDySiagAIzYZhhwIhBSSDoDCr+SIwR/OaP87RzaXqmb3ff7uaZer+qpvqc5zz33O9zf3zuc8+5906qCklSfx612gVIkhbHAJekThngktQpA1ySOmWAS1Kn1qzklR122GG1bt26lbxKSereli1b7quqqdntKxrg69atY/PmzSt5lZLUvSR3zdU+VoAn+TzwdeBB4IGqmk5yKHAZsA74PHBGVX1lEsVKkua3kGPgz6+qY6pquq2/DthUVUcBm9q6JGmFLOUk5kuBjW15I3Da0suRJI1r3AAv4CNJtiTZ0NrWVtW9bfkLwNqJVydJ2qNxT2KeWFX3JPle4Joknx7dWFWVZM4fVWmBvwHgyCOPXFKxkqTdxpqBV9U97e9O4IPAccAXkxwO0P7u3MNlL66q6aqanpp62KdgJEmLNG+AJ/meJI+dWQb+C3ArcBWwvnVbD1y5XEVKkh5unEMoa4EPJpnp/0dV9ZdJbgIuT3I2cBdwxvKVKUmabd4Ar6o7gefM0f4l4JTlKEqSNL8V/Sam9j0nvOWE1S5hwa77xetWuwRpIvwxK0nqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6tTYAZ5kvyQ3J/lQW39ykhuSfDbJZUkevXxlSpJmW8gM/NXA7SPrFwIXVdVTga8AZ0+yMEnS3o0V4EmOAF4MvLOtB3gBcEXrshE4bTkKlCTNbdwZ+JuAXwG+09afANxfVQ+09buBJ851wSQbkmxOsnnXrl1LKlaStNu8AZ7kx4CdVbVlMVdQVRdX1XRVTU9NTS1mF5KkOawZo88JwI8n+VHgAOBxwJuBg5OsabPwI4B7lq9MSdJs887Aq+pXq+qIqloH/Azw0ap6GfAx4PTWbT1w5bJVKUl6mKV8Dvx/Ar+c5LMMx8TfNZmSJEnjGOcQyndV1bXAtW35TuC4yZckSRqH38SUpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROzRvgSQ5IcmOSW5LsSPL61v7kJDck+WySy5I8evnLlSTNGGcG/i3gBVX1HOAY4NQkPwxcCFxUVU8FvgKcvXxlSpJmmzfAa/CNtrp/+1fAC4ArWvtG4LRlqVCSNKexjoEn2S/JVmAncA3w98D9VfVA63I38MQ9XHZDks1JNu/atWsSNUuSGDPAq+rBqjoGOAI4DnjGuFdQVRdX1XRVTU9NTS2yTEnSbAv6FEpV3Q98DDgeODjJmrbpCOCeCdcmSdqLcT6FMpXk4LZ8IPAi4HaGID+9dVsPXLlcRUqSHm7N/F04HNiYZD+GwL+8qj6U5DbgfUl+C7gZeNcy1ilJmmXeAK+qbcAPzdF+J8PxcEnSKvCbmJLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6NW+AJ/m+JB9LcluSHUle3doPTXJNkjva30OWv1xJ0oxxZuAPAOdW1TOBHwZ+IckzgdcBm6rqKGBTW5ckrZB5A7yq7q2qv23LXwduB54IvBTY2LptBE5briIlSQ+3oGPgSdYBPwTcAKytqnvbpi8Aa/dwmQ1JNifZvGvXriWUKkkaNXaAJzkIeD/wmqr62ui2qiqg5rpcVV1cVdNVNT01NbWkYiVJu40V4En2ZwjvS6vqA635i0kOb9sPB3YuT4mSpLmM8ymUAO8Cbq+qN45sugpY35bXA1dOvjxJ0p6sGaPPCcArgO1Jtra2XwMuAC5PcjZwF3DG8pQoSZrLvAFeVX8DZA+bT5lsOZKkcflNTEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6tS8AZ7kkiQ7k9w60nZokmuS3NH+HrK8ZUqSZhtnBv5u4NRZba8DNlXVUcCmti5JWkHzBnhVfQL48qzmlwIb2/JG4LQJ1yVJmsdij4Gvrap72/IXgLV76phkQ5LNSTbv2rVrkVcnSZptyScxq6qA2sv2i6tquqqmp6amlnp1kqRmsQH+xSSHA7S/OydXkiRpHIsN8KuA9W15PXDlZMqRJI1rzXwdkvwxcDJwWJK7gfOAC4DLk5wN3AWcsZxF9uwf/vfRq13Cgh35G9tXuwRJY5g3wKvqzD1sOmXCtUiSFsBvYkpSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqfm/SampH3XW8+9erVLWJBz3vCS1S7hEcUZuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnfJz4JL2Sb/98tNXu4QF+/X3XrGg/s7AJalTBrgkdcoAl6ROeQxc2ouP/8hJq13Cgpz0iY+vdglaQc7AJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktSpJf2YVZJTgTcD+wHvrKoLFrqPY1/7h0spYVVs+b2fXe0SJGnxM/Ak+wH/F/ivwDOBM5M8c1KFSZL2bimHUI4DPltVd1bVvwHvA146mbIkSfNJVS3ugsnpwKlV9aq2/grguVV1zqx+G4ANbfXpwN8tvtwFOwy4bwWvb6Xty+Pbl8cGjq93Kz2+J1XV1OzGZf8PHarqYuDi5b6euSTZXFXTq3HdK2FfHt++PDZwfL17pIxvKYdQ7gG+b2T9iNYmSVoBSwnwm4Cjkjw5yaOBnwGumkxZkqT5LPoQSlU9kOQc4MMMHyO8pKp2TKyyyViVQzcraF8e3748NnB8vXtEjG/RJzElSavLb2JKUqcMcEnqVDcBnuSiJK8ZWf9wkneOrL8hyS8vYr8nJ/nQpOpcLkl+PcmOJNuSbE3y3CTXJvm7JLckuS7J01e5xgdbbbcmuTrJwRPa71lJ3jqJfS3VyBh3tNv93CSPatumk/yfvVz2Pya5Yp7973Ufrc/BSf774kbwkP38hyTvS/L3SbYk+fMkG1b6+ZDk11by+kaut5K8d2R9TZJdkxp/knVJbp3EvvakmwAHrgOeB9CeMIcBzxrZ/jzg+vl20n4CoCtJjgd+DPhPVfVs4IXAP7bNL6uq5wAbgd9bpRJnfLOqjqmqHwS+DPzCKtezHGbG+CzgRQw/JXEeQFVtrqpf2tMFq+qfqur0ve18vn00BwNLCvAkAT4IXFtVT6mqY4FfBdYucb+L+WDEggN8Qs/jfwZ+MMmBbf1FLPCj0Isc78T0FODXA8e35WcBtwJfT3JIkscAPwA8PsnNSbYnuaS1k+TzSS5M8rfATyc5Ncmn2/pPrsZgFuhw4L6q+hZAVd1XVf80q88ngKeueGV79kngiQBJjkvyyXbfXD/zTqHNrD+Q5C+T3JHkd2cunOSVST6T5EbghJH2dUk+2t6JbEpyZGt/d5L/l+RTSe5s76wuSXJ7kncvxwCraifDt4zPyeC77+aSnNRm6lvbuB87OiNLckCSP2iP1ZuTPL+1j+7j/DaGa9uYZoL9AuApbd+LfdF+PvDtqnr7yHhuAf4aOCjJFe05cmkLe5L8RpKb2jusi0far03ypiSbgVcneUmSG9q4/irJ2tbvoJExb0vyU0kuAA5sY7m09Xt5khtb2/+fCesk38jwTvsWdmfBUv058OK2fCbwxzMbkhya5E9brZ9K8uzWfn6S9yS5DnhPexxf2W6HO5KcN7L//ZK8I8M7to+MvFhMRlV18w/4HHAk8PPAfwN+E/hRhif4TQyz0qe1vn8IvKYtfx74lbZ8QOt3FBDgcuBDqz22ecZ9ELAV+AzwNuCk1n4tMN2WXwtctsp1fqP93Q/4E4afWgB4HLCmLb8QeH9bPgu4E3h8u1/uYvhy2OHAPwBTwKMZ3n29tV3mamB9W/454E/b8rsZfo8nDL/J8zXgaIZJyhbgmEmOcVbb/Qwz15NnHkutzhNG7r81wDrg1tZ2LsNHbwGe0cZ7wKx9nM8wcXkMwzvOLwH7j+5nCeP4JeCiOdpPBr7K8MW8RzG8EJ/Yth060u89wEtGHodvG9l2CLs/4fYq4A1t+ULgTaP9Zt+mDBOxq4H92/rbgJ9tywWcMcnHK/Bs4Ip222+ddfu/BTivLb8A2Dpyv2wBDhx5HN8LPAE4kGFyOd3upwdmHnsMWfPyST7nepqBw/Bgfl7798n2b2b9buBzVfWZ1ncj8CMjl72s/X1G63dHDbfqe3mEq6pvAMcyzPZ2AZclOattvjTJVoYXsf+xOhV+14Gtli8wBNo1rf3xwJ+02edFPPTQ16aq+mpV/StwG/Ak4LkMb+131fBDaZeN9D8e+KO2/B7gxJFtV7f7dDvwxaraXlXfAXYwPJlW0nXAG9us+eCqemDW9hNpj72q+jTDi9fT5tjPn1XVt6rqPmAnSzzEMaYbq+rudtttZfdt9/w2s97OEGij9+PofXQE8OHW77Uj/V7I8AumAFTVV+a47lMYHus3tcfSKcD3t20PAu9fysBmq6ptDOM7k2E2PupEhscYVfVR4AlJHte2XVVV3xzpe01Vfam1fYDdj8vPVdXWtryFCT8OewvwmePgRzO8yn2K4Qn9PIZZwN7887JWtsyq6sGquraqzgPOAX6qbXpZDcdkT6uqf9zLLlbCN6vqGIYQDruPgf8m8LEajo2/hGG2M+NbI8sPsrTf55nZ13dm7fc7S9zvHiX5foa6d4621/Db+K9imJFdl+QZi7yKSd4+o3YwBOVY15nkAIbZ8OlVdTTwDh56P44+v97C8I7paIZ3y6P95hNgY3tMH1NVT6+q89u2f62qBxewr3FdBfw+I4dPxjA7T2Z/oWZmfbnuP6C/AL+e4WTel1ugfZnhhM7xDK/M65LMHAd+BfDxOfbx6dbvKW39zGWuecmSPD3JUSNNxzDM2B6RqupfGN6in5vhJM/j2X1y6KwxdnEDcFKSJyTZH/jpkW3XM/xsA8DLGI7ZrookU8DbGcKqZm17SnsHcCHD4b3ZAf7XDPWT5GkMhwbH/aXOrwOPXUrtwEeBx2T4tdCZmp8N/Oc99J8J4fuSHATs7WTs6P29fqT9GkZObCc5pC1+u93PAJuA05N8b+tzaJInjTGepbgEeH1VbZ/VPnofncxwHupre9jHi1qtBwKnMUw2l11vAb6d4Vjgp2a1fbWq7gZeyfBWfTvDrOvts3fQ3qpvAP4sw0nMnbP7PAIdBGxMcluSbQz/gcb5q1vS3lXVzcA2hhfI3wV+J8nNjDEDqap7Gcb3SYYnwu0jm38ReGW7HV4BvHqylc9r5oTbDuCvgI8Ar5+j32vayb5twLeBv2jtM0H/NuBR7bF6GXBWtZPU86mqLzHM6m9d7EnM9oLzE8ALM3yMcAfwOwyHv+bqfz/DrPtWhp/PuGkvuz+f4Xm4hYf+5OpvAYe0um9hOJEKw9fStyW5tKpuA/4X8JF2213DcE5k2bTDRXN9dPN84NhWxwU89MVothsZJpHbGM7xbJ54oXPwq/TSCklyLPDGqjpptWvR5LTzUdM16/9CWAm9zcClLiWZZjjG+ubVrkX7DmfgktQpZ+CS1CkDXJI6ZYBLUqcMcEnqlAEuSZ36d//uSCP0EDBrAAAAAElFTkSuQmCC\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAASP0lEQVR4nO3df5BlZX3n8fdHBoUKKiCdWVaCYxQ1GpQsXRiEDSi6xcYYSUKoUGoGgzXZ2pBohTVrkq2Am6QCSRRdXddFJU6URAhqAPNDySiagAIzYZhhwIhBSSDoDCr+SIwR/OaP87RzaXqmb3ff7uaZer+qpvqc5zz33O9zf3zuc8+5906qCklSfx612gVIkhbHAJekThngktQpA1ySOmWAS1Kn1qzklR122GG1bt26lbxKSereli1b7quqqdntKxrg69atY/PmzSt5lZLUvSR3zdU+VoAn+TzwdeBB4IGqmk5yKHAZsA74PHBGVX1lEsVKkua3kGPgz6+qY6pquq2/DthUVUcBm9q6JGmFLOUk5kuBjW15I3Da0suRJI1r3AAv4CNJtiTZ0NrWVtW9bfkLwNqJVydJ2qNxT2KeWFX3JPle4Joknx7dWFWVZM4fVWmBvwHgyCOPXFKxkqTdxpqBV9U97e9O4IPAccAXkxwO0P7u3MNlL66q6aqanpp62KdgJEmLNG+AJ/meJI+dWQb+C3ArcBWwvnVbD1y5XEVKkh5unEMoa4EPJpnp/0dV9ZdJbgIuT3I2cBdwxvKVKUmabd4Ar6o7gefM0f4l4JTlKEqSNL8V/Sam9j0nvOWE1S5hwa77xetWuwRpIvwxK0nqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6tTYAZ5kvyQ3J/lQW39ykhuSfDbJZUkevXxlSpJmW8gM/NXA7SPrFwIXVdVTga8AZ0+yMEnS3o0V4EmOAF4MvLOtB3gBcEXrshE4bTkKlCTNbdwZ+JuAXwG+09afANxfVQ+09buBJ851wSQbkmxOsnnXrl1LKlaStNu8AZ7kx4CdVbVlMVdQVRdX1XRVTU9NTS1mF5KkOawZo88JwI8n+VHgAOBxwJuBg5OsabPwI4B7lq9MSdJs887Aq+pXq+qIqloH/Azw0ap6GfAx4PTWbT1w5bJVKUl6mKV8Dvx/Ar+c5LMMx8TfNZmSJEnjGOcQyndV1bXAtW35TuC4yZckSRqH38SUpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROzRvgSQ5IcmOSW5LsSPL61v7kJDck+WySy5I8evnLlSTNGGcG/i3gBVX1HOAY4NQkPwxcCFxUVU8FvgKcvXxlSpJmmzfAa/CNtrp/+1fAC4ArWvtG4LRlqVCSNKexjoEn2S/JVmAncA3w98D9VfVA63I38MQ9XHZDks1JNu/atWsSNUuSGDPAq+rBqjoGOAI4DnjGuFdQVRdX1XRVTU9NTS2yTEnSbAv6FEpV3Q98DDgeODjJmrbpCOCeCdcmSdqLcT6FMpXk4LZ8IPAi4HaGID+9dVsPXLlcRUqSHm7N/F04HNiYZD+GwL+8qj6U5DbgfUl+C7gZeNcy1ilJmmXeAK+qbcAPzdF+J8PxcEnSKvCbmJLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6NW+AJ/m+JB9LcluSHUle3doPTXJNkjva30OWv1xJ0oxxZuAPAOdW1TOBHwZ+IckzgdcBm6rqKGBTW5ckrZB5A7yq7q2qv23LXwduB54IvBTY2LptBE5briIlSQ+3oGPgSdYBPwTcAKytqnvbpi8Aa/dwmQ1JNifZvGvXriWUKkkaNXaAJzkIeD/wmqr62ui2qiqg5rpcVV1cVdNVNT01NbWkYiVJu40V4En2ZwjvS6vqA635i0kOb9sPB3YuT4mSpLmM8ymUAO8Cbq+qN45sugpY35bXA1dOvjxJ0p6sGaPPCcArgO1Jtra2XwMuAC5PcjZwF3DG8pQoSZrLvAFeVX8DZA+bT5lsOZKkcflNTEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6pQBLkmdMsAlqVMGuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktQpA1ySOmWAS1KnDHBJ6tS8AZ7kkiQ7k9w60nZokmuS3NH+HrK8ZUqSZhtnBv5u4NRZba8DNlXVUcCmti5JWkHzBnhVfQL48qzmlwIb2/JG4LQJ1yVJmsdij4Gvrap72/IXgLV76phkQ5LNSTbv2rVrkVcnSZptyScxq6qA2sv2i6tquqqmp6amlnp1kqRmsQH+xSSHA7S/OydXkiRpHIsN8KuA9W15PXDlZMqRJI1rzXwdkvwxcDJwWJK7gfOAC4DLk5wN3AWcsZxF9uwf/vfRq13Cgh35G9tXuwRJY5g3wKvqzD1sOmXCtUiSFsBvYkpSpwxwSeqUAS5JnTLAJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqfm/SampH3XW8+9erVLWJBz3vCS1S7hEcUZuCR1ygCXpE4Z4JLUKQNckjplgEtSpwxwSeqUAS5JnfJz4JL2Sb/98tNXu4QF+/X3XrGg/s7AJalTBrgkdcoAl6ROeQxc2ouP/8hJq13Cgpz0iY+vdglaQc7AJalTBrgkdcoAl6ROGeCS1CkDXJI6ZYBLUqcMcEnqlAEuSZ0ywCWpUwa4JHXKAJekThngktSpJf2YVZJTgTcD+wHvrKoLFrqPY1/7h0spYVVs+b2fXe0SJGnxM/Ak+wH/F/ivwDOBM5M8c1KFSZL2bimHUI4DPltVd1bVvwHvA146mbIkSfNJVS3ugsnpwKlV9aq2/grguVV1zqx+G4ANbfXpwN8tvtwFOwy4bwWvb6Xty+Pbl8cGjq93Kz2+J1XV1OzGZf8PHarqYuDi5b6euSTZXFXTq3HdK2FfHt++PDZwfL17pIxvKYdQ7gG+b2T9iNYmSVoBSwnwm4Cjkjw5yaOBnwGumkxZkqT5LPoQSlU9kOQc4MMMHyO8pKp2TKyyyViVQzcraF8e3748NnB8vXtEjG/RJzElSavLb2JKUqcMcEnqVDcBnuSiJK8ZWf9wkneOrL8hyS8vYr8nJ/nQpOpcLkl+PcmOJNuSbE3y3CTXJvm7JLckuS7J01e5xgdbbbcmuTrJwRPa71lJ3jqJfS3VyBh3tNv93CSPatumk/yfvVz2Pya5Yp7973Ufrc/BSf774kbwkP38hyTvS/L3SbYk+fMkG1b6+ZDk11by+kaut5K8d2R9TZJdkxp/knVJbp3EvvakmwAHrgOeB9CeMIcBzxrZ/jzg+vl20n4CoCtJjgd+DPhPVfVs4IXAP7bNL6uq5wAbgd9bpRJnfLOqjqmqHwS+DPzCKtezHGbG+CzgRQw/JXEeQFVtrqpf2tMFq+qfqur0ve18vn00BwNLCvAkAT4IXFtVT6mqY4FfBdYucb+L+WDEggN8Qs/jfwZ+MMmBbf1FLPCj0Isc78T0FODXA8e35WcBtwJfT3JIkscAPwA8PsnNSbYnuaS1k+TzSS5M8rfATyc5Ncmn2/pPrsZgFuhw4L6q+hZAVd1XVf80q88ngKeueGV79kngiQBJjkvyyXbfXD/zTqHNrD+Q5C+T3JHkd2cunOSVST6T5EbghJH2dUk+2t6JbEpyZGt/d5L/l+RTSe5s76wuSXJ7kncvxwCraifDt4zPyeC77+aSnNRm6lvbuB87OiNLckCSP2iP1ZuTPL+1j+7j/DaGa9uYZoL9AuApbd+LfdF+PvDtqnr7yHhuAf4aOCjJFe05cmkLe5L8RpKb2jusi0far03ypiSbgVcneUmSG9q4/irJ2tbvoJExb0vyU0kuAA5sY7m09Xt5khtb2/+fCesk38jwTvsWdmfBUv058OK2fCbwxzMbkhya5E9brZ9K8uzWfn6S9yS5DnhPexxf2W6HO5KcN7L//ZK8I8M7to+MvFhMRlV18w/4HHAk8PPAfwN+E/hRhif4TQyz0qe1vn8IvKYtfx74lbZ8QOt3FBDgcuBDqz22ecZ9ELAV+AzwNuCk1n4tMN2WXwtctsp1fqP93Q/4E4afWgB4HLCmLb8QeH9bPgu4E3h8u1/uYvhy2OHAPwBTwKMZ3n29tV3mamB9W/454E/b8rsZfo8nDL/J8zXgaIZJyhbgmEmOcVbb/Qwz15NnHkutzhNG7r81wDrg1tZ2LsNHbwGe0cZ7wKx9nM8wcXkMwzvOLwH7j+5nCeP4JeCiOdpPBr7K8MW8RzG8EJ/Yth060u89wEtGHodvG9l2CLs/4fYq4A1t+ULgTaP9Zt+mDBOxq4H92/rbgJ9tywWcMcnHK/Bs4Ip222+ddfu/BTivLb8A2Dpyv2wBDhx5HN8LPAE4kGFyOd3upwdmHnsMWfPyST7nepqBw/Bgfl7798n2b2b9buBzVfWZ1ncj8CMjl72s/X1G63dHDbfqe3mEq6pvAMcyzPZ2AZclOattvjTJVoYXsf+xOhV+14Gtli8wBNo1rf3xwJ+02edFPPTQ16aq+mpV/StwG/Ak4LkMb+131fBDaZeN9D8e+KO2/B7gxJFtV7f7dDvwxaraXlXfAXYwPJlW0nXAG9us+eCqemDW9hNpj72q+jTDi9fT5tjPn1XVt6rqPmAnSzzEMaYbq+rudtttZfdt9/w2s97OEGij9+PofXQE8OHW77Uj/V7I8AumAFTVV+a47lMYHus3tcfSKcD3t20PAu9fysBmq6ptDOM7k2E2PupEhscYVfVR4AlJHte2XVVV3xzpe01Vfam1fYDdj8vPVdXWtryFCT8OewvwmePgRzO8yn2K4Qn9PIZZwN7887JWtsyq6sGquraqzgPOAX6qbXpZDcdkT6uqf9zLLlbCN6vqGIYQDruPgf8m8LEajo2/hGG2M+NbI8sPsrTf55nZ13dm7fc7S9zvHiX5foa6d4621/Db+K9imJFdl+QZi7yKSd4+o3YwBOVY15nkAIbZ8OlVdTTwDh56P44+v97C8I7paIZ3y6P95hNgY3tMH1NVT6+q89u2f62qBxewr3FdBfw+I4dPxjA7T2Z/oWZmfbnuP6C/AL+e4WTel1ugfZnhhM7xDK/M65LMHAd+BfDxOfbx6dbvKW39zGWuecmSPD3JUSNNxzDM2B6RqupfGN6in5vhJM/j2X1y6KwxdnEDcFKSJyTZH/jpkW3XM/xsA8DLGI7ZrookU8DbGcKqZm17SnsHcCHD4b3ZAf7XDPWT5GkMhwbH/aXOrwOPXUrtwEeBx2T4tdCZmp8N/Oc99J8J4fuSHATs7WTs6P29fqT9GkZObCc5pC1+u93PAJuA05N8b+tzaJInjTGepbgEeH1VbZ/VPnofncxwHupre9jHi1qtBwKnMUw2l11vAb6d4Vjgp2a1fbWq7gZeyfBWfTvDrOvts3fQ3qpvAP4sw0nMnbP7PAIdBGxMcluSbQz/gcb5q1vS3lXVzcA2hhfI3wV+J8nNjDEDqap7Gcb3SYYnwu0jm38ReGW7HV4BvHqylc9r5oTbDuCvgI8Ar5+j32vayb5twLeBv2jtM0H/NuBR7bF6GXBWtZPU86mqLzHM6m9d7EnM9oLzE8ALM3yMcAfwOwyHv+bqfz/DrPtWhp/PuGkvuz+f4Xm4hYf+5OpvAYe0um9hOJEKw9fStyW5tKpuA/4X8JF2213DcE5k2bTDRXN9dPN84NhWxwU89MVothsZJpHbGM7xbJ54oXPwq/TSCklyLPDGqjpptWvR5LTzUdM16/9CWAm9zcClLiWZZjjG+ubVrkX7DmfgktQpZ+CS1CkDXJI6ZYBLUqcMcEnqlAEuSZ36d//uSCP0EDBrAAAAAElFTkSuQmCC", "text/plain": [ "
" ] @@ -861,7 +999,7 @@ }, { "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD5CAYAAADLL+UrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYAUlEQVR4nO3de5gcVZ3G8e9rIgFFSSBjxFyciAGWmwgjoKCAQQx4CbuiDzwIAeMz6xpUFlYE3cegro/gLYIIboRIQJaLiBI0CjEQcYFAJhByAYTZIGRiIAOBeEEugd/+UWdM0cxkZrp7ehLO+3meeabqnNNVp6Z73j59qrpbEYGZmeXhVYPdATMzaxyHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRob21kDSLOCDwNqI2KNU/hlgGvAC8KuIOD2VnwlMTeWfjYgbUvkk4FxgCHBRRJzd275HjhwZzc3N/T0mM7OsLV68+PGIaOqurtfQBy4Bzgcu7SqQdCgwGXhbRDwr6Q2pfDfgGGB34E3AbyXtnG72A+B9QAewSNKciLh3Uztubm6mra2tD100M7Mukh7uqa7X0I+IWyQ1VxT/G3B2RDyb2qxN5ZOBK1P5Q5Lagf1SXXtErEwdujK13WTom5lZfVU7p78z8G5Jd0j6naR3pPLRwKpSu45U1lO5mZk1UF+md3q63fbAAcA7gKslvaUeHZLUCrQCjBs3rh6bNDOzpNqRfgdwbRTuBF4ERgKrgbGldmNSWU/lLxMRMyOiJSJampq6PQ9hZmZVqjb0fwEcCpBO1G4FPA7MAY6RNEzSeGACcCewCJggabykrShO9s6ptfNmZtY/fblk8wrgEGCkpA5gOjALmCVpOfAcMCWKj+tcIelqihO0G4BpEfFC2s7JwA0Ul2zOiogVA3A8Zma2CdqcP1q5paUlfMmmmVn/SFocES3d1fkduWZmGXHom5llpNpLNm0APfLVPQe7C/0y7svLBrsLZtZHHumbmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRnoNfUmzJK1N34dbWXeapJA0Mq1L0nmS2iUtlbRPqe0USQ+mnyn1PQwzM+uLvoz0LwEmVRZKGgscDjxSKj4CmJB+WoELU9vtKb5QfX9gP2C6pBG1dNzMzPqv19CPiFuAdd1UzQBOB8rfrD4ZuDQKC4HhknYE3g/Mi4h1EfEkMI9unkjMzGxgVTWnL2kysDoi7qmoGg2sKq13pLKeys3MrIH6/R25kl4DfJFiaqfuJLVSTA0xbty4gdiFmVm2qhnp7wSMB+6R9EdgDHCXpDcCq4GxpbZjUllP5S8TETMjoiUiWpqamqronpmZ9aTfoR8RyyLiDRHRHBHNFFM1+0TEo8Ac4IR0Fc8BwPqIWAPcABwuaUQ6gXt4KjMzswbqyyWbVwC3A7tI6pA0dRPN5wIrgXbgR8CnASJiHfA1YFH6+WoqMzOzBup1Tj8iju2lvrm0HMC0HtrNAmb1s39mZlZHfkeumVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhnpy3fkzpK0VtLyUtm3JN0vaamkn0saXqo7U1K7pD9Ien+pfFIqa5d0Rv0PxczMetOXkf4lwKSKsnnAHhGxF/AAcCaApN2AY4Dd020ukDRE0hDgB8ARwG7AsamtmZk1UK+hHxG3AOsqym6MiA1pdSEwJi1PBq6MiGcj4iGgHdgv/bRHxMqIeA64MrU1M7MGqsec/ieAX6fl0cCqUl1HKuup3MzMGqim0Jf0JWADcHl9ugOSWiW1SWrr7Oys12bNzIwaQl/SicAHgeMiIlLxamBsqdmYVNZT+ctExMyIaImIlqampmq7Z2Zm3agq9CVNAk4HPhwRT5eq5gDHSBomaTwwAbgTWARMkDRe0lYUJ3vn1NZ1MzPrr6G9NZB0BXAIMFJSBzCd4mqdYcA8SQALI+JTEbFC0tXAvRTTPtMi4oW0nZOBG4AhwKyIWDEAx2NmZpvQa+hHxLHdFF+8ifZfB77eTflcYG6/emdmZnXld+SamWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWWk19CXNEvSWknLS2XbS5on6cH0e0Qql6TzJLVLWippn9JtpqT2D0qaMjCHY2Zmm9KXkf4lwKSKsjOA+RExAZif1gGOoPgy9AlAK3AhFE8SFN+tuz+wHzC964nCzMwap9fQj4hbgHUVxZOB2Wl5NnBUqfzSKCwEhkvaEXg/MC8i1kXEk8A8Xv5EYmZmA6zaOf1REbEmLT8KjErLo4FVpXYdqayncjMza6CaT+RGRABRh74AIKlVUpukts7Oznpt1szMqD70H0vTNqTfa1P5amBsqd2YVNZT+ctExMyIaImIlqampiq7Z2Zm3ak29OcAXVfgTAGuK5WfkK7iOQBYn6aBbgAOlzQincA9PJWZmVkDDe2tgaQrgEOAkZI6KK7CORu4WtJU4GHgY6n5XOBIoB14GjgJICLWSfoasCi1+2pEVJ4cNjOzAdZr6EfEsT1UTeymbQDTetjOLGBWv3pnZmZ15XfkmpllxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpaRmkJf0r9LWiFpuaQrJG0tabykOyS1S7pK0lap7bC03p7qm+txAGZm1ndVh76k0cBngZaI2AMYAhwDnAPMiIi3Ak8CU9NNpgJPpvIZqZ2ZmTVQrdM7Q4FtJA0FXgOsAd4LXJPqZwNHpeXJaZ1UP1GSaty/mZn1Q9WhHxGrgW8Dj1CE/XpgMfBURGxIzTqA0Wl5NLAq3XZDar9Dtfs3M7P+q2V6ZwTF6H088CbgtcCkWjskqVVSm6S2zs7OWjdnZmYltUzvHAY8FBGdEfE8cC1wIDA8TfcAjAFWp+XVwFiAVL8d8ETlRiNiZkS0RERLU1NTDd0zM7NKtYT+I8ABkl6T5uYnAvcCNwNHpzZTgOvS8py0Tqq/KSKihv2bmVk/1TKnfwfFCdm7gGVpWzOBLwCnSmqnmLO/ON3kYmCHVH4qcEYN/TYzsyoM7b1JzyJiOjC9onglsF83bZ8BPlrL/szMrDZ+R66ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGakp9CUNl3SNpPsl3SfpnZK2lzRP0oPp94jUVpLOk9QuaamkfepzCGZm1le1jvTPBX4TEbsCbwPuo/jC8/kRMQGYz8YvQD8CmJB+WoELa9y3mZn1U9WhL2k74D3AxQAR8VxEPAVMBmanZrOBo9LyZODSKCwEhkvaseqem5lZv9Uy0h8PdAI/lnS3pIskvRYYFRFrUptHgVFpeTSwqnT7jlRmZmYNUkvoDwX2AS6MiLcDf2PjVA4AERFA9GejkloltUlq6+zsrKF7ZmZWqZbQ7wA6IuKOtH4NxZPAY13TNun32lS/Ghhbuv2YVPYSETEzIloioqWpqamG7pmZWaWqQz8iHgVWSdolFU0E7gXmAFNS2RTgurQ8BzghXcVzALC+NA1kZmYNMLTG238GuFzSVsBK4CSKJ5KrJU0FHgY+ltrOBY4E2oGnU1szM2ugmkI/IpYALd1UTeymbQDTatmfmZnVxu/INTPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLSM2hL2mIpLsl/TKtj5d0h6R2SVelr1JE0rC03p7qm2vdt5mZ9U89RvqfA+4rrZ8DzIiItwJPAlNT+VTgyVQ+I7UzM7MGqin0JY0BPgBclNYFvBe4JjWZDRyVliendVL9xNTezMwapNaR/veA04EX0/oOwFMRsSGtdwCj0/JoYBVAql+f2puZWYNUHfqSPgisjYjFdewPkloltUlq6+zsrOemzcyyV8tI/0Dgw5L+CFxJMa1zLjBc0tDUZgywOi2vBsYCpPrtgCcqNxoRMyOiJSJampqaauiemZlVqjr0I+LMiBgTEc3AMcBNEXEccDNwdGo2BbguLc9J66T6myIiqt2/mZn130Bcp/8F4FRJ7RRz9hen8ouBHVL5qcAZA7BvMzPbhKG9N+ldRCwAFqTllcB+3bR5BvhoPfZnZmbV8Ttyzcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8tI1aEvaaykmyXdK2mFpM+l8u0lzZP0YPo9IpVL0nmS2iUtlbRPvQ7CzMz6ppaR/gbgtIjYDTgAmCZpN4ovPJ8fEROA+Wz8AvQjgAnppxW4sIZ9m5lZFaoO/YhYExF3peW/APcBo4HJwOzUbDZwVFqeDFwahYXAcEk7Vt1zMzPrt7rM6UtqBt4O3AGMiog1qepRYFRaHg2sKt2sI5WZmVmD1Bz6krYFfgacEhF/LtdFRADRz+21SmqT1NbZ2Vlr98zMrKSm0Jf0aorAvzwirk3Fj3VN26Tfa1P5amBs6eZjUtlLRMTMiGiJiJampqZaumdmZhVquXpHwMXAfRHx3VLVHGBKWp4CXFcqPyFdxXMAsL40DWRmZg0wtIbbHggcDyyTtCSVfRE4G7ha0lTgYeBjqW4ucCTQDjwNnFTDvs3MrApVh35E/C+gHqondtM+gGnV7s/MzGrnd+SamWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlpGGh76kSZL+IKld0hmN3r+ZWc4aGvqShgA/AI4AdgOOlbRbI/tgZpazqr8YvUr7Ae0RsRJA0pXAZODeBvfDzKpw/mnXD3YX+u3k73xosLuwWWl06I8GVpXWO4D9G9wHG2QHfv/Awe5Cv9z6mVv73PZ37zl4AHsyMA6+5XeD3YXNxtc/fvRgd6HfvvSTa/rVXhExQF3pZmfS0cCkiPhkWj8e2D8iTi61aQVa0+ouwB8a1kEYCTzewP01mo9vy+bj23I1+tjeHBFN3VU0eqS/GhhbWh+Tyv4hImYCMxvZqS6S2iKiZTD23Qg+vi2bj2/LtTkdW6Ov3lkETJA0XtJWwDHAnAb3wcwsWw0d6UfEBkknAzcAQ4BZEbGikX0wM8tZo6d3iIi5wNxG77ePBmVaqYF8fFs2H9+Wa7M5toaeyDUzs8Hlj2EwM8vIKzr0Jc2QdEpp/QZJF5XWvyPp1Cq2e4ikX9arnwNF0pckrZC0VNISSftLWpA+BuMeSbdK2mUQ+/dC6tdySddLGl6n7Z4o6fx6bKseSse5Iv3dT5P0qlTXIum8Tdz2TZI2eSF2b9tIbYZL+nR1R/CPbbxR0pWS/k/SYklzJbU2+n9B0hcbub+KfYekn5TWh0rqrNffQFKzpOX12FZPXtGhD9wKvAsg/ZONBHYv1b8LuK23jaSPj9iiSHon8EFgn4jYCziMjW+MOy4i3gbMBr41SF0E+HtE7B0RewDrgGmD2JeB1HWcuwPvo/gYkukAEdEWEZ/t6YYR8aeI2OQ7hnrbRjIcqDr0JQn4ObAgInaKiH2BM4FR1W4zbbea84r9Dv06/g//DdhD0jZp/X1UXHbeh740/Fxq2Ss99G8D3pmWdweWA3+RNELSMOCfgO0k3S1pmaRZqRxJf5R0jqS7gI+mD4q7P63/y2AcTD/tCDweEc8CRMTjEfGnija3AG9teM+6dzvFO7aRtJ+k29P9clvXq5E0gr9W0m8kPSjpm103lnSSpAck3QkcWCpvlnRTerUzX9K4VH6JpAslLZS0Mr16myXpPkmXDNRBRsRaijcfnqzCP141Sjo4vSJYko79deWRn6StJf04PVbvlnRoKi9v46x0HAvScXU9GZwN7JS2Xc0T/aHA8xHxw9Kx3AP8HthW0jXp/+Py9ASBpC9LWpReyc0slS+Q9D1JbcDnJH1I0h3pmH4raVRqt23peJdK+oiks4Ft0nFcntp9XNKdqey/uwJe0l9VvJq/h405UA9zgQ+k5WOBK7oqJG0v6Repvwsl7ZXKz5J0maRbgcvSY/m69Ld4UNL00vaHSPqRileGN5aeYOojIl7RP8BDwDjgX4FPAV8DjqQIhkUUo9+dU9tLgVPS8h+B09Py1qndBEDA1cAvB/vYejnubYElwAPABcDBqXwB0JKWPw9cNYh9/Gv6PQT4KcW7tQFeDwxNy4cBP0vLJwIrge3SffIwxZv9dgQeAZqArShe4Z2fbnM9MCUtfwL4RVq+BLgy3Z+TgT8De1IMhBYDe9f7OCvKnqIYJR/S9VhKfT2wdP8NBZqB5ansNIrLnAF2Tce8dcU2zqIY7AyjeGX7BPDq8naqPIbPAjO6KT8EWE/xRstXUTx5H5Tqti+1uwz4UOkxeEGpbgQbLyr5JPCdtHwO8L1yu8q/J8XA7Xrg1Wn9AuCEtBzAx+r9mAX2Aq5Jf/slFX//7wPT0/J7gSWl+2UxsE3psbwG2AHYhmJA2pLupw1djz+KrPl4PY/hlT7Sh+If4F3p5/b007XeATwUEQ+ktrOB95Rue1X6vWtq92AU98RP2MxFxF+BfSlGlZ3AVZJOTNWXS1pC8cT3H4PTQyCN2IBHKQJwXirfDvhpGuHO4KVTcvMjYn1EPEPxQX1vpvj8pgUR0RkRz7HxfoNihPc/afky4KBS3fXp/lwGPBYRyyLiRWAFxT9fo90KfDeNzodHxIaK+oNIj72IuJ/iSW/nbrbzq4h4NiIeB9ZS4xRMH9wZER3pb7eEjX+7Q9MIfhlFAJbvx/J9NAa4IbX7fKndYRSfygtARDzZzb4nUjzOF6XH0kTgLanuBeBntRxYdyJiKcUxHsvLLz8/iOJxRkTcBOwg6fWpbk5E/L3Udl5EPJHKrmXjY/OhiFiSlhdT58diDqHfNa+/J8Wz6UKKIHgXxYhjU/42oD0bYBHxQkQsiIjpwMnAR1LVcVHMMR8VEas2sYmB9veI2JsiuMXGOf2vATdHMdf/IYoRVZdnS8svUNt7Tbq29WLFdl+scbubJOktFH1fWy6PiLMpRrrbALdK2rXKXdTzb9RlBUW49ml/kramGHUfHRF7Aj/ipfdj+X/r+xSvzPakeEVebtcbAbPT43nviNglIs5Kdc9ExAv92FZ/zAG+TWlqpw8q86Tyevmu9YG4//4hh9C/jeKE5roUgusoTmq9k2IU0Cypa177eKC7jxy8P7XbKa0fO8B9rpmkXSRNKBXtTTEy3OxExNMU0wenqTjJtR0bT46d2IdN3AEcLGkHSa8GPlqqu43i4z4AjqOYgx40kpqAH1KEXFTU7ZRebZxDMfVYGfq/pzgGJO1MMW3Z1w8k/Avwuhq6fhMwTMUHInb1dy/g3T207wruxyVtC2zqZHT5/p5SKp9H6eS+pBFp8fl0PwPMB46W9IbUZntJb+7D8dRqFvCViFhWUV6+jw6hOK/25x628b7U322AoygGqAMuh9BfRjG3ubCibH1EdAAnUUwlLKMY4f2wcgNpKqEV+JWKE7lrK9tshrYFZku6V9JSii+tOWtwu9SziLgbWErxhPpN4BuS7qYPo5yIWENxbLdT/OPcV6r+DHBS+hscD3yuvj3vk64TjyuA3wI3Al/ppt0p6aTnUuB54NepvOvJ4QLgVemxehVwYqQT9b2JiCcoXj0sr+ZEbnqC+mfgMBWXbK4AvkExNddd+6coRvfLKT52ZdEmNn8Wxf/gYl76SZT/BYxIfb6H4mQyFO9uXSrp8oi4F/hP4Mb0d5tHcY5nQKXprO4ukz0L2Df15Wxe+iRW6U6KgedSivNWbXXvaDf8jlyzzZikfYHvRsSW90H91qN0fq0lSh8r3yg5jPTNtkiSWijmjM8d7L7YK4dH+mZmGfFI38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OM/D93X6YmJiS8QgAAAABJRU5ErkJggg==\n", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAD5CAYAAADLL+UrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAYAUlEQVR4nO3de5gcVZ3G8e9rIgFFSSBjxFyciAGWmwgjoKCAQQx4CbuiDzwIAeMz6xpUFlYE3cegro/gLYIIboRIQJaLiBI0CjEQcYFAJhByAYTZIGRiIAOBeEEugd/+UWdM0cxkZrp7ehLO+3meeabqnNNVp6Z73j59qrpbEYGZmeXhVYPdATMzaxyHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRob21kDSLOCDwNqI2KNU/hlgGvAC8KuIOD2VnwlMTeWfjYgbUvkk4FxgCHBRRJzd275HjhwZzc3N/T0mM7OsLV68+PGIaOqurtfQBy4Bzgcu7SqQdCgwGXhbRDwr6Q2pfDfgGGB34E3AbyXtnG72A+B9QAewSNKciLh3Uztubm6mra2tD100M7Mukh7uqa7X0I+IWyQ1VxT/G3B2RDyb2qxN5ZOBK1P5Q5Lagf1SXXtErEwdujK13WTom5lZfVU7p78z8G5Jd0j6naR3pPLRwKpSu45U1lO5mZk1UF+md3q63fbAAcA7gKslvaUeHZLUCrQCjBs3rh6bNDOzpNqRfgdwbRTuBF4ERgKrgbGldmNSWU/lLxMRMyOiJSJampq6PQ9hZmZVqjb0fwEcCpBO1G4FPA7MAY6RNEzSeGACcCewCJggabykrShO9s6ptfNmZtY/fblk8wrgEGCkpA5gOjALmCVpOfAcMCWKj+tcIelqihO0G4BpEfFC2s7JwA0Ul2zOiogVA3A8Zma2CdqcP1q5paUlfMmmmVn/SFocES3d1fkduWZmGXHom5llpNpLNm0APfLVPQe7C/0y7svLBrsLZtZHHumbmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRnoNfUmzJK1N34dbWXeapJA0Mq1L0nmS2iUtlbRPqe0USQ+mnyn1PQwzM+uLvoz0LwEmVRZKGgscDjxSKj4CmJB+WoELU9vtKb5QfX9gP2C6pBG1dNzMzPqv19CPiFuAdd1UzQBOB8rfrD4ZuDQKC4HhknYE3g/Mi4h1EfEkMI9unkjMzGxgVTWnL2kysDoi7qmoGg2sKq13pLKeys3MrIH6/R25kl4DfJFiaqfuJLVSTA0xbty4gdiFmVm2qhnp7wSMB+6R9EdgDHCXpDcCq4GxpbZjUllP5S8TETMjoiUiWpqamqronpmZ9aTfoR8RyyLiDRHRHBHNFFM1+0TEo8Ac4IR0Fc8BwPqIWAPcABwuaUQ6gXt4KjMzswbqyyWbVwC3A7tI6pA0dRPN5wIrgXbgR8CnASJiHfA1YFH6+WoqMzOzBup1Tj8iju2lvrm0HMC0HtrNAmb1s39mZlZHfkeumVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhlx6JuZZcShb2aWEYe+mVlGHPpmZhnpy3fkzpK0VtLyUtm3JN0vaamkn0saXqo7U1K7pD9Ien+pfFIqa5d0Rv0PxczMetOXkf4lwKSKsnnAHhGxF/AAcCaApN2AY4Dd020ukDRE0hDgB8ARwG7AsamtmZk1UK+hHxG3AOsqym6MiA1pdSEwJi1PBq6MiGcj4iGgHdgv/bRHxMqIeA64MrU1M7MGqsec/ieAX6fl0cCqUl1HKuup3MzMGqim0Jf0JWADcHl9ugOSWiW1SWrr7Oys12bNzIwaQl/SicAHgeMiIlLxamBsqdmYVNZT+ctExMyIaImIlqampmq7Z2Zm3agq9CVNAk4HPhwRT5eq5gDHSBomaTwwAbgTWARMkDRe0lYUJ3vn1NZ1MzPrr6G9NZB0BXAIMFJSBzCd4mqdYcA8SQALI+JTEbFC0tXAvRTTPtMi4oW0nZOBG4AhwKyIWDEAx2NmZpvQa+hHxLHdFF+8ifZfB77eTflcYG6/emdmZnXld+SamWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWWk19CXNEvSWknLS2XbS5on6cH0e0Qql6TzJLVLWippn9JtpqT2D0qaMjCHY2Zmm9KXkf4lwKSKsjOA+RExAZif1gGOoPgy9AlAK3AhFE8SFN+tuz+wHzC964nCzMwap9fQj4hbgHUVxZOB2Wl5NnBUqfzSKCwEhkvaEXg/MC8i1kXEk8A8Xv5EYmZmA6zaOf1REbEmLT8KjErLo4FVpXYdqayncjMza6CaT+RGRABRh74AIKlVUpukts7Oznpt1szMqD70H0vTNqTfa1P5amBsqd2YVNZT+ctExMyIaImIlqampiq7Z2Zm3ak29OcAXVfgTAGuK5WfkK7iOQBYn6aBbgAOlzQincA9PJWZmVkDDe2tgaQrgEOAkZI6KK7CORu4WtJU4GHgY6n5XOBIoB14GjgJICLWSfoasCi1+2pEVJ4cNjOzAdZr6EfEsT1UTeymbQDTetjOLGBWv3pnZmZ15XfkmpllxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpaRmkJf0r9LWiFpuaQrJG0tabykOyS1S7pK0lap7bC03p7qm+txAGZm1ndVh76k0cBngZaI2AMYAhwDnAPMiIi3Ak8CU9NNpgJPpvIZqZ2ZmTVQrdM7Q4FtJA0FXgOsAd4LXJPqZwNHpeXJaZ1UP1GSaty/mZn1Q9WhHxGrgW8Dj1CE/XpgMfBURGxIzTqA0Wl5NLAq3XZDar9Dtfs3M7P+q2V6ZwTF6H088CbgtcCkWjskqVVSm6S2zs7OWjdnZmYltUzvHAY8FBGdEfE8cC1wIDA8TfcAjAFWp+XVwFiAVL8d8ETlRiNiZkS0RERLU1NTDd0zM7NKtYT+I8ABkl6T5uYnAvcCNwNHpzZTgOvS8py0Tqq/KSKihv2bmVk/1TKnfwfFCdm7gGVpWzOBLwCnSmqnmLO/ON3kYmCHVH4qcEYN/TYzsyoM7b1JzyJiOjC9onglsF83bZ8BPlrL/szMrDZ+R66ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGXHom5llxKFvZpYRh76ZWUYc+mZmGakp9CUNl3SNpPsl3SfpnZK2lzRP0oPp94jUVpLOk9QuaamkfepzCGZm1le1jvTPBX4TEbsCbwPuo/jC8/kRMQGYz8YvQD8CmJB+WoELa9y3mZn1U9WhL2k74D3AxQAR8VxEPAVMBmanZrOBo9LyZODSKCwEhkvaseqem5lZv9Uy0h8PdAI/lnS3pIskvRYYFRFrUptHgVFpeTSwqnT7jlRmZmYNUkvoDwX2AS6MiLcDf2PjVA4AERFA9GejkloltUlq6+zsrKF7ZmZWqZbQ7wA6IuKOtH4NxZPAY13TNun32lS/Ghhbuv2YVPYSETEzIloioqWpqamG7pmZWaWqQz8iHgVWSdolFU0E7gXmAFNS2RTgurQ8BzghXcVzALC+NA1kZmYNMLTG238GuFzSVsBK4CSKJ5KrJU0FHgY+ltrOBY4E2oGnU1szM2ugmkI/IpYALd1UTeymbQDTatmfmZnVxu/INTPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLiEPfzCwjDn0zs4w49M3MMuLQNzPLSM2hL2mIpLsl/TKtj5d0h6R2SVelr1JE0rC03p7qm2vdt5mZ9U89RvqfA+4rrZ8DzIiItwJPAlNT+VTgyVQ+I7UzM7MGqin0JY0BPgBclNYFvBe4JjWZDRyVliendVL9xNTezMwapNaR/veA04EX0/oOwFMRsSGtdwCj0/JoYBVAql+f2puZWYNUHfqSPgisjYjFdewPkloltUlq6+zsrOemzcyyV8tI/0Dgw5L+CFxJMa1zLjBc0tDUZgywOi2vBsYCpPrtgCcqNxoRMyOiJSJampqaauiemZlVqjr0I+LMiBgTEc3AMcBNEXEccDNwdGo2BbguLc9J66T6myIiqt2/mZn130Bcp/8F4FRJ7RRz9hen8ouBHVL5qcAZA7BvMzPbhKG9N+ldRCwAFqTllcB+3bR5BvhoPfZnZmbV8Ttyzcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8uIQ9/MLCMOfTOzjDj0zcwy4tA3M8tI1aEvaaykmyXdK2mFpM+l8u0lzZP0YPo9IpVL0nmS2iUtlbRPvQ7CzMz6ppaR/gbgtIjYDTgAmCZpN4ovPJ8fEROA+Wz8AvQjgAnppxW4sIZ9m5lZFaoO/YhYExF3peW/APcBo4HJwOzUbDZwVFqeDFwahYXAcEk7Vt1zMzPrt7rM6UtqBt4O3AGMiog1qepRYFRaHg2sKt2sI5WZmVmD1Bz6krYFfgacEhF/LtdFRADRz+21SmqT1NbZ2Vlr98zMrKSm0Jf0aorAvzwirk3Fj3VN26Tfa1P5amBs6eZjUtlLRMTMiGiJiJampqZaumdmZhVquXpHwMXAfRHx3VLVHGBKWp4CXFcqPyFdxXMAsL40DWRmZg0wtIbbHggcDyyTtCSVfRE4G7ha0lTgYeBjqW4ucCTQDjwNnFTDvs3MrApVh35E/C+gHqondtM+gGnV7s/MzGrnd+SamWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlhGHvplZRhz6ZmYZceibmWXEoW9mlpGGh76kSZL+IKld0hmN3r+ZWc4aGvqShgA/AI4AdgOOlbRbI/tgZpazqr8YvUr7Ae0RsRJA0pXAZODeBvfDzKpw/mnXD3YX+u3k73xosLuwWWl06I8GVpXWO4D9G9wHG2QHfv/Awe5Cv9z6mVv73PZ37zl4AHsyMA6+5XeD3YXNxtc/fvRgd6HfvvSTa/rVXhExQF3pZmfS0cCkiPhkWj8e2D8iTi61aQVa0+ouwB8a1kEYCTzewP01mo9vy+bj23I1+tjeHBFN3VU0eqS/GhhbWh+Tyv4hImYCMxvZqS6S2iKiZTD23Qg+vi2bj2/LtTkdW6Ov3lkETJA0XtJWwDHAnAb3wcwsWw0d6UfEBkknAzcAQ4BZEbGikX0wM8tZo6d3iIi5wNxG77ePBmVaqYF8fFs2H9+Wa7M5toaeyDUzs8Hlj2EwM8vIKzr0Jc2QdEpp/QZJF5XWvyPp1Cq2e4ikX9arnwNF0pckrZC0VNISSftLWpA+BuMeSbdK2mUQ+/dC6tdySddLGl6n7Z4o6fx6bKseSse5Iv3dT5P0qlTXIum8Tdz2TZI2eSF2b9tIbYZL+nR1R/CPbbxR0pWS/k/SYklzJbU2+n9B0hcbub+KfYekn5TWh0rqrNffQFKzpOX12FZPXtGhD9wKvAsg/ZONBHYv1b8LuK23jaSPj9iiSHon8EFgn4jYCziMjW+MOy4i3gbMBr41SF0E+HtE7B0RewDrgGmD2JeB1HWcuwPvo/gYkukAEdEWEZ/t6YYR8aeI2OQ7hnrbRjIcqDr0JQn4ObAgInaKiH2BM4FR1W4zbbea84r9Dv06/g//DdhD0jZp/X1UXHbeh740/Fxq2Ss99G8D3pmWdweWA3+RNELSMOCfgO0k3S1pmaRZqRxJf5R0jqS7gI+mD4q7P63/y2AcTD/tCDweEc8CRMTjEfGnija3AG9teM+6dzvFO7aRtJ+k29P9clvXq5E0gr9W0m8kPSjpm103lnSSpAck3QkcWCpvlnRTerUzX9K4VH6JpAslLZS0Mr16myXpPkmXDNRBRsRaijcfnqzCP141Sjo4vSJYko79deWRn6StJf04PVbvlnRoKi9v46x0HAvScXU9GZwN7JS2Xc0T/aHA8xHxw9Kx3AP8HthW0jXp/+Py9ASBpC9LWpReyc0slS+Q9D1JbcDnJH1I0h3pmH4raVRqt23peJdK+oiks4Ft0nFcntp9XNKdqey/uwJe0l9VvJq/h405UA9zgQ+k5WOBK7oqJG0v6Repvwsl7ZXKz5J0maRbgcvSY/m69Ld4UNL00vaHSPqRileGN5aeYOojIl7RP8BDwDjgX4FPAV8DjqQIhkUUo9+dU9tLgVPS8h+B09Py1qndBEDA1cAvB/vYejnubYElwAPABcDBqXwB0JKWPw9cNYh9/Gv6PQT4KcW7tQFeDwxNy4cBP0vLJwIrge3SffIwxZv9dgQeAZqArShe4Z2fbnM9MCUtfwL4RVq+BLgy3Z+TgT8De1IMhBYDe9f7OCvKnqIYJR/S9VhKfT2wdP8NBZqB5ansNIrLnAF2Tce8dcU2zqIY7AyjeGX7BPDq8naqPIbPAjO6KT8EWE/xRstXUTx5H5Tqti+1uwz4UOkxeEGpbgQbLyr5JPCdtHwO8L1yu8q/J8XA7Xrg1Wn9AuCEtBzAx+r9mAX2Aq5Jf/slFX//7wPT0/J7gSWl+2UxsE3psbwG2AHYhmJA2pLupw1djz+KrPl4PY/hlT7Sh+If4F3p5/b007XeATwUEQ+ktrOB95Rue1X6vWtq92AU98RP2MxFxF+BfSlGlZ3AVZJOTNWXS1pC8cT3H4PTQyCN2IBHKQJwXirfDvhpGuHO4KVTcvMjYn1EPEPxQX1vpvj8pgUR0RkRz7HxfoNihPc/afky4KBS3fXp/lwGPBYRyyLiRWAFxT9fo90KfDeNzodHxIaK+oNIj72IuJ/iSW/nbrbzq4h4NiIeB9ZS4xRMH9wZER3pb7eEjX+7Q9MIfhlFAJbvx/J9NAa4IbX7fKndYRSfygtARDzZzb4nUjzOF6XH0kTgLanuBeBntRxYdyJiKcUxHsvLLz8/iOJxRkTcBOwg6fWpbk5E/L3Udl5EPJHKrmXjY/OhiFiSlhdT58diDqHfNa+/J8Wz6UKKIHgXxYhjU/42oD0bYBHxQkQsiIjpwMnAR1LVcVHMMR8VEas2sYmB9veI2JsiuMXGOf2vATdHMdf/IYoRVZdnS8svUNt7Tbq29WLFdl+scbubJOktFH1fWy6PiLMpRrrbALdK2rXKXdTzb9RlBUW49ml/kramGHUfHRF7Aj/ipfdj+X/r+xSvzPakeEVebtcbAbPT43nviNglIs5Kdc9ExAv92FZ/zAG+TWlqpw8q86Tyevmu9YG4//4hh9C/jeKE5roUgusoTmq9k2IU0Cypa177eKC7jxy8P7XbKa0fO8B9rpmkXSRNKBXtTTEy3OxExNMU0wenqTjJtR0bT46d2IdN3AEcLGkHSa8GPlqqu43i4z4AjqOYgx40kpqAH1KEXFTU7ZRebZxDMfVYGfq/pzgGJO1MMW3Z1w8k/Avwuhq6fhMwTMUHInb1dy/g3T207wruxyVtC2zqZHT5/p5SKp9H6eS+pBFp8fl0PwPMB46W9IbUZntJb+7D8dRqFvCViFhWUV6+jw6hOK/25x628b7U322AoygGqAMuh9BfRjG3ubCibH1EdAAnUUwlLKMY4f2wcgNpKqEV+JWKE7lrK9tshrYFZku6V9JSii+tOWtwu9SziLgbWErxhPpN4BuS7qYPo5yIWENxbLdT/OPcV6r+DHBS+hscD3yuvj3vk64TjyuA3wI3Al/ppt0p6aTnUuB54NepvOvJ4QLgVemxehVwYqQT9b2JiCcoXj0sr+ZEbnqC+mfgMBWXbK4AvkExNddd+6coRvfLKT52ZdEmNn8Wxf/gYl76SZT/BYxIfb6H4mQyFO9uXSrp8oi4F/hP4Mb0d5tHcY5nQKXprO4ukz0L2Df15Wxe+iRW6U6KgedSivNWbXXvaDf8jlyzzZikfYHvRsSW90H91qN0fq0lSh8r3yg5jPTNtkiSWijmjM8d7L7YK4dH+mZmGfFI38wsIw59M7OMOPTNzDLi0Dczy4hD38wsIw59M7OM/D93X6YmJiS8QgAAAABJRU5ErkJggg==", "text/plain": [ "
" ] @@ -971,7 +1109,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "Python 3.8.10 64-bit", "language": "python", "name": "python3" }, @@ -985,7 +1123,12 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.2" + "version": "3.8.10" + }, + "vscode": { + "interpreter": { + "hash": "916dbcbb3f70747c44a77c7bcd40155683ae19c65e1c03b4aa3499c5328201f1" + } } }, "nbformat": 4, diff --git a/tkseem/_base.py b/tkseem/_base.py index 0cebf3a..5f36c1b 100644 --- a/tkseem/_base.py +++ b/tkseem/_base.py @@ -4,11 +4,11 @@ import pickle import itertools import numpy as np -from tqdm import tqdm +from tqdm.notebook import tqdm from pathlib import Path from .util import split_on_binary from collections import Counter, defaultdict - +from .const import * class BaseTokenizer: """ @@ -16,20 +16,26 @@ class BaseTokenizer: """ def __init__( - self, unk_token="", pad_token="", vocab_size=10000, special_tokens=[], + self, vocab_size=10000, ): """Constructor Args: - unk_token (str, optional): unkown symbol. Defaults to "". - pad_token (str, optional): pad symbol. Defaults to "". vocab_size (int, optional): max vocab size. Defaults to 10000. - special_tokens (list, optional): user defined special tokens. Defaults to []. """ self.vocab_size = vocab_size - self.unk_token = unk_token - self.pad_token = pad_token - self.special_tokens = special_tokens + self.pad_idx = 0 + self.unk_idx = 1 + self.sow_idx = 2 + self.sos_idx = 3 + self.eos_idx = 4 + self.special_tokens = [PAD, UNK, SOW, SOS, EOS] + self.vocab = [PAD, UNK, SOW, SOS, EOS] + self.sow = SOW + self.sos = SOS + self.eos = EOS + self.pad = PAD + self.unk = UNK self.rel_path = os.path.dirname(__file__) cach_dict_path = os.path.join(self.rel_path, "dictionaries/cached.pl") self.cached = pickle.load(open(cach_dict_path, "rb")) @@ -76,8 +82,12 @@ def _get_tokens_frequency(self, file_path): """ text = open(file_path, "r").read() tokens_frequency = defaultdict(int) - for word in text.split(" "): + words = text.split(" ") + pbar = tqdm(total=len(words)) + for word in words: tokens_frequency[word] += 1 + pbar.update(1) + pbar.close() return dict(tokens_frequency) def _split_word(self, word, number_of_subwords): @@ -146,7 +156,7 @@ def _tokenize_from_dict_deprecated(self, text, freq_dict, cache=False, max_size= for word in text.split(): if len(word) >= max_size: print(f"{word} is too long ...") - output_tokens.append(self.unk_token) + output_tokens.append(self.unk) continue if word in freq_dict: output_tokens.append(word) @@ -170,7 +180,7 @@ def _tokenize_from_dict_deprecated(self, text, freq_dict, cache=False, max_size= if groups_of_valid_subwords: break if len(groups_of_valid_subwords) == 0: - output_tokens.append(self.unk_token) + output_tokens.append(self.unk) else: sorted_groups_of_valid_subwords = sorted( groups_of_valid_subwords, @@ -206,7 +216,7 @@ def _tokenize_from_dict( num_tokens += 1 chars = list(token) if len(chars) > max_word_size: - output_tokens.append(self.unk_token) + output_tokens.append(self.unk) continue is_bad = False @@ -235,7 +245,7 @@ def _tokenize_from_dict( sub_tokens.append(cur_substr) start = end if is_bad: - sub_tokens = [self.unk_token] + sub_tokens = [self.unk] output_tokens.extend(sub_tokens) if use_cache: if len(cache) < max_cache_size: @@ -257,8 +267,6 @@ def _truncate_dict(self, freq_dict): } limited_tokens_frequency = dict() - limited_tokens_frequency[self.unk_token] = -1 - limited_tokens_frequency[self.pad_token] = -1 for token in self.special_tokens: limited_tokens_frequency[token] = -1 limited_tokens_frequency.update( @@ -306,6 +314,23 @@ def tokenize(self, text, use_cache=False, max_cache_size=1000): ) return output_tokens + def _tokenize_word(self, text, remove_sow = True): + """tokenize a single word + + Args: + text (str): input text + use_cache (bool, optional): speed up using caching. Defaults to False. + max_cache_size (int, optional): max cacne size. Defaults to 1000. + + Returns: + list: output list of tokens + """ + output_tokens = self.tokenize(text) + if remove_sow: + return [token.replace(self.sow, "") for token in output_tokens] + else: + return output_tokens + def detokenize(self, tokens): """ Convert tokens to a string @@ -315,7 +340,7 @@ def detokenize(self, tokens): Returns: str: detokenized string """ - detokenized = " ".join(tokens).replace(" ##", "") + detokenized = " ".join(tokens).replace(f" {self.sow}", "") return detokenized def decode(self, encoded): @@ -329,6 +354,19 @@ def decode(self, encoded): """ decoded = [self.id_to_token(id) for id in encoded] return decoded + + def decode_sentences(self, encoded): + """ Decode list of lists of ids + + Args: + encoded (list of list): list of list of ids to decode + + Returns: + list: sentences + """ + decoded = [[self.id_to_token(id) for id in ids if id not in [0, 3, 4]] for ids in encoded] + decoded = [self.detokenize(tokens) for tokens in decoded] + return decoded def encode(self, text): """ Convert string to a list of ids @@ -343,7 +381,20 @@ def encode(self, text): encoded = [self.token_to_id(token) for token in tokens] return encoded - def pad(self, ids, length = 0): + def _encode_word(self, word, remove_sow = False): + """ convert a word to ids + + Args: + text (str): input string + + Returns: + list: list of ids + """ + tokens = self._tokenize_word(word, remove_sow=remove_sow) + encoded = [self.token_to_id(token) for token in tokens] + return encoded + + def pad_ids(self, ids, length = 0): """pad a set of ids to a specific length Args: @@ -353,7 +404,7 @@ def pad(self, ids, length = 0): Returns: list: padded ids. """ - pad_id = self.token_to_id(self.pad_token) + pad_id = self.token_to_id(self.pad) if length <= len(ids): return ids else: @@ -361,58 +412,66 @@ def pad(self, ids, length = 0): ids.append(pad_id) return ids - def encode_sentences(self, sentences, boundries=None, out_length=None): + def encode_sentences(self, sentences, add_boundry = False, out_length=None): """ Encode a list of sentences using the trained model Args: sentences (list): list of sentences - boundries (tuple): boundries for each sentence. + add_boundry (boolean): whether to add sos and eos. out_length (int, optional): specify the max length of encodings. Defaults to 100. Returns: - [np.array]: numpy array of encodings + [list]: array of encodings """ encodings = [] + if add_boundry: + boundries = [SOS, EOS] + max_length = 0 + pbar = tqdm(total=len(sentences)*2) for sent in sentences: encoded = self.encode(sent) - if boundries: + if add_boundry: encoded = [self.token_to_id(boundries[0])] + encoded + [self.token_to_id(boundries[1])] if len(encoded) > max_length: max_length = len(encoded) encodings.append(encoded) + pbar.update(1) if out_length: max_length = max(max_length, out_length) - pad_id = self.token_to_id(self.pad_token) + pad_id = self.token_to_id(self.pad) for i in range(len(encodings)): - encodings[i] = self.pad(encodings[i], max_length)[:out_length] - if encodings[i][-1] != pad_id and boundries: + encodings[i] = self.pad_ids(encodings[i], max_length)[:out_length] + if encodings[i][-1] != pad_id and add_boundry: encodings[i][-1] = self.token_to_id(boundries[1]) - - return np.array(encodings) + pbar.update(1) + pbar.close() + return encodings - def load_model(self, file_path): + def load(self, file_path, name = 'tok'): """Load a saved model as a frequency dictionary Args: file_path (str): file path of the dictionary """ - print("Loading as pickle file ...") - self.vocab = pickle.load(open(file_path, "rb")) + with open(f'{file_path}/{name}.model', 'rb') as handle: + self.vocab = pickle.load(handle) - def save_model(self, file_path): + def save(self, file_path, name = 'tok'): """Save a model as a freqency dictionary Args: file_path (str): file path to save the model + name (str): name of the file """ assert self.vocab - with open(f"{file_path}", "wb") as pickle_file: - print("Saving as pickle file ...") - pickle.dump(self.vocab, pickle_file) + os.makedirs(file_path, exist_ok=True) + + with open(f'{file_path}/{name}.model', 'wb') as handle: + pickle.dump(self.vocab, handle, protocol=pickle.HIGHEST_PROTOCOL) def __str__(self): return f"{self.__class__.__name__}" @@ -424,7 +483,7 @@ def calculate_compression_factor(self, text, normalized=True): tokenized = self.tokenize(word) factor += ( len(word) + 1 - if self.unk_token in tokenized + if self.unk in tokenized else len(tokenized) ) if normalized: diff --git a/tkseem/bruteforce_tokenizer.py b/tkseem/bruteforce_tokenizer.py index 59e880c..979b58b 100644 --- a/tkseem/bruteforce_tokenizer.py +++ b/tkseem/bruteforce_tokenizer.py @@ -7,8 +7,13 @@ class BruteForceTokenizer(BaseTokenizer): - """ Randomized based tokenization + """ BruteForceTokenizer tokenization """ + def __init__( + self, vocab_size=10000, + ): + super(BruteForceTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "BruteForceTokenizer" def train(self, file_path): """Train data using randomly splitted subwords diff --git a/tkseem/character_tokenizer.py b/tkseem/character_tokenizer.py index 36bbb96..0db1887 100644 --- a/tkseem/character_tokenizer.py +++ b/tkseem/character_tokenizer.py @@ -8,6 +8,11 @@ class CharacterTokenizer(BaseTokenizer): """ Character based tokenization """ + def __init__( + self, vocab_size=10000, + ): + super(CharacterTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "CharacterTokenizer" def train(self, file_path): """Train data using characters @@ -45,5 +50,5 @@ def tokenize(self, text): if token in self.vocab: output_tokens.append(token) else: - output_tokens.append(self.unk_token) + output_tokens.append(self.unk) return output_tokens diff --git a/tkseem/const.py b/tkseem/const.py new file mode 100644 index 0000000..3524c8c --- /dev/null +++ b/tkseem/const.py @@ -0,0 +1,6 @@ +SOW = '##' # START OF WORD +UNK = '' # UNKNOWN +PAD = '' # PADDING +SOS = '' +EOS = '' +RESERVERD_WORDS = [PAD, UNK, SOS, EOS] \ No newline at end of file diff --git a/tkseem/disjoint_letters_tokenizer.py b/tkseem/disjoint_letters_tokenizer.py index 0fbf2ef..20c62ef 100644 --- a/tkseem/disjoint_letters_tokenizer.py +++ b/tkseem/disjoint_letters_tokenizer.py @@ -8,6 +8,11 @@ class DisjointLetterTokenizer(BaseTokenizer): """ Disjoint Letters based tokenization """ + def __init__( + self, vocab_size=10000, + ): + super(DisjointLetterTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "DisjointLetterTokenizer" def train(self, file_path): """Train data using disjoint letters diff --git a/tkseem/morphological_tokenizer.py b/tkseem/morphological_tokenizer.py index 7919101..2e4afd4 100644 --- a/tkseem/morphological_tokenizer.py +++ b/tkseem/morphological_tokenizer.py @@ -7,6 +7,12 @@ class MorphologicalTokenizer(BaseTokenizer): """ Auto tokenization using a saved dictionary""" + def __init__( + self, vocab_size=10000, + ): + super(MorphologicalTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "MorphologicalTokenizer" + def train(self): """Use a default dictionary for training""" print("Training MorphologicalTokenizer ...") diff --git a/tkseem/random_tokenizder.py b/tkseem/random_tokenizder.py index 1d3096b..04f7af4 100644 --- a/tkseem/random_tokenizder.py +++ b/tkseem/random_tokenizder.py @@ -9,7 +9,12 @@ class RandomTokenizer(BaseTokenizer): """ Randomized based tokenization """ - + def __init__( + self, vocab_size=10000, + ): + super(RandomTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "RandomTokenizer" + def train(self, file_path): """Train data using randomly splitted subwords diff --git a/tkseem/sentencepiece_tokenizer.py b/tkseem/sentencepiece_tokenizer.py index be06e7c..bfec2c6 100644 --- a/tkseem/sentencepiece_tokenizer.py +++ b/tkseem/sentencepiece_tokenizer.py @@ -1,7 +1,6 @@ import io - +import os import sentencepiece as spm - from ._base import BaseTokenizer @@ -9,6 +8,13 @@ class SentencePieceTokenizer(BaseTokenizer): """ Sentencepiece based tokenization. """ + def __init__( + self, vocab_size=10000, + ): + super(SentencePieceTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "SentencePieceTokenizer" + self.sow = '▁' + def train(self, file_path, model_type="bpe"): """ Train using sentence piece @@ -25,15 +31,13 @@ def train(self, file_path, model_type="bpe"): vocab_size=self.vocab_size, model_type=model_type, character_coverage=1.0, - unk_id=0, - pad_id=1, - bos_id=-1, - eos_id=-1, - user_defined_symbols=self.special_tokens, + unk_id=self.unk_idx, + pad_id=self.pad_idx, + bos_id=self.sos_idx, + eos_id=self.eos_idx, normalization_rule_name="identity", ) - self.save_model("m.model") - self.sp = spm.SentencePieceProcessor(model_file="m.model") + self.sp = spm.SentencePieceProcessor(model_proto=self.model.getvalue()) self.vocab_size = self.sp.vocab_size() def tokenize(self, text): @@ -47,22 +51,23 @@ def tokenize(self, text): """ return self.sp.encode(text, out_type=str) - def load_model(self, file_path): + def load(self, file_path, name = 'tok'): """Load a saved sp model Args: file_path (str): file path of the trained model """ self.sp = spm.SentencePieceProcessor() - self.sp.Load(file_path) + self.sp.Load(f'{file_path}/{name}.model') - def save_model(self, file_path): + def save(self, file_path, name = 'tok'): """Save a model as a freqency dictionary Args: file_path (str): file path to save the model """ - with open(file_path, "wb") as f: + os.makedirs(file_path, exist_ok=True) + with open(f'{file_path}/{name}.model', "wb") as f: f.write(self.model.getvalue()) def id_to_token(self, id): @@ -102,4 +107,4 @@ def detokenize(self, tokens): Returns: str: detokenized string """ - return "".join(tokens).replace("▁", " ") + return "".join(tokens).replace(f"{self.sow}", " ").strip() diff --git a/tkseem/tokenizers.py b/tkseem/tokenizers.py index f2042f6..dcdaa73 100644 --- a/tkseem/tokenizers.py +++ b/tkseem/tokenizers.py @@ -25,18 +25,18 @@ class BaseTokenizer: """ def __init__( - self, unk_token="", pad_token="", vocab_size=10000, special_tokens=[], + self, unk_token="", pad="", vocab_size=10000, special_tokens=[], ): """Constructor Args: unk_token (str, optional): reserved token for unknowns. Defaults to "". - pad_token (str, optional): reserved token for padding. Defaults to "". + pad (str, optional): reserved token for padding. Defaults to "". max_tokens (int, optional): max number of vocabulary. Defaults to 10000. """ self.vocab_size = vocab_size self.unk_token = unk_token - self.pad_token = pad_token + self.pad = pad self.special_tokens = special_tokens self.rel_path = os.path.dirname(__file__) @@ -216,7 +216,7 @@ def _truncate_dict(self, freq_dict): limited_tokens_frequency = dict() limited_tokens_frequency[self.unk_token] = -1 - limited_tokens_frequency[self.pad_token] = -1 + limited_tokens_frequency[self.pad] = -1 for token in self.special_tokens: limited_tokens_frequency[token] = -1 limited_tokens_frequency.update( @@ -294,7 +294,7 @@ def encode_sentences(self, sentences, boundries=("", ""), out_length=None): encoded = self.encode(boundries[0] + " " + sent + " " + boundries[1]) encodings.append(encoded) - pad_id = self.encode(self.pad_token)[0] + pad_id = self.encode(self.pad)[0] # pad to equal size from https://stackoverflow.com/a/38619333 encodings = np.array( diff --git a/tkseem/word_tokenizer.py b/tkseem/word_tokenizer.py index b516150..9a9543c 100644 --- a/tkseem/word_tokenizer.py +++ b/tkseem/word_tokenizer.py @@ -8,6 +8,12 @@ class WordTokenizer(BaseTokenizer): tokens_frequency = None + def __init__( + self, vocab_size=10000, + ): + super(WordTokenizer, self).__init__(vocab_size=vocab_size) + self.name = "WordTokenizer" + def train(self, file_path): """Train using words' frequency @@ -34,7 +40,7 @@ def tokenize(self, text): if word in self.vocab.keys(): output_tokens.append(word) else: - output_tokens.append(self.unk_token) + output_tokens.append(self.unk) return output_tokens def detokenize(self, tokens):