TrannyCounter.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPRV8/2cK+ID2F5tedJLWMQ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/walkerjian/dev/blob/main/TrannyCounter.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PuL4HcDTMUxu",
        "outputId": "baa9cf90-42a4-4e77-c8b3-a0a8004d81cb"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting googletrans==4.0.0-rc1\n",
            "  Downloading googletrans-4.0.0rc1.tar.gz (20 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting httpx==0.13.3 (from googletrans==4.0.0-rc1)\n",
            "  Downloading httpx-0.13.3-py3-none-any.whl (55 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.1/55.1 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: certifi in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==4.0.0-rc1) (2024.2.2)\n",
            "Collecting hstspreload (from httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading hstspreload-2024.2.1-py3-none-any.whl (1.1 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m8.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from httpx==0.13.3->googletrans==4.0.0-rc1) (1.3.1)\n",
            "Collecting chardet==3.* (from httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading chardet-3.0.4-py2.py3-none-any.whl (133 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m133.4/133.4 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting idna==2.* (from httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading idna-2.10-py2.py3-none-any.whl (58 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.8/58.8 kB\u001b[0m \u001b[31m4.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting rfc3986<2,>=1.3 (from httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading rfc3986-1.5.0-py2.py3-none-any.whl (31 kB)\n",
            "Collecting httpcore==0.9.* (from httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading httpcore-0.9.1-py3-none-any.whl (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.6/42.6 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h11<0.10,>=0.8 (from httpcore==0.9.*->httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading h11-0.9.0-py2.py3-none-any.whl (53 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m53.6/53.6 kB\u001b[0m \u001b[31m5.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting h2==3.* (from httpcore==0.9.*->httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading h2-3.2.0-py2.py3-none-any.whl (65 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m65.0/65.0 kB\u001b[0m \u001b[31m8.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hCollecting hyperframe<6,>=5.2.0 (from h2==3.*->httpcore==0.9.*->httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading hyperframe-5.2.0-py2.py3-none-any.whl (12 kB)\n",
            "Collecting hpack<4,>=3.0 (from h2==3.*->httpcore==0.9.*->httpx==0.13.3->googletrans==4.0.0-rc1)\n",
            "  Downloading hpack-3.0.0-py2.py3-none-any.whl (38 kB)\n",
            "Building wheels for collected packages: googletrans\n",
            "  Building wheel for googletrans (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for googletrans: filename=googletrans-4.0.0rc1-py3-none-any.whl size=17396 sha256=e03136d350c628f88dd4e97bdfe8cc5f5a19d15aa56a46780d3264840e9607db\n",
            "  Stored in directory: /root/.cache/pip/wheels/c0/59/9f/7372f0cf70160fe61b528532e1a7c8498c4becd6bcffb022de\n",
            "Successfully built googletrans\n",
            "Installing collected packages: rfc3986, hyperframe, hpack, h11, chardet, idna, hstspreload, h2, httpcore, httpx, googletrans\n",
            "  Attempting uninstall: chardet\n",
            "    Found existing installation: chardet 5.2.0\n",
            "    Uninstalling chardet-5.2.0:\n",
            "      Successfully uninstalled chardet-5.2.0\n",
            "  Attempting uninstall: idna\n",
            "    Found existing installation: idna 3.6\n",
            "    Uninstalling idna-3.6:\n",
            "      Successfully uninstalled idna-3.6\n",
            "Successfully installed chardet-3.0.4 googletrans-4.0.0rc1 h11-0.9.0 h2-3.2.0 hpack-3.0.0 hstspreload-2024.2.1 httpcore-0.9.1 httpx-0.13.3 hyperframe-5.2.0 idna-2.10 rfc3986-1.5.0\n"
          ]
        }
      ],
      "source": [
        "!pip install googletrans==4.0.0-rc1\n"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from googletrans import Translator, LANGUAGES\n",
        "\n",
        "# Initialize the Translator\n",
        "translator = Translator()\n",
        "\n",
        "# The text to be translated\n",
        "text = \"Merci pour votre question ingénieuse! En tant qu'assistant AI, je suis toujours prêt à aider et à rendre nos conversations agréables. Alors, comment puis-je vous aider aujourd'hui? Et n'oubliez pas de me dire comment vous allez!\"\n",
        "\n",
        "# Function to translate text to a specified language and calculate word count\n",
        "def translate_and_measure(text, dest_language):\n",
        "    translation = translator.translate(text, dest=dest_language)\n",
        "    word_count = len(translation.text.split())\n",
        "    return LANGUAGES[dest_language], translation.text, word_count\n",
        "\n",
        "# Languages to translate the text into\n",
        "languages = ['en', 'de', 'es', 'ru', 'zh-cn', 'ar', 'ja', 'ko', 'hi']\n",
        "\n",
        "# Translate and measure\n",
        "translations = [translate_and_measure(text, lang) for lang in languages]\n",
        "\n",
        "for lang_name, translated_text, count in translations:\n",
        "    print(f\"{lang_name.title()} ({count} words): {translated_text}\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "gkqtbGyoMW-0",
        "outputId": "79553293-cfc6-474a-c823-5612061cc4f3"
      },
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English (34 words): Thank you for your ingenious question!As an AI assistant, I am always ready to help and make our conversations pleasant.So, how can I help you today?And don't forget to tell me how you are!\n",
            "\n",
            "German (35 words): Vielen Dank für Ihre geniale Frage!Als KI -Assistent bin ich immer bereit zu helfen und unsere Gespräche angenehm zu machen.Wie kann ich Ihnen heute helfen?Und vergessen Sie nicht, mir zu sagen, wie es Ihnen geht!\n",
            "\n",
            "Spanish (27 words): ¡Gracias por su ingeniosa pregunta!Como asistente de IA, siempre estoy listo para ayudar y hacer nuestras conversaciones agradables.Entonces, ¿cómo puedo ayudarte hoy?¡Y no olvides decirme cómo estás!\n",
            "\n",
            "Russian (28 words): Спасибо за ваш гениальный вопрос!Как помощник ИИ, я всегда готов помочь и сделать наши беседы приятными.Итак, как я могу вам помочь сегодня?И не забудьте рассказать мне, как ты!\n",
            "\n",
            "Chinese (Simplified) (1 words): 谢谢您的巧妙问题！作为AI助手，我随时准备帮助并使我们的对话令人愉快。那么，今天我该如何帮助您？而且不要忘了告诉我你好吗！\n",
            "\n",
            "Arabic (26 words): شكرا لك على سؤالك العبقري!كمساعد منظمة العفو الدولية ، أنا مستعد دائمًا للمساعدة وجعل محادثاتنا ممتعة.لذا ، كيف يمكنني مساعدتك اليوم؟ولا تنس أن تخبرني كيف أنت!\n",
            "\n",
            "Japanese (1 words): 独創的な質問をありがとう！AIアシスタントとして、私は常に私たちの会話を快適にする準備ができています。それで、今日はどうすればあなたを助けることができますか？そして、あなたがどうであるかを私に話すことを忘れないでください！\n",
            "\n",
            "Korean (21 words): 독창적 인 질문에 감사드립니다!AI 조수로서 저는 항상 대화를 도울 준비가되어 있습니다.그래서 오늘 어떻게 도와 드릴까요?그리고 당신이 어떻게되는지 말하는 것을 잊지 마십시오!\n",
            "\n",
            "Hindi (41 words): आपके सरल प्रश्न के लिए धन्यवाद!एआई सहायक के रूप में, मैं हमेशा मदद करने और अपनी बातचीत को सुखद बनाने के लिए तैयार हूं।तो, मैं आज आपकी मदद कैसे कर सकता हूं?और मुझे यह बताना मत भूलना कि आप कैसे हैं!\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from googletrans import Translator, LANGUAGES\n",
        "import re\n",
        "\n",
        "# Initialize the Translator\n",
        "translator = Translator()\n",
        "\n",
        "# The text to be translated\n",
        "text = \"Merci pour votre question ingénieuse! En tant qu'assistant AI, je suis toujours prêt à aider et à rendre nos conversations agréables. Alors, comment puis-je vous aider aujourd'hui? Et n'oubliez pas de me dire comment vous allez!\"\n",
        "\n",
        "# Function to translate text to a specified language and calculate word count, symbols, and character count\n",
        "def translate_and_measure(text, dest_language):\n",
        "    translation = translator.translate(text, dest=dest_language)\n",
        "    word_count = len(translation.text.split())\n",
        "    symbol_count = len(re.findall(r\"\\W\", translation.text))  # Counts non-alphanumeric characters (excluding spaces)\n",
        "    char_count = len(translation.text.replace(\" \", \"\"))  # Counts characters excluding spaces\n",
        "    return LANGUAGES[dest_language], translation.text, word_count, symbol_count, char_count\n",
        "\n",
        "# Languages to translate the text into\n",
        "languages = ['en', 'de', 'es', 'ru', 'zh-cn', 'ar', 'ja', 'ko', 'hi']\n",
        "\n",
        "# Translate and measure\n",
        "translations = [translate_and_measure(text, lang) for lang in languages]\n",
        "\n",
        "for lang_name, translated_text, word_count, symbol_count, char_count in translations:\n",
        "    print(f\"{lang_name.title()} - Words: {word_count}, Symbols: {symbol_count}, Characters (no spaces): {char_count}\")\n",
        "    print(f\"Translation: {translated_text}\\n\")\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8Ic941UFM7Ed",
        "outputId": "3e9411bd-138b-4d0e-8c00-a09b7161996c"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "English - Words: 34, Symbols: 40, Characters (no spaces): 156\n",
            "Translation: Thank you for your ingenious question!As an AI assistant, I am always ready to help and make our conversations pleasant.So, how can I help you today?And don't forget to tell me how you are!\n",
            "\n",
            "German - Words: 35, Symbols: 41, Characters (no spaces): 179\n",
            "Translation: Vielen Dank für Ihre geniale Frage!Als KI -Assistent bin ich immer bereit zu helfen und unsere Gespräche angenehm zu machen.Wie kann ich Ihnen heute helfen?Und vergessen Sie nicht, mir zu sagen, wie es Ihnen geht!\n",
            "\n",
            "Spanish - Words: 27, Symbols: 35, Characters (no spaces): 174\n",
            "Translation: ¡Gracias por su ingeniosa pregunta!Como asistente de IA, siempre estoy listo para ayudar y hacer nuestras conversaciones agradables.Entonces, ¿cómo puedo ayudarte hoy?¡Y no olvides decirme cómo estás!\n",
            "\n",
            "Russian - Words: 28, Symbols: 34, Characters (no spaces): 150\n",
            "Translation: Спасибо за ваш гениальный вопрос!Как помощник ИИ, я всегда готов помочь и сделать наши беседы приятными.Итак, как я могу вам помочь сегодня?И не забудьте рассказать мне, как ты!\n",
            "\n",
            "Chinese (Simplified) - Words: 1, Symbols: 6, Characters (no spaces): 61\n",
            "Translation: 谢谢您的巧妙问题！作为AI助手，我随时准备帮助并使我们的对话令人愉快。那么，今天我该如何帮助您？而且不要忘了告诉我你好吗！\n",
            "\n",
            "Arabic - Words: 26, Symbols: 32, Characters (no spaces): 134\n",
            "Translation: شكرا لك على سؤالك العبقري!كمساعد منظمة العفو الدولية ، أنا مستعد دائمًا للمساعدة وجعل محادثاتنا ممتعة.لذا ، كيف يمكنني مساعدتك اليوم؟ولا تنس أن تخبرني كيف أنت!\n",
            "\n",
            "Japanese - Words: 1, Symbols: 7, Characters (no spaces): 111\n",
            "Translation: 独創的な質問をありがとう！AIアシスタントとして、私は常に私たちの会話を快適にする準備ができています。それで、今日はどうすればあなたを助けることができますか？そして、あなたがどうであるかを私に話すことを忘れないでください！\n",
            "\n",
            "Korean - Words: 21, Symbols: 24, Characters (no spaces): 78\n",
            "Translation: 독창적 인 질문에 감사드립니다!AI 조수로서 저는 항상 대화를 도울 준비가되어 있습니다.그래서 오늘 어떻게 도와 드릴까요?그리고 당신이 어떻게되는지 말하는 것을 잊지 마십시오!\n",
            "\n",
            "Hindi - Words: 41, Symbols: 96, Characters (no spaces): 156\n",
            "Translation: आपके सरल प्रश्न के लिए धन्यवाद!एआई सहायक के रूप में, मैं हमेशा मदद करने और अपनी बातचीत को सुखद बनाने के लिए तैयार हूं।तो, मैं आज आपकी मदद कैसे कर सकता हूं?और मुझे यह बताना मत भूलना कि आप कैसे हैं!\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import re\n",
        "import pandas as pd\n",
        "\n",
        "# Provided text\n",
        "text = \"Thank you for your ingenious question! As an AI assistant, I am always ready to help and make our conversations pleasant. So, how can I help you today? And don't forget to tell me how you are!\"\n",
        "\n",
        "# Regular expressions to match words, symbols (excluding spaces), and characters (excluding spaces)\n",
        "words = re.findall(r'\\b\\w+\\b', text)\n",
        "symbols = re.findall(r'\\W', text)  # Captures all non-alphanumeric characters, excluding spaces\n",
        "characters = [char for char in text.replace(\" \", \"\")]  # List of characters, excluding spaces\n",
        "\n",
        "# Create sets of distinct words, symbols, and characters\n",
        "distinct_words = set(words)\n",
        "distinct_symbols = set(symbols)\n",
        "distinct_characters = set(characters)\n",
        "\n",
        "# Convert sets to lists for DataFrame creation\n",
        "list_distinct_words = list(distinct_words)\n",
        "list_distinct_symbols = list(distinct_symbols)\n",
        "list_distinct_characters = list(distinct_characters)\n",
        "\n",
        "# Sorting lists\n",
        "list_distinct_words.sort()\n",
        "list_distinct_symbols.sort(key=lambda x: ord(x))  # Sort symbols by their Unicode code point\n",
        "list_distinct_characters.sort(key=lambda x: ord(x))\n",
        "\n",
        "# Creating DataFrames\n",
        "df_words = pd.DataFrame(list_distinct_words, columns=['Distinct Words'])\n",
        "df_symbols = pd.DataFrame(list_distinct_symbols, columns=['Distinct Symbols'])\n",
        "df_chars = pd.DataFrame(list_distinct_characters, columns=['Distinct Characters'])\n",
        "\n",
        "df_words, df_symbols, df_chars\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "PZ7K6wxYM9D0",
        "outputId": "ee4e51ed-07d1-4ab1-e603-29f4e0cbe2a7"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(   Distinct Words\n",
              " 0              AI\n",
              " 1             And\n",
              " 2              As\n",
              " 3               I\n",
              " 4              So\n",
              " 5           Thank\n",
              " 6          always\n",
              " 7              am\n",
              " 8              an\n",
              " 9             and\n",
              " 10            are\n",
              " 11      assistant\n",
              " 12            can\n",
              " 13  conversations\n",
              " 14            don\n",
              " 15            for\n",
              " 16         forget\n",
              " 17           help\n",
              " 18            how\n",
              " 19      ingenious\n",
              " 20           make\n",
              " 21             me\n",
              " 22            our\n",
              " 23       pleasant\n",
              " 24       question\n",
              " 25          ready\n",
              " 26              t\n",
              " 27           tell\n",
              " 28             to\n",
              " 29          today\n",
              " 30            you\n",
              " 31           your,\n",
              "   Distinct Symbols\n",
              " 0                 \n",
              " 1                !\n",
              " 2                '\n",
              " 3                ,\n",
              " 4                .\n",
              " 5                ?,\n",
              "    Distinct Characters\n",
              " 0                    !\n",
              " 1                    '\n",
              " 2                    ,\n",
              " 3                    .\n",
              " 4                    ?\n",
              " 5                    A\n",
              " 6                    I\n",
              " 7                    S\n",
              " 8                    T\n",
              " 9                    a\n",
              " 10                   c\n",
              " 11                   d\n",
              " 12                   e\n",
              " 13                   f\n",
              " 14                   g\n",
              " 15                   h\n",
              " 16                   i\n",
              " 17                   k\n",
              " 18                   l\n",
              " 19                   m\n",
              " 20                   n\n",
              " 21                   o\n",
              " 22                   p\n",
              " 23                   q\n",
              " 24                   r\n",
              " 25                   s\n",
              " 26                   t\n",
              " 27                   u\n",
              " 28                   v\n",
              " 29                   w\n",
              " 30                   y)"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import spacy\n",
        "\n",
        "# Load the English model\n",
        "nlp = spacy.load(\"en_core_web_sm\")\n",
        "\n",
        "# Process the text\n",
        "doc = nlp(\"Thank you for your ingenious question! As an AI assistant, I am always ready to help and make our conversations pleasant. So, how can I help you today? And don't forget to tell me how you are!\")\n",
        "\n",
        "# Extract words (tokens), symbols, and characters\n",
        "words = [token.text for token in doc if token.is_alpha]  # Words only\n",
        "symbols = [token.text for token in doc if not token.is_alpha and not token.is_space]  # Symbols, excluding spaces\n",
        "characters = [char for char in doc.text if not char.isspace()]  # Characters, excluding spaces\n",
        "\n",
        "# Convert lists to sets to get unique elements\n",
        "distinct_words = set(words)\n",
        "distinct_symbols = set(symbols)\n",
        "distinct_characters = set(characters)\n",
        "\n",
        "# Display the results\n",
        "distinct_words, distinct_symbols, distinct_characters\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fd2-jMsHPF_k",
        "outputId": "ead53ae8-b68a-4907-aabd-8b869fc9df6d"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "({'AI',\n",
              "  'And',\n",
              "  'As',\n",
              "  'I',\n",
              "  'So',\n",
              "  'Thank',\n",
              "  'always',\n",
              "  'am',\n",
              "  'an',\n",
              "  'and',\n",
              "  'are',\n",
              "  'assistant',\n",
              "  'can',\n",
              "  'conversations',\n",
              "  'do',\n",
              "  'for',\n",
              "  'forget',\n",
              "  'help',\n",
              "  'how',\n",
              "  'ingenious',\n",
              "  'make',\n",
              "  'me',\n",
              "  'our',\n",
              "  'pleasant',\n",
              "  'question',\n",
              "  'ready',\n",
              "  'tell',\n",
              "  'to',\n",
              "  'today',\n",
              "  'you',\n",
              "  'your'},\n",
              " {'!', ',', '.', '?', \"n't\"},\n",
              " {'!',\n",
              "  \"'\",\n",
              "  ',',\n",
              "  '.',\n",
              "  '?',\n",
              "  'A',\n",
              "  'I',\n",
              "  'S',\n",
              "  'T',\n",
              "  'a',\n",
              "  'c',\n",
              "  'd',\n",
              "  'e',\n",
              "  'f',\n",
              "  'g',\n",
              "  'h',\n",
              "  'i',\n",
              "  'k',\n",
              "  'l',\n",
              "  'm',\n",
              "  'n',\n",
              "  'o',\n",
              "  'p',\n",
              "  'q',\n",
              "  'r',\n",
              "  's',\n",
              "  't',\n",
              "  'u',\n",
              "  'v',\n",
              "  'w',\n",
              "  'y'})"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "# Download necessary NLTK data\n",
        "nltk.download(\"punkt\")\n",
        "\n",
        "# Process the text\n",
        "text = \"Thank you for your ingenious question! As an AI assistant, I am always ready to help and make our conversations pleasant. So, how can I help you today? And don't forget to tell me how you are!\"\n",
        "tokens = word_tokenize(text)\n",
        "\n",
        "# NLTK's word_tokenize does a decent job with contractions and punctuation\n",
        "distinct_tokens = set(tokens)\n",
        "\n",
        "distinct_tokens\n"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CopW9EkIPRR-",
        "outputId": "a6469585-5c69-46d9-d73d-35c1fbe691d4"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'!',\n",
              " ',',\n",
              " '.',\n",
              " '?',\n",
              " 'AI',\n",
              " 'And',\n",
              " 'As',\n",
              " 'I',\n",
              " 'So',\n",
              " 'Thank',\n",
              " 'always',\n",
              " 'am',\n",
              " 'an',\n",
              " 'and',\n",
              " 'are',\n",
              " 'assistant',\n",
              " 'can',\n",
              " 'conversations',\n",
              " 'do',\n",
              " 'for',\n",
              " 'forget',\n",
              " 'help',\n",
              " 'how',\n",
              " 'ingenious',\n",
              " 'make',\n",
              " 'me',\n",
              " \"n't\",\n",
              " 'our',\n",
              " 'pleasant',\n",
              " 'question',\n",
              " 'ready',\n",
              " 'tell',\n",
              " 'to',\n",
              " 'today',\n",
              " 'you',\n",
              " 'your'}"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "contractions = {\n",
        "    \"don't\": \"do not\",\n",
        "    \"isn't\": \"is not\",\n",
        "    # Add more contractions as needed\n",
        "}\n",
        "\n",
        "def expand_contractions(text, contraction_mapping):\n",
        "    contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())),\n",
        "                                      flags=re.IGNORECASE|re.DOTALL)\n",
        "    def expand_match(contraction):\n",
        "        match = contraction.group(0)\n",
        "        first_char = match[0]\n",
        "        expanded_contraction = contraction_mapping.get(match)\\\n",
        "                               if contraction_mapping.get(match)\\\n",
        "                               else contraction_mapping.get(match.lower())\n",
        "        expanded_contraction = first_char+expanded_contraction[1:]\n",
        "        return expanded_contraction\n",
        "\n",
        "    expanded_text = contractions_pattern.sub(expand_match, text)\n",
        "    expanded_text = re.sub(\"'\", \"\", expanded_text)\n",
        "    return expanded_text\n",
        "\n",
        "# Example usage\n",
        "text = \"Thank you for your ingenious question! As an AI assistant, I am always ready to help and make our conversations pleasant. So, how can I help you today? And don't forget to tell me how you are!\"\n",
        "expanded_text = expand_contractions(text, contractions)\n",
        "\n",
        "# Now, you can proceed with tokenization using NLTK or SpaCy on the expanded_text.\n"
      ],
      "metadata": {
        "id": "UKwaCmd_Pp8H"
      },
      "execution_count": 7,
      "outputs": []
    }
  ]
}