dataFilter.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "9cb2449e",
   "metadata": {},
   "source": [
    "# Data processing for training.\n",
    "- Removing Null data\n",
    "- Split the variable name according to the naming convention\n",
    "- Creating certain number of sample data\n",
    "- Creating single file with all data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0410d695",
   "metadata": {},
   "source": [
    "## Importing necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "05b82a75",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import os\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "734d4bf4",
   "metadata": {},
   "source": [
    "## Constant datas"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "050c852a",
   "metadata": {},
   "outputs": [],
   "source": [
    "pathName = os.getcwd()+\"/Datas/\"\n",
    "fracSample = 0.001"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6c06a550",
   "metadata": {},
   "source": [
    "### Reading all the file in the given path and storing all the csv file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "cdd62a36",
   "metadata": {},
   "outputs": [],
   "source": [
    "allFileNames= os.listdir(pathName)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "18e8cf28",
   "metadata": {},
   "outputs": [],
   "source": [
    "numFiles=  []\n",
    "fileNames= []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "289d8d89",
   "metadata": {},
   "outputs": [],
   "source": [
    "for fileName in allFileNames:\n",
    "    if fileName.endswith(\".csv\"):\n",
    "        numFiles.append(pathName+fileName)\n",
    "        fileNames.append(fileName)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8f1c233",
   "metadata": {},
   "source": [
    "## Looping over csv file to remove all the null values and overwrite it"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5c01f6fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "datas=pd.DataFrame()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "0d557c69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available Files are: ['Go.csv', 'java.csv', 'JS.csv', 'php.csv', 'python.csv', 'Ruby.csv']\n"
     ]
    }
   ],
   "source": [
    "print(f'Available Files are: {fileNames}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "7fe09566",
   "metadata": {},
   "outputs": [],
   "source": [
    "x=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1e5d903f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Starting Data processing ...\n",
      "Done Processing c:\\Users\\sunny\\Desktop\\prabigya\\prabigya-datasets/Datas/Go.csv\n",
      "Done Processing c:\\Users\\sunny\\Desktop\\prabigya\\prabigya-datasets/Datas/java.csv\n",
      "Done Processing c:\\Users\\sunny\\Desktop\\prabigya\\prabigya-datasets/Datas/JS.csv\n",
      "Done Processing c:\\Users\\sunny\\Desktop\\prabigya\\prabigya-datasets/Datas/php.csv\n",
      "Done Processing c:\\Users\\sunny\\Desktop\\prabigya\\prabigya-datasets/Datas/python.csv\n",
      "Done Processing c:\\Users\\sunny\\Desktop\\prabigya\\prabigya-datasets/Datas/Ruby.csv\n"
     ]
    }
   ],
   "source": [
    "print(\"Starting Data processing ...\")\n",
    "for file in numFiles:\n",
    "    data= pd.read_csv(file,keep_default_na=False)\n",
    "    x = x+data.shape[0]\n",
    "    datas = pd.concat([datas,data],axis=0)\n",
    "    print(f\"Done Processing {file}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "32611cc9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total number of data: 2411672\n"
     ]
    }
   ],
   "source": [
    "print(f\"Total number of data: {datas.shape[0]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "e780898b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>docstring</th>\n",
       "      <th>variableName</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>waits up to 3-second until connection is up (p...</td>\n",
       "      <td>mustWaitPinReady</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>returns the cobra command for \"gateway\".</td>\n",
       "      <td>newGatewayCommand</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>returns a reader that reads from the given rea...</td>\n",
       "      <td>NewLimitedBufferReader</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>specifies the context for permanently aborting...</td>\n",
       "      <td>WithAbortContext</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>is a hint to prefetch a list of keys before tr...</td>\n",
       "      <td>WithPrefetch</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           docstring            variableName\n",
       "0  waits up to 3-second until connection is up (p...        mustWaitPinReady\n",
       "1           returns the cobra command for \"gateway\".       newGatewayCommand\n",
       "2  returns a reader that reads from the given rea...  NewLimitedBufferReader\n",
       "3  specifies the context for permanently aborting...        WithAbortContext\n",
       "4  is a hint to prefetch a list of keys before tr...            WithPrefetch"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datas.rename(columns={\"Revised\":\"variableName\"},inplace=True)\n",
    "datas.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "97e891ac",
   "metadata": {},
   "source": [
    "# Spliting the variable name to give better naming convetion while training"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "70e7cf69",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def split_naming_convention(name):\n",
    "    convention = \"None\"\n",
    "    if re.match(r'^[a-z]+(?:[A-Z][a-z]*)*$', name):\n",
    "        # Camel case\n",
    "        convention= \"camel\"\n",
    "        words = re.findall(r'[a-z]+|[A-Z][a-z]*', name)\n",
    "    elif re.match(r'^[A-Z][a-z]*(?:[A-Z][a-z]*)*$', name):\n",
    "        # Pascal case\n",
    "        convention = \"pascal\"\n",
    "        words = re.findall(r'[A-Z][a-z]*', name)\n",
    "    elif re.match(r'^[a-z]+(?:_[a-z]+)*$', name):\n",
    "        # Snake case\n",
    "        convention = \"snake\"\n",
    "        words = name.split('_')\n",
    "    elif re.match(r'^[a-z]+(?:-[a-z]+)*$', name):\n",
    "        # Kebab case\n",
    "        convention = \"kebab\"\n",
    "        words = name.split('-')\n",
    "    elif re.match(r'^[a-z]+[A-Z][a-zA-Z]*$', name):\n",
    "        # CamelSnake case\n",
    "        convention = \"CamelSnake\"\n",
    "        words = re.findall(r'[a-zA-Z][^A-Z]*', name)\n",
    "        words = re.findall(r'[a-z]+|[A-Z][a-z]*', name)\n",
    "    elif re.match(r\"(\\D+)(\\d+)?(\\D*)$\", name):\n",
    "        # with numbers\n",
    "        convention =\"numberic\"\n",
    "        words = re.findall(r'[a-zA-Z]+|\\d+', name)\n",
    "    elif re.match(r'^[a-z]+_[a-zA-Z]+$', name):\n",
    "        # Hungarian notation\n",
    "        convention = \"hungerian\"\n",
    "        words = name.split('_')\n",
    "    elif re.match(r'^[A-Z_]+$', name):\n",
    "        # Upper case\n",
    "        convention = \"upper\"\n",
    "        words = name.split('_')\n",
    "    elif re.match(r'^[a-z_]+$', name):\n",
    "        # Lower case\n",
    "        convention = \"lower\"\n",
    "        words = name.split('_')\n",
    "    elif name.__contains__(\"_\"):\n",
    "        # with underscores\n",
    "        words =name.split(\"_\")\n",
    "    else:\n",
    "        # Unknown naming convention\n",
    "        words = [name]\n",
    "    \n",
    "    words = [word.lower() for word in words]\n",
    "    return ' '.join(words),convention\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "7bcf49c9",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 46213/2411672 [07:01<4:17:35, 153.05it/s]"
     ]
    }
   ],
   "source": [
    "datas[\"seperatedVariableName\"] =''\n",
    "for i in tqdm(range(datas.shape[0])):\n",
    "    datas[\"seperatedVariableName\"][i]=(split_naming_convention(datas[\"variableName\"].values[i])[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fbb26888",
   "metadata": {},
   "source": [
    "## Converting data into csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7d641615",
   "metadata": {},
   "outputs": [],
   "source": [
    "datas.to_csv(\"train.csv\",index = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7594972d",
   "metadata": {},
   "source": [
    "#### Taking sample as fraction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "32111347",
   "metadata": {},
   "outputs": [],
   "source": [
    "smallSample = datas.sample(frac=fracSample)\n",
    "smallSample.to_csv(\"sampleTrain.csv\", index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a18134fc",
   "metadata": {},
   "source": [
    "## Displaying sample data"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8db66934",
   "metadata": {},
   "source": [
    "#### Total Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "abef9f15",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of datas: 2412\n",
      "                                           docstring                  Revised  \\\n",
      "0  <!-- begin-user-doc -->\\n<!-- end-user-doc -->...  getIfcProfileProperties   \n",
      "1  Use the thread context class loader to resolve...             resolveClass   \n",
      "2  Create dialog to choose a file to unpack\\n    ...            on_btn_unpack   \n",
      "3  Create a not in expression\\n@param propertyNam...                    notIn   \n",
      "4  Finds a method on the given type for the given...        findMethodsByName   \n",
      "\n",
      "                 variableName  \n",
      "0  get ifc profile properties  \n",
      "1               resolve class  \n",
      "2               on btn unpack  \n",
      "3                      not in  \n",
      "4        find methods by name  \n"
     ]
    }
   ],
   "source": [
    "print(\"Number of datas: {shape}\".format(shape=pd.read_csv(\"train.csv\",keep_default_na=False).shape[0]))\n",
    "train_csv_df = pd.read_csv(\"train.csv\")\n",
    "print(train_csv_df.head())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "05994165",
   "metadata": {},
   "source": [
    "#### Sample data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5e9d5705",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of datas: 2\n",
      "                                           docstring  \\\n",
      "0  Returns all nodes before the given $referenceN...   \n",
      "1  Method sessionAttributeAccessed\\n<p>\\n\\n@see c...   \n",
      "\n",
      "                    Revised                variableName  \n",
      "0               previousAll                previous all  \n",
      "1  sessionAttributeAccessed  session attribute accessed  \n"
     ]
    }
   ],
   "source": [
    "print(\"Number of datas: {shape}\".format(shape=pd.read_csv(\"sampleTrain.csv\",keep_default_na=False).shape[0]))\n",
    "print(pd.read_csv(\"sampleTrain.csv\").head(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d45a662d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d584476",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>docstring</th>\n",
       "      <th>Revised</th>\n",
       "      <th>variableName</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Returns all nodes before the given $referenceN...</td>\n",
       "      <td>previousAll</td>\n",
       "      <td>previous all</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Method sessionAttributeAccessed\\n&lt;p&gt;\\n\\n@see c...</td>\n",
       "      <td>sessionAttributeAccessed</td>\n",
       "      <td>session attribute accessed</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           docstring  \\\n",
       "0  Returns all nodes before the given $referenceN...   \n",
       "1  Method sessionAttributeAccessed\\n<p>\\n\\n@see c...   \n",
       "\n",
       "                    Revised                variableName  \n",
       "0               previousAll                previous all  \n",
       "1  sessionAttributeAccessed  session attribute accessed  "
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"sampleTrain.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ad74501",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Calls the associated hook function.\\n@param defaultScheduler the hook's input value\\n@return the value returned by the hook\""
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data[\"docstring\"][350]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d36e3657",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('merge 4 d', 'numberic')"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "split_naming_convention(\"merge4d\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "617b23c4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n"
     ]
    }
   ],
   "source": [
    "print(re.match(r'^[0-9]+$',\"merge4d\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c9a8858a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['hello123world']\n"
     ]
    }
   ],
   "source": [
    "string = \"merge423d\"\n",
    "pattern = r\"(\\D+)(\\d+)?(\\D*)$\"\n",
    "re.match(r\"(\\D+)(\\d+)?(\\D*)$\", string)\n",
    "re.findall(r'[a-zA-Z0-9]+', string)\n",
    "s = \"hello123world\"\n",
    "substrings = re.findall(r'[a-zA-Z0-9]+', s)\n",
    "print(substrings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "115e4ec5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['merge', '423', 'd']\n"
     ]
    }
   ],
   "source": [
    "name = \"merge423d\"\n",
    "words = re.findall(r'[a-zA-Z]+|\\d+', name)\n",
    "\n",
    "print(words) # Output: ['merge', '4', 'd']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cb2256c9",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.2"
  },
  "vscode": {
   "interpreter": {
    "hash": "5238573367df39f7286bb46f9ff5f08f63a01a80960060ce41e3c79b190280fa"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}