Filter Methods.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.read_csv('data/housing.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Constant Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "x_train, x_test, y_train, y_test = train_test_split(data.drop(['SalePrice'], axis=1), \n",
    "                                                    data.SalePrice, test_size=0.3, \n",
    "                                                    random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "numerical_x_train = x_train[x_train.select_dtypes([np.number]).columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold\n",
    "vs_constant = VarianceThreshold(threshold=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "vs_constant.fit(numerical_x_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(x_train[x_train.select_dtypes([np.number]).columns].columns[vs_constant.get_support()])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "constant_columns = [column for column in numerical_x_train.columns\n",
    "                    if column not in numerical_x_train.columns[vs_constant.get_support()]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(constant_columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "constant_cat_columns = [column for column in x_train.columns \n",
    "                        if (x_train[column].dtype == \"O\" and len(x_train[column].unique())  == 1 )]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_constant_columns = constant_cat_columns + constant_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train.drop(labels=constant_columns, axis=1, inplace=True)\n",
    "x_test.drop(labels=constant_columns, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Quasi Constant features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "threshold = 0.98\n",
    "\n",
    "# create empty list\n",
    "quasi_constant_feature = []\n",
    "\n",
    "# loop over all the columns\n",
    "for feature in x_train.columns:\n",
    "\n",
    "    # calculate the ratio.\n",
    "    predominant = (x_train[feature].value_counts() / np.float(len(x_train))).sort_values(ascending=False).values[0]\n",
    "    \n",
    "    # append the column name if it is bigger than the threshold\n",
    "    if predominant >= threshold:\n",
    "        quasi_constant_feature.append(feature)   \n",
    "        \n",
    "print(quasi_constant_feature)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop the quasi constant columns\n",
    "x_train.drop(labels=quasi_constant_feature, axis=1, inplace=True)\n",
    "x_test.drop(labels=quasi_constant_feature, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Duplicated Features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_features_T = x_train.T\n",
    "train_features_T.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(train_features_T.duplicated().sum())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "duplicated_columns = train_features_T[train_features_T.duplicated()].index.values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train.drop(labels=duplicated_columns, axis=1, inplace=True)\n",
    "x_test.drop(labels=duplicated_columns, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Correlation methods"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "correlated_features = set()\n",
    "correlation_matrix = x_train.corr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "plt.figure(figsize=(11,11))\n",
    "sns.heatmap(correlation_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in range(len(correlation_matrix .columns)):\n",
    "    for j in range(i):\n",
    "        if abs(correlation_matrix.iloc[i, j]) > 0.8:\n",
    "            colname = correlation_matrix.columns[i]\n",
    "            correlated_features.add(colname)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "correlated_features"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train.drop(labels=correlated_features, axis=1, inplace=True)\n",
    "x_test.drop(labels=correlated_features, axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Statistical Measures"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import mutual_info_classif, chi2\n",
    "from sklearn.feature_selection import SelectKBest, SelectPercentile\n",
    "from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor\n",
    "from sklearn.metrics import roc_auc_score, mean_squared_error"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Mutual Information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# select the number of features you want to retain.\n",
    "select_k = 10\n",
    "\n",
    "# get only the numerical features.\n",
    "numerical_x_train = x_train[x_train.select_dtypes([np.number]).columns]\n",
    "\n",
    "\n",
    "# create the SelectKBest with the mutual info strategy.\n",
    "selection = SelectKBest(mutual_info_classif, k=select_k).fit(numerical_x_train, y_train)\n",
    "\n",
    "# display the retained features.\n",
    "features = x_train.columns[selection.get_support()]\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "mutual_info(x_train, y_train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Chi Squared Score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# change this to how much features you want to keep from the top ones.\n",
    "select_k = 10\n",
    "\n",
    "# apply the chi2 score on the data and target (target should be binary).  \n",
    "selection = SelectKBest(chi2, k=select_k).fit(x_train, y_train)\n",
    "\n",
    "# display the k selected features.\n",
    "features = x_train.columns[selection.get_support()]\n",
    "print(features)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Univariate RMSE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def univariate_rmse():\n",
    "    mse_values = []\n",
    "    for feature in x_train.columns:\n",
    "        clf = DecisionTreeRegressor()\n",
    "        clf.fit(x_train[feature].to_frame(), y_train)\n",
    "        y_scored = clf.predict(x_test[feature].to_frame())\n",
    "        mse_values.append(mean_squared_error(y_test, y_scored))\n",
    "    mse_values = pd.Series(mse_values)\n",
    "    mse_values.index = x_train.columns\n",
    "    print(mse_values.sort_values(ascending=False))\n",
    "    print(len(mse_values[mse_values > threshold]),'out of the %s featues are kept'% len(x_train.columns))\n",
    "    keep_col = mse_values[mse_values > threshold]\n",
    "    return keep_col   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "univariate_rmse()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Univariate ROC-AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def univariate_roc_auc():\n",
    "    roc_values = []\n",
    "    for feature in x_train.columns:\n",
    "        clf = DecisionTreeClassifier()\n",
    "        clf.fit(x_train[feature].to_frame(), y_train)\n",
    "        y_scored = clf.predict_proba(x_test[feature].to_frame())\n",
    "        roc_values.append(roc_auc_score(y_test, y_scored[:, 1]))\n",
    "    roc_values = pd.Series(roc_values)\n",
    "    roc_values.index = X_train.columns\n",
    "    print(roc_values.sort_values(ascending=False))\n",
    "    print(len(roc_values[roc_values > threshold]),'out of the %s featues are kept'% len(X_train.columns))\n",
    "    keep_col = roc_values[roc_values > threshold]\n",
    "    return keep_col"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use it for a classification task.\n",
    "# univariate_roc_auc()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}