Week3DataCleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "72e8b159",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "eb87a0f1",
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = '/Users/emilyflores/Desktop/Work/OAFall23/Data-NotebookEF/twcs.csv'\n",
    "dataset = pd.read_csv('/Users/emilyflores/Desktop/Work/OAFall23/Data-NotebookEF/twcs.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "31e84bf5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>2811769</th>\n",
       "      <td>2987947</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Nov 22 08:43:51 +0000 2017</td>\n",
       "      <td>@823869 Hey, we'd be happy to look into this f...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2987948.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811770</th>\n",
       "      <td>2987948</td>\n",
       "      <td>823869</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Nov 22 08:35:16 +0000 2017</td>\n",
       "      <td>@115714 wtf!? I’ve been having really shitty s...</td>\n",
       "      <td>2987947</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811771</th>\n",
       "      <td>2812240</td>\n",
       "      <td>121673</td>\n",
       "      <td>True</td>\n",
       "      <td>Thu Nov 23 04:13:07 +0000 2017</td>\n",
       "      <td>@143549 @sprintcare You have to go to https://...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2812239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811772</th>\n",
       "      <td>2987949</td>\n",
       "      <td>AldiUK</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Nov 22 08:31:24 +0000 2017</td>\n",
       "      <td>@823870 Sounds delicious, Sarah! 😋 https://t.c...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2987950.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811773</th>\n",
       "      <td>2987950</td>\n",
       "      <td>823870</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Nov 21 22:01:04 +0000 2017</td>\n",
       "      <td>@AldiUK  warm sloe gin mince pies with ice cre...</td>\n",
       "      <td>2987951,2987949</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         tweet_id   author_id  inbound                      created_at  \\\n",
       "2811769   2987947  sprintcare    False  Wed Nov 22 08:43:51 +0000 2017   \n",
       "2811770   2987948      823869     True  Wed Nov 22 08:35:16 +0000 2017   \n",
       "2811771   2812240      121673     True  Thu Nov 23 04:13:07 +0000 2017   \n",
       "2811772   2987949      AldiUK    False  Wed Nov 22 08:31:24 +0000 2017   \n",
       "2811773   2987950      823870     True  Tue Nov 21 22:01:04 +0000 2017   \n",
       "\n",
       "                                                      text response_tweet_id  \\\n",
       "2811769  @823869 Hey, we'd be happy to look into this f...               NaN   \n",
       "2811770  @115714 wtf!? I’ve been having really shitty s...           2987947   \n",
       "2811771  @143549 @sprintcare You have to go to https://...               NaN   \n",
       "2811772  @823870 Sounds delicious, Sarah! 😋 https://t.c...               NaN   \n",
       "2811773  @AldiUK  warm sloe gin mince pies with ice cre...   2987951,2987949   \n",
       "\n",
       "         in_response_to_tweet_id  \n",
       "2811769                2987948.0  \n",
       "2811770                      NaN  \n",
       "2811771                2812239.0  \n",
       "2811772                2987950.0  \n",
       "2811773                      NaN  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "bb53a477",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811769</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811770</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811771</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811772</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811773</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2811774 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         tweet_id  author_id  inbound  created_at   text  response_tweet_id  \\\n",
       "0           False      False    False       False  False              False   \n",
       "1           False      False    False       False  False               True   \n",
       "2           False      False    False       False  False              False   \n",
       "3           False      False    False       False  False              False   \n",
       "4           False      False    False       False  False              False   \n",
       "...           ...        ...      ...         ...    ...                ...   \n",
       "2811769     False      False    False       False  False               True   \n",
       "2811770     False      False    False       False  False              False   \n",
       "2811771     False      False    False       False  False               True   \n",
       "2811772     False      False    False       False  False               True   \n",
       "2811773     False      False    False       False  False              False   \n",
       "\n",
       "         in_response_to_tweet_id  \n",
       "0                          False  \n",
       "1                          False  \n",
       "2                          False  \n",
       "3                          False  \n",
       "4                          False  \n",
       "...                          ...  \n",
       "2811769                    False  \n",
       "2811770                     True  \n",
       "2811771                    False  \n",
       "2811772                    False  \n",
       "2811773                     True  \n",
       "\n",
       "[2811774 rows x 7 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isnull()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "a8d48d41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811769</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811770</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811771</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811772</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811773</th>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2811774 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         tweet_id  author_id  inbound  created_at   text  response_tweet_id  \\\n",
       "0           False      False    False       False  False              False   \n",
       "1           False      False    False       False  False               True   \n",
       "2           False      False    False       False  False              False   \n",
       "3           False      False    False       False  False              False   \n",
       "4           False      False    False       False  False              False   \n",
       "...           ...        ...      ...         ...    ...                ...   \n",
       "2811769     False      False    False       False  False               True   \n",
       "2811770     False      False    False       False  False              False   \n",
       "2811771     False      False    False       False  False               True   \n",
       "2811772     False      False    False       False  False               True   \n",
       "2811773     False      False    False       False  False              False   \n",
       "\n",
       "         in_response_to_tweet_id  \n",
       "0                          False  \n",
       "1                          False  \n",
       "2                          False  \n",
       "3                          False  \n",
       "4                          False  \n",
       "...                          ...  \n",
       "2811769                    False  \n",
       "2811770                     True  \n",
       "2811771                    False  \n",
       "2811772                    False  \n",
       "2811773                     True  \n",
       "\n",
       "[2811774 rows x 7 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "c1d16886",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tweet_id                   False\n",
       "author_id                  False\n",
       "inbound                    False\n",
       "created_at                 False\n",
       "text                       False\n",
       "response_tweet_id           True\n",
       "in_response_to_tweet_id     True\n",
       "dtype: bool"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isna().any()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1894c1e8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "tweet_id                         0\n",
       "author_id                        0\n",
       "inbound                          0\n",
       "created_at                       0\n",
       "text                             0\n",
       "response_tweet_id          1040629\n",
       "in_response_to_tweet_id     794335\n",
       "dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isna().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "98686a25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.isna().any().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "e235fb8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0          False\n",
       "1          False\n",
       "2          False\n",
       "3          False\n",
       "4          False\n",
       "           ...  \n",
       "2811769    False\n",
       "2811770    False\n",
       "2811771    False\n",
       "2811772    False\n",
       "2811773    False\n",
       "Length: 2811774, dtype: bool"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.duplicated()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "108a8111",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>tweet_id</th>\n",
       "      <th>author_id</th>\n",
       "      <th>inbound</th>\n",
       "      <th>created_at</th>\n",
       "      <th>text</th>\n",
       "      <th>response_tweet_id</th>\n",
       "      <th>in_response_to_tweet_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 22:10:47 +0000 2017</td>\n",
       "      <td>@115712 I understand. I would like to assist y...</td>\n",
       "      <td>2</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 22:11:45 +0000 2017</td>\n",
       "      <td>@sprintcare and how do you propose we do that</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 22:08:27 +0000 2017</td>\n",
       "      <td>@sprintcare I have sent several private messag...</td>\n",
       "      <td>1</td>\n",
       "      <td>4.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Tue Oct 31 21:54:49 +0000 2017</td>\n",
       "      <td>@115712 Please send us a Private Message so th...</td>\n",
       "      <td>3</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>115712</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Oct 31 21:49:35 +0000 2017</td>\n",
       "      <td>@sprintcare I did.</td>\n",
       "      <td>4</td>\n",
       "      <td>6.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811769</th>\n",
       "      <td>2987947</td>\n",
       "      <td>sprintcare</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Nov 22 08:43:51 +0000 2017</td>\n",
       "      <td>@823869 Hey, we'd be happy to look into this f...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2987948.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811770</th>\n",
       "      <td>2987948</td>\n",
       "      <td>823869</td>\n",
       "      <td>True</td>\n",
       "      <td>Wed Nov 22 08:35:16 +0000 2017</td>\n",
       "      <td>@115714 wtf!? I’ve been having really shitty s...</td>\n",
       "      <td>2987947</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811771</th>\n",
       "      <td>2812240</td>\n",
       "      <td>121673</td>\n",
       "      <td>True</td>\n",
       "      <td>Thu Nov 23 04:13:07 +0000 2017</td>\n",
       "      <td>@143549 @sprintcare You have to go to https://...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2812239.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811772</th>\n",
       "      <td>2987949</td>\n",
       "      <td>AldiUK</td>\n",
       "      <td>False</td>\n",
       "      <td>Wed Nov 22 08:31:24 +0000 2017</td>\n",
       "      <td>@823870 Sounds delicious, Sarah! 😋 https://t.c...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2987950.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2811773</th>\n",
       "      <td>2987950</td>\n",
       "      <td>823870</td>\n",
       "      <td>True</td>\n",
       "      <td>Tue Nov 21 22:01:04 +0000 2017</td>\n",
       "      <td>@AldiUK  warm sloe gin mince pies with ice cre...</td>\n",
       "      <td>2987951,2987949</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>2811774 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         tweet_id   author_id  inbound                      created_at  \\\n",
       "0               1  sprintcare    False  Tue Oct 31 22:10:47 +0000 2017   \n",
       "1               2      115712     True  Tue Oct 31 22:11:45 +0000 2017   \n",
       "2               3      115712     True  Tue Oct 31 22:08:27 +0000 2017   \n",
       "3               4  sprintcare    False  Tue Oct 31 21:54:49 +0000 2017   \n",
       "4               5      115712     True  Tue Oct 31 21:49:35 +0000 2017   \n",
       "...           ...         ...      ...                             ...   \n",
       "2811769   2987947  sprintcare    False  Wed Nov 22 08:43:51 +0000 2017   \n",
       "2811770   2987948      823869     True  Wed Nov 22 08:35:16 +0000 2017   \n",
       "2811771   2812240      121673     True  Thu Nov 23 04:13:07 +0000 2017   \n",
       "2811772   2987949      AldiUK    False  Wed Nov 22 08:31:24 +0000 2017   \n",
       "2811773   2987950      823870     True  Tue Nov 21 22:01:04 +0000 2017   \n",
       "\n",
       "                                                      text response_tweet_id  \\\n",
       "0        @115712 I understand. I would like to assist y...                 2   \n",
       "1            @sprintcare and how do you propose we do that               NaN   \n",
       "2        @sprintcare I have sent several private messag...                 1   \n",
       "3        @115712 Please send us a Private Message so th...                 3   \n",
       "4                                       @sprintcare I did.                 4   \n",
       "...                                                    ...               ...   \n",
       "2811769  @823869 Hey, we'd be happy to look into this f...               NaN   \n",
       "2811770  @115714 wtf!? I’ve been having really shitty s...           2987947   \n",
       "2811771  @143549 @sprintcare You have to go to https://...               NaN   \n",
       "2811772  @823870 Sounds delicious, Sarah! 😋 https://t.c...               NaN   \n",
       "2811773  @AldiUK  warm sloe gin mince pies with ice cre...   2987951,2987949   \n",
       "\n",
       "         in_response_to_tweet_id  \n",
       "0                            3.0  \n",
       "1                            1.0  \n",
       "2                            4.0  \n",
       "3                            5.0  \n",
       "4                            6.0  \n",
       "...                          ...  \n",
       "2811769                2987948.0  \n",
       "2811770                      NaN  \n",
       "2811771                2812239.0  \n",
       "2811772                2987950.0  \n",
       "2811773                      NaN  \n",
       "\n",
       "[2811774 rows x 7 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.drop_duplicates(subset=None, keep='first', inplace=False, ignore_index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "828c854f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Ways to clean the data\n",
    "# Error-free data -- remove\n",
    "# Data quality -- correct types numeric, Boolean ect.\n",
    "# Accurate & Efficent: correct values (cross-check data for accuracy)\n",
    "# Completeness (know all required values)\n",
    "# Maintain Data Consitency (same dataset by comparing two similar systems)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}