Further Analysis and EDA.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "6e9b9ef6-5221-4ad2-b108-d3551b338fba",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import pandas as pd\n",
    "from pyspark.sql.functions import *\n",
    "from pyspark.sql.types import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5a8e8445-fde8-44c9-9c29-13002375ba36",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option(\"display.max_colwidth\",None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1a45c449-85e9-4336-b5c4-6d9dc49c8c99",
   "metadata": {},
   "outputs": [],
   "source": [
    "from google.cloud import storage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "b7853cea-7da6-4208-a95f-00a20f9cd0ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3846ee36-c06c-4756-81e9-f0d02d71bfb6",
   "metadata": {},
   "outputs": [],
   "source": [
    "spark = SparkSession.builder.getOrCreate()\n",
    "spark.conf.set(\"spark.sql.repl.eagerEval.enabled\",True) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "def338ea-a44a-4329-9fc2-2f91e0e21ebd",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/01 18:43:33 WARN org.apache.spark.sql.catalyst.util.package: Truncated the string representation of a plan since it was too large. This behavior can be adjusted by setting 'spark.sql.debug.maxToStringFields'.\n"
     ]
    }
   ],
   "source": [
    "twitter_df=spark.read.parquet('gs://msca-bdp-students-bucket/shared_data/kishorkumarreddy/filtered_data_for_analysis_1')\n",
    "c = twitter_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "c6b9675c-62dd-4388-89cd-b3282e70b88a",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df.createOrReplaceTempView(\"twitter_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "e1ede0be-8835-47c4-b2ef-286b1de5319c",
   "metadata": {},
   "outputs": [],
   "source": [
    "query = '''\n",
    "select retweeted as is_retweeted,\n",
    "retweeted_status.id as rt_original_id,\n",
    "retweeted_status.user.screen_name as rt_original_user,\n",
    "user.screen_name as handle_name,\n",
    "user.verified as is_verified,\n",
    "user.followers_count as followers,\n",
    "user.statuses_count as total_tweets\n",
    "from twitter_df'''\n",
    "tw_sq1 = spark.sql(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "236b2445-578e-4054-abe5-6c746afd285a",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df1 = tw_sq1.withColumn(\"tweet_type\",when(col(\"rt_original_id\").isNull() != True,\"retweet\").otherwise(\"tweet\")).filter(col('tweet_type') == 'tweet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "3b53ec95-f0d7-4d56-954a-2aa2f54cbd23",
   "metadata": {},
   "outputs": [],
   "source": [
    "twitter_df1.createOrReplaceTempView(\"twitter_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "bde8fab2-ff44-450c-a8e4-c7573006c201",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "root\n",
      " |-- is_retweeted: string (nullable = true)\n",
      " |-- rt_original_id: long (nullable = true)\n",
      " |-- rt_original_user: string (nullable = true)\n",
      " |-- handle_name: string (nullable = true)\n",
      " |-- is_verified: boolean (nullable = true)\n",
      " |-- followers: long (nullable = true)\n",
      " |-- total_tweets: long (nullable = true)\n",
      " |-- tweet_type: string (nullable = false)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "twitter_df1.printSchema()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5d37b2a-3cbb-454e-bbad-b2f6f0421b35",
   "metadata": {},
   "source": [
    "# Based on Overall Content (Tweets), the Most Prolific Tweeters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "424d94ea-c90e-47a2-aba2-3309ed5d4452",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>total_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>soldier_777</td>\n",
       "      <td>4327697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>zazoomblog</td>\n",
       "      <td>4197074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PulpNews</td>\n",
       "      <td>4187315</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>missb62</td>\n",
       "      <td>3137543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>sectest9</td>\n",
       "      <td>3080006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>marekingu</td>\n",
       "      <td>2897601</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Knewz_Currently</td>\n",
       "      <td>2850254</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>jornalistavitor</td>\n",
       "      <td>2815923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>filafresh</td>\n",
       "      <td>2736894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>paul_cude</td>\n",
       "      <td>2662698</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              user  total_content\n",
       "0      soldier_777        4327697\n",
       "1       zazoomblog        4197074\n",
       "2         PulpNews        4187315\n",
       "3          missb62        3137543\n",
       "4         sectest9        3080006\n",
       "5        marekingu        2897601\n",
       "6  Knewz_Currently        2850254\n",
       "7  jornalistavitor        2815923\n",
       "8        filafresh        2736894\n",
       "9        paul_cude        2662698"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql('SELECT handle_name as user, max(total_tweets) as total_content FROM twitter_df GROUP BY user ORDER BY total_content DESC LIMIT 10').toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "696bc0cd-aeb9-4917-973b-15c4f6c22986",
   "metadata": {},
   "source": [
    "# Based on Original Content (Tweets), the Most Prolific Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "37f25451-ed0e-4c37-bd2c-51c6389ffa36",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user</th>\n",
       "      <th>original_content</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ParentSecurity</td>\n",
       "      <td>3128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AJBlackston</td>\n",
       "      <td>1323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PulpNews</td>\n",
       "      <td>1290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>getthatrightgtr</td>\n",
       "      <td>1147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AirLiveRadio</td>\n",
       "      <td>1038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Bgm117771</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MarchingTruth2</td>\n",
       "      <td>873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>oodlu_tweets</td>\n",
       "      <td>858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>itsrohitchouhan</td>\n",
       "      <td>857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DuoInspirations</td>\n",
       "      <td>837</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              user  original_content\n",
       "0   ParentSecurity              3128\n",
       "1      AJBlackston              1323\n",
       "2         PulpNews              1290\n",
       "3  getthatrightgtr              1147\n",
       "4     AirLiveRadio              1038\n",
       "5        Bgm117771              1002\n",
       "6   MarchingTruth2               873\n",
       "7     oodlu_tweets               858\n",
       "8  itsrohitchouhan               857\n",
       "9  DuoInspirations               837"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql('SELECT handle_name as user, count(*) as original_content FROM twitter_df WHERE tweet_type = \"tweet\" GROUP BY user ORDER BY original_content DESC LIMIT 10').toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "1be8b88e-97c9-4493-ac63-720bcea181ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1005167"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df1 = tw_Q1.withColumn(\"tweet_type\",when(col(\"rt_original_id\").isNull() == True,\"tweet\"))\n",
    "twitter_df1.createOrReplaceTempView(\"twitter_df\")\n",
    "twitter_df1.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e452329d-2e1f-4f4b-822b-20280cbd5a58",
   "metadata": {},
   "source": [
    "Identifying the Most Prolific Authors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "dd90106f-69b7-44e0-b539-1299d9e8bc53",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>handle_name</th>\n",
       "      <th>count(1)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ParentSecurity</td>\n",
       "      <td>3128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>AJBlackston</td>\n",
       "      <td>1323</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PulpNews</td>\n",
       "      <td>1290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>getthatrightgtr</td>\n",
       "      <td>1147</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>AirLiveRadio</td>\n",
       "      <td>1038</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Bgm117771</td>\n",
       "      <td>1002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>MarchingTruth2</td>\n",
       "      <td>873</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>oodlu_tweets</td>\n",
       "      <td>858</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>itsrohitchouhan</td>\n",
       "      <td>857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>DuoInspirations</td>\n",
       "      <td>837</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       handle_name  count(1)\n",
       "0   ParentSecurity      3128\n",
       "1      AJBlackston      1323\n",
       "2         PulpNews      1290\n",
       "3  getthatrightgtr      1147\n",
       "4     AirLiveRadio      1038\n",
       "5        Bgm117771      1002\n",
       "6   MarchingTruth2       873\n",
       "7     oodlu_tweets       858\n",
       "8  itsrohitchouhan       857\n",
       "9  DuoInspirations       837"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query = '''select handle_name, \n",
    "count(*) \n",
    "from twitter_df \n",
    "where tweet_type = \"tweet\" \n",
    "group by handle_name \n",
    "order by count(*) \n",
    "desc limit 10'''\n",
    "spark.sql(query).toPandas()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fc923be4-f50e-402c-8e90-49999588f709",
   "metadata": {},
   "source": [
    "# Based on Retweets Count (Most Prolific Retweets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8a5b097d-4888-472c-9863-ea5e3f30d0d4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>rt_original_user</th>\n",
       "      <th>count(1)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Golshifteh</td>\n",
       "      <td>7451</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GretaThunberg</td>\n",
       "      <td>4504</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>PahlaviReza</td>\n",
       "      <td>3786</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Hornystepsis1</td>\n",
       "      <td>3638</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>robbeorn</td>\n",
       "      <td>3277</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80269</th>\n",
       "      <td>baddiejenny92</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80270</th>\n",
       "      <td>rouse_dr</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80271</th>\n",
       "      <td>CabellsMedicine</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80272</th>\n",
       "      <td>mattitakubel</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>80273</th>\n",
       "      <td>NACCE</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>80274 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      rt_original_user  count(1)\n",
       "0           Golshifteh      7451\n",
       "1        GretaThunberg      4504\n",
       "2          PahlaviReza      3786\n",
       "3        Hornystepsis1      3638\n",
       "4             robbeorn      3277\n",
       "...                ...       ...\n",
       "80269    baddiejenny92         1\n",
       "80270         rouse_dr         1\n",
       "80271  CabellsMedicine         1\n",
       "80272     mattitakubel         1\n",
       "80273            NACCE         1\n",
       "\n",
       "[80274 rows x 2 columns]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spark.sql('select rt_original_user, count(*) from twitter_df where rt_original_user IS NOT NULL group by rt_original_user order by count(*) desc').toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ff22c84c-9a2a-4954-96af-57ce99f65b34",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border='1'>\n",
       "<tr><th>summary</th><th>created_at</th><th>favorite_count</th><th>filter_level</th><th>id</th><th>id_str</th><th>in_reply_to_screen_name</th><th>in_reply_to_status_id</th><th>in_reply_to_status_id_str</th><th>in_reply_to_user_id</th><th>in_reply_to_user_id_str</th><th>lang</th><th>quote_count</th><th>quoted_status_id</th><th>quoted_status_id_str</th><th>quoted_text</th><th>reply_count</th><th>retweet_count</th><th>retweeted</th><th>retweeted_from</th><th>source</th><th>text</th><th>timestamp_ms</th><th>tweet_text</th><th>stripped</th></tr>\n",
       "<tr><td>count</td><td>1005167</td><td>1005167</td><td>1005167</td><td>1005167</td><td>1005167</td><td>33709</td><td>27232</td><td>27232</td><td>33709</td><td>33709</td><td>1005167</td><td>1005167</td><td>72963</td><td>72963</td><td>72946</td><td>1005167</td><td>1005167</td><td>1005167</td><td>703935</td><td>1005167</td><td>1005167</td><td>1005167</td><td>1005167</td><td>1005167</td></tr>\n",
       "<tr><td>mean</td><td>null</td><td>0.0</td><td>null</td><td>1.566841370743179...</td><td>1.566841370743179...</td><td>433.0</td><td>1.563454478121966...</td><td>1.563454478121966...</td><td>5.277987172089125...</td><td>5.277987172089125...</td><td>null</td><td>0.0</td><td>1.559602036301624...</td><td>1.559602036301624...</td><td>420.0</td><td>0.0</td><td>0.0</td><td>null</td><td>15417.625925925926</td><td>null</td><td>null</td><td>1.662399067946775...</td><td>null</td><td>null</td></tr>\n",
       "<tr><td>stddev</td><td>null</td><td>0.0</td><td>null</td><td>3.115244689557993...</td><td>3.115244689557993...</td><td>null</td><td>4.269803723556284...</td><td>4.269803723556284...</td><td>6.307440523906169...</td><td>6.307440523906169...</td><td>null</td><td>0.0</td><td>5.343131196135945...</td><td>5.343131196135945...</td><td>null</td><td>0.0</td><td>0.0</td><td>null</td><td>543175.3414855197</td><td>null</td><td>null</td><td>7.427322124370399E9</td><td>null</td><td>null</td></tr>\n",
       "<tr><td>min</td><td>Fri Apr 08 00:00:...</td><td>0</td><td>low</td><td>1511197436458459137</td><td>1511197436458459137</td><td>00Fuzz</td><td>1559129819</td><td>1018119192426491904</td><td>2651</td><td>1000060610518896640</td><td>en</td><td>0</td><td>124623659041161217</td><td>1003686491070128129</td><td>!!! Parents Alert...</td><td>0</td><td>0</td><td></td><td>\n",
       "\n",
       "https</td><td>&lt;a href=&quot;HTTP://b...</td><td>! #students #scho...</td><td>1649132520199</td><td>! #students #scho...</td><td>\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "after a lon...</td></tr>\n",
       "<tr><td>max</td><td>Wed Sep 28 23:59:...</td><td>0</td><td>low</td><td>1623307508390694913</td><td>1623307508390694913</td><td>zzzzzryu</td><td>1623288183676096512</td><td>993241772317913088</td><td>1622359362378375171</td><td>999988014032945152</td><td>en</td><td>0</td><td>1623294581193203714</td><td>999010989751259136</td><td>🫶~The ❤️ back to...</td><td>0</td><td>0</td><td>RT</td><td>… https</td><td>&lt;a href=&quot;https://...</td><td>🫶🏽 Give back to...</td><td>1675861644252</td><td>🫶🏽 give back to...</td><td>🫶🏽 give back to...</td></tr>\n",
       "</table>\n"
      ],
      "text/plain": [
       "+-------+--------------------+--------------+------------+--------------------+--------------------+-----------------------+---------------------+-------------------------+--------------------+-----------------------+-------+-----------+--------------------+--------------------+--------------------+-----------+-------------+---------+------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
       "|summary|          created_at|favorite_count|filter_level|                  id|              id_str|in_reply_to_screen_name|in_reply_to_status_id|in_reply_to_status_id_str| in_reply_to_user_id|in_reply_to_user_id_str|   lang|quote_count|    quoted_status_id|quoted_status_id_str|         quoted_text|reply_count|retweet_count|retweeted|    retweeted_from|              source|                text|        timestamp_ms|          tweet_text|            stripped|\n",
       "+-------+--------------------+--------------+------------+--------------------+--------------------+-----------------------+---------------------+-------------------------+--------------------+-----------------------+-------+-----------+--------------------+--------------------+--------------------+-----------+-------------+---------+------------------+--------------------+--------------------+--------------------+--------------------+--------------------+\n",
       "|  count|             1005167|       1005167|     1005167|             1005167|             1005167|                  33709|                27232|                    27232|               33709|                  33709|1005167|    1005167|               72963|               72963|               72946|    1005167|      1005167|  1005167|            703935|             1005167|             1005167|             1005167|             1005167|             1005167|\n",
       "|   mean|                null|           0.0|        null|1.566841370743179...|1.566841370743179...|                  433.0| 1.563454478121966...|     1.563454478121966...|5.277987172089125...|   5.277987172089125...|   null|        0.0|1.559602036301624...|1.559602036301624...|               420.0|        0.0|          0.0|     null|15417.625925925926|                null|                null|1.662399067946775...|                null|                null|\n",
       "| stddev|                null|           0.0|        null|3.115244689557993...|3.115244689557993...|                   null| 4.269803723556284...|     4.269803723556284...|6.307440523906169...|   6.307440523906169...|   null|        0.0|5.343131196135945...|5.343131196135945...|                null|        0.0|          0.0|     null| 543175.3414855197|                null|                null| 7.427322124370399E9|                null|                null|\n",
       "|    min|Fri Apr 08 00:00:...|             0|         low| 1511197436458459137| 1511197436458459137|                 00Fuzz|           1559129819|      1018119192426491904|                2651|    1000060610518896640|     en|          0|  124623659041161217| 1003686491070128129|!!! Parents Alert...|          0|            0|         |           \n",
       "\n",
       "https|<a href=\"HTTP://b...|! #students #scho...|       1649132520199|! #students #scho...|\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "\n",
       "after a lon...|\n",
       "|    max|Wed Sep 28 23:59:...|             0|         low| 1623307508390694913| 1623307508390694913|               zzzzzryu|  1623288183676096512|       993241772317913088| 1622359362378375171|     999988014032945152|     en|          0| 1623294581193203714|  999010989751259136|🫶~The ❤️ back to...|          0|            0|       RT|           … https|<a href=\"https://...|🫶🏽 Give back to...|       1675861644252|🫶🏽 give back to...|🫶🏽 give back to...|\n",
       "+-------+--------------------+--------------+------------+--------------------+--------------------+-----------------------+---------------------+-------------------------+--------------------+-----------------------+-------+-----------+--------------------+--------------------+--------------------+-----------+-------------+---------+------------------+--------------------+--------------------+--------------------+--------------------+--------------------+"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7aba5d28-2664-4e02-bfa9-63dd1ccce8ac",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_retweetCounts = twitter_df.select([\n",
    "    twitter_df.retweet_count.alias(\"direct_retweet_count\"),\n",
    "    twitter_df.quoted_status.retweet_count.alias(\"quoted_status.retweet_count\"),\n",
    "    twitter_df.retweeted_status.quoted_status.retweet_count.alias(\"retweeted_status.quoted_status.retweet_count\"),\n",
    "    twitter_df.retweeted_status.retweet_count.alias(\"retweeted_status.retweet_count\"),\n",
    "    twitter_df.retweeted_status.reply_count.alias(\"retweeted_status.reply_count\"),\n",
    "    twitter_df.reply_count.alias(\"reply_count\"),\n",
    "    twitter_df.tweet_text\n",
    "]).limit(10000).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "deca7df6-1f00-458c-a62f-b01d50c2144e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "direct_retweet_count                            10000\n",
       "quoted_status.retweet_count                       535\n",
       "retweeted_status.quoted_status.retweet_count      329\n",
       "retweeted_status.retweet_count                   7935\n",
       "retweeted_status.reply_count                     7935\n",
       "reply_count                                     10000\n",
       "tweet_text                                      10000\n",
       "dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_retweetCounts.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "24bb088e-9a82-4e14-8066-7391bc528963",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>direct_retweet_count</th>\n",
       "      <th>quoted_status.retweet_count</th>\n",
       "      <th>retweeted_status.quoted_status.retweet_count</th>\n",
       "      <th>retweeted_status.retweet_count</th>\n",
       "      <th>retweeted_status.reply_count</th>\n",
       "      <th>reply_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>10000.0</td>\n",
       "      <td>535.000000</td>\n",
       "      <td>329.000000</td>\n",
       "      <td>7935.000000</td>\n",
       "      <td>7935.000000</td>\n",
       "      <td>10000.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.0</td>\n",
       "      <td>1073.067290</td>\n",
       "      <td>907.468085</td>\n",
       "      <td>1370.853938</td>\n",
       "      <td>232.817139</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.0</td>\n",
       "      <td>2663.100875</td>\n",
       "      <td>2459.589249</td>\n",
       "      <td>2528.899777</td>\n",
       "      <td>425.764715</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>10.000000</td>\n",
       "      <td>21.000000</td>\n",
       "      <td>19.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>106.000000</td>\n",
       "      <td>109.000000</td>\n",
       "      <td>184.000000</td>\n",
       "      <td>13.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.0</td>\n",
       "      <td>969.500000</td>\n",
       "      <td>969.000000</td>\n",
       "      <td>1390.500000</td>\n",
       "      <td>232.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.0</td>\n",
       "      <td>22407.000000</td>\n",
       "      <td>20944.000000</td>\n",
       "      <td>14588.000000</td>\n",
       "      <td>2293.000000</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       direct_retweet_count  quoted_status.retweet_count  \\\n",
       "count               10000.0                   535.000000   \n",
       "mean                    0.0                  1073.067290   \n",
       "std                     0.0                  2663.100875   \n",
       "min                     0.0                     0.000000   \n",
       "25%                     0.0                    10.000000   \n",
       "50%                     0.0                   106.000000   \n",
       "75%                     0.0                   969.500000   \n",
       "max                     0.0                 22407.000000   \n",
       "\n",
       "       retweeted_status.quoted_status.retweet_count  \\\n",
       "count                                    329.000000   \n",
       "mean                                     907.468085   \n",
       "std                                     2459.589249   \n",
       "min                                        0.000000   \n",
       "25%                                       21.000000   \n",
       "50%                                      109.000000   \n",
       "75%                                      969.000000   \n",
       "max                                    20944.000000   \n",
       "\n",
       "       retweeted_status.retweet_count  retweeted_status.reply_count  \\\n",
       "count                     7935.000000                   7935.000000   \n",
       "mean                      1370.853938                    232.817139   \n",
       "std                       2528.899777                    425.764715   \n",
       "min                          1.000000                      0.000000   \n",
       "25%                         19.000000                      0.000000   \n",
       "50%                        184.000000                     13.000000   \n",
       "75%                       1390.500000                    232.000000   \n",
       "max                      14588.000000                   2293.000000   \n",
       "\n",
       "       reply_count  \n",
       "count      10000.0  \n",
       "mean           0.0  \n",
       "std            0.0  \n",
       "min            0.0  \n",
       "25%            0.0  \n",
       "50%            0.0  \n",
       "75%            0.0  \n",
       "max            0.0  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_retweetCounts.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "d07e197e-6d4c-48d2-b655-4fa7536729ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "652559"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_original_tweets = twitter_df.filter(\"retweeted_status.retweet_count is not null\") \n",
    "df_original_tweets.count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "4b53cb71-e36f-4421-a5f9-144be89ceece",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Stage 31:=====================================================>(209 + 1) / 210]\r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Count of all tweets: 1005167\n",
      "Count of all original tweets: 352608\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "count1 = twitter_df.count()\n",
    "count2 = twitter_df.filter('retweeted_status is null').count()\n",
    "print('Count of all tweets:', count1)\n",
    "print('Count of all original tweets:', count2)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "71a20d95-57e6-458a-8a57-0e4d08a42372",
   "metadata": {},
   "source": [
    "## Identifying the Most Prolific (or) Influential Twitterers Using the Count of Tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "97c4514c-7d8f-484b-a124-4d6ecabecbd4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['coordinates', 'created_at', 'display_text_range', 'entities', 'extended_entities', 'extended_tweet', 'favorite_count', 'favorited', 'filter_level', 'geo', 'id', 'id_str', 'in_reply_to_screen_name', 'in_reply_to_status_id', 'in_reply_to_status_id_str', 'in_reply_to_user_id', 'in_reply_to_user_id_str', 'is_quote_status', 'lang', 'place', 'possibly_sensitive', 'quote_count', 'quoted_status', 'quoted_status_id', 'quoted_status_id_str', 'quoted_status_permalink', 'quoted_text', 'reply_count', 'retweet_count', 'retweeted', 'retweeted_from', 'retweeted_status', 'source', 'text', 'timestamp_ms', 'truncated', 'tweet_text', 'user', 'withheld_in_countries', 'stripped']\n"
     ]
    }
   ],
   "source": [
    "print(twitter_df.columns)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9b11d610-a136-4017-88b1-16403efa4d02",
   "metadata": {},
   "source": [
    "### Extracting Important Columns and fields from the filtered dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "72d064a5-e6db-4b2c-a50e-9b0532355567",
   "metadata": {},
   "outputs": [],
   "source": [
    "#tw_df.printSchema()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "ec0fc236-7872-42d8-bf97-bdef221d3858",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "df_coords = twitter_df.select([\n",
    "    twitter_df.coordinates.alias(\"direct_coordinates\"),\n",
    "    twitter_df.geo.coordinates.alias(\"geo.coordinates\"),\n",
    "    twitter_df.place.bounding_box.coordinates.alias(\"place.bounding_box.coordinates\"),\n",
    "    twitter_df.place.country_code.alias(\"place.country_code\"),\n",
    "    twitter_df.place.country.alias(\"place.country\"),\n",
    "    twitter_df.place['name'].alias(\"place.name\"),\n",
    "    twitter_df.place.full_name.alias(\"place.full_name\"),\n",
    "    twitter_df.user.location.alias(\"user.location\"),\n",
    "    twitter_df.tweet_text,\n",
    "    twitter_df.user.withheld_in_countries\n",
    "]).limit(30000).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "ed09ee8c-0cbf-4d2b-8ed6-ff8f320bd9d5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>direct_coordinates</th>\n",
       "      <th>geo.coordinates</th>\n",
       "      <th>place.bounding_box.coordinates</th>\n",
       "      <th>place.country_code</th>\n",
       "      <th>place.country</th>\n",
       "      <th>place.name</th>\n",
       "      <th>place.full_name</th>\n",
       "      <th>user.location</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>user.withheld_in_countries</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>72</td>\n",
       "      <td>72</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>193</td>\n",
       "      <td>12523</td>\n",
       "      <td>30000</td>\n",
       "      <td>30000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unique</th>\n",
       "      <td>60</td>\n",
       "      <td>60</td>\n",
       "      <td>161</td>\n",
       "      <td>15</td>\n",
       "      <td>17</td>\n",
       "      <td>162</td>\n",
       "      <td>162</td>\n",
       "      <td>5560</td>\n",
       "      <td>14065</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>top</th>\n",
       "      <td>([-71.4128343, 41.8239891], Point)</td>\n",
       "      <td>[41.8239891, -71.4128343]</td>\n",
       "      <td>[[[-71.474186, 41.772455], [-71.474186, 41.861713], [-71.369479, 41.861713], [-71.369479, 41.772455]]]</td>\n",
       "      <td>US</td>\n",
       "      <td>United States</td>\n",
       "      <td>Providence</td>\n",
       "      <td>Providence, RI</td>\n",
       "      <td>United States</td>\n",
       "      <td>let go of students in #sharif_university #mahsaamini https://t.co/jjnszzjjf5</td>\n",
       "      <td>[]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>freq</th>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>124</td>\n",
       "      <td>123</td>\n",
       "      <td>5</td>\n",
       "      <td>5</td>\n",
       "      <td>225</td>\n",
       "      <td>2563</td>\n",
       "      <td>30000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        direct_coordinates            geo.coordinates  \\\n",
       "count                                   72                         72   \n",
       "unique                                  60                         60   \n",
       "top     ([-71.4128343, 41.8239891], Point)  [41.8239891, -71.4128343]   \n",
       "freq                                     5                          5   \n",
       "\n",
       "                                                                                place.bounding_box.coordinates  \\\n",
       "count                                                                                                      193   \n",
       "unique                                                                                                     161   \n",
       "top     [[[-71.474186, 41.772455], [-71.474186, 41.861713], [-71.369479, 41.861713], [-71.369479, 41.772455]]]   \n",
       "freq                                                                                                         5   \n",
       "\n",
       "       place.country_code  place.country  place.name place.full_name  \\\n",
       "count                 193            193         193             193   \n",
       "unique                 15             17         162             162   \n",
       "top                    US  United States  Providence  Providence, RI   \n",
       "freq                  124            123           5               5   \n",
       "\n",
       "        user.location  \\\n",
       "count           12523   \n",
       "unique           5560   \n",
       "top     United States   \n",
       "freq              225   \n",
       "\n",
       "                                                                          tweet_text  \\\n",
       "count                                                                          30000   \n",
       "unique                                                                         14065   \n",
       "top     let go of students in #sharif_university #mahsaamini https://t.co/jjnszzjjf5   \n",
       "freq                                                                            2563   \n",
       "\n",
       "       user.withheld_in_countries  \n",
       "count                       30000  \n",
       "unique                          1  \n",
       "top                            []  \n",
       "freq                        30000  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_coords.describe()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "555bf457-9266-4964-8d3e-377a51ef0009",
   "metadata": {},
   "source": [
    "### Taking the non-null values from the field user.withheld_in_countries to avoid bad plotting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "122e99f7-7266-4330-98ed-df7631573478",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "1005167"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df.filter(\"user.withheld_in_countries is not NULL\").count()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "3b64b764-a104-4015-a31c-6b462d92fa87",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/plain": [
       "Index(['coordinates', 'created_at', 'display_text_range', 'entities',\n",
       "       'extended_entities', 'extended_tweet', 'favorite_count', 'favorited',\n",
       "       'filter_level', 'geo', 'id', 'id_str', 'in_reply_to_screen_name',\n",
       "       'in_reply_to_status_id', 'in_reply_to_status_id_str',\n",
       "       'in_reply_to_user_id', 'in_reply_to_user_id_str', 'is_quote_status',\n",
       "       'lang', 'place', 'possibly_sensitive', 'quote_count', 'quoted_status',\n",
       "       'quoted_status_id', 'quoted_status_id_str', 'quoted_status_permalink',\n",
       "       'quoted_text', 'reply_count', 'retweet_count', 'retweeted',\n",
       "       'retweeted_from', 'retweeted_status', 'source', 'text', 'timestamp_ms',\n",
       "       'truncated', 'tweet_text', 'user', 'withheld_in_countries', 'stripped'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "twitter_df.limit(5).toPandas().columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "9ba01280-864e-4252-a8aa-ecc5c35758ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "23/03/01 18:57:54 ERROR org.apache.spark.network.client.TransportResponseHandler: Still have 2 requests outstanding when connection from /10.128.0.68:40754 is closed\n",
      "23/03/01 18:57:54 WARN org.apache.spark.storage.BlockManagerMasterEndpoint: Error trying to remove broadcast 36 from block manager BlockManagerId(32, hub-msca-bdp-dphub-students-kishorkumarreddy-sw-qdhr.c.msca-bdp-students.internal, 40227, None)\n",
      "java.io.IOException: Connection from /10.128.0.68:40754 closed\n",
      "\tat org.apache.spark.network.client.TransportResponseHandler.channelInactive(TransportResponseHandler.java:146)\n",
      "\tat org.apache.spark.network.server.TransportChannelHandler.channelInactive(TransportChannelHandler.java:117)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)\n",
      "\tat io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)\n",
      "\tat org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:225)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818)\n",
      "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n",
      "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n",
      "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n",
      "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "23/03/01 18:57:54 WARN org.apache.spark.storage.BlockManagerMasterEndpoint: Error trying to remove shuffle 10 from block manager BlockManagerId(32, hub-msca-bdp-dphub-students-kishorkumarreddy-sw-qdhr.c.msca-bdp-students.internal, 40227, None)\n",
      "java.io.IOException: Connection from /10.128.0.68:40754 closed\n",
      "\tat org.apache.spark.network.client.TransportResponseHandler.channelInactive(TransportResponseHandler.java:146)\n",
      "\tat org.apache.spark.network.server.TransportChannelHandler.channelInactive(TransportChannelHandler.java:117)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)\n",
      "\tat io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)\n",
      "\tat org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:225)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818)\n",
      "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n",
      "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n",
      "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n",
      "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "23/03/01 18:57:54 WARN org.apache.spark.storage.BlockManagerMasterEndpoint: No more replicas available for broadcast_36_piece0 !\n",
      "23/03/01 18:57:54 ERROR org.apache.spark.network.client.TransportResponseHandler: Still have 1 requests outstanding when connection from /10.128.0.68:40736 is closed\n",
      "23/03/01 18:57:54 WARN org.apache.spark.storage.BlockManagerMasterEndpoint: Error trying to remove broadcast 36 from block manager BlockManagerId(30, hub-msca-bdp-dphub-students-kishorkumarreddy-sw-qdhr.c.msca-bdp-students.internal, 38187, None)\n",
      "java.io.IOException: Connection from /10.128.0.68:40736 closed\n",
      "\tat org.apache.spark.network.client.TransportResponseHandler.channelInactive(TransportResponseHandler.java:146)\n",
      "\tat org.apache.spark.network.server.TransportChannelHandler.channelInactive(TransportChannelHandler.java:117)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)\n",
      "\tat io.netty.handler.timeout.IdleStateHandler.channelInactive(IdleStateHandler.java:277)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.ChannelInboundHandlerAdapter.channelInactive(ChannelInboundHandlerAdapter.java:81)\n",
      "\tat org.apache.spark.network.util.TransportFrameDecoder.channelInactive(TransportFrameDecoder.java:225)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.fireChannelInactive(AbstractChannelHandlerContext.java:241)\n",
      "\tat io.netty.channel.DefaultChannelPipeline$HeadContext.channelInactive(DefaultChannelPipeline.java:1405)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:262)\n",
      "\tat io.netty.channel.AbstractChannelHandlerContext.invokeChannelInactive(AbstractChannelHandlerContext.java:248)\n",
      "\tat io.netty.channel.DefaultChannelPipeline.fireChannelInactive(DefaultChannelPipeline.java:901)\n",
      "\tat io.netty.channel.AbstractChannel$AbstractUnsafe$8.run(AbstractChannel.java:818)\n",
      "\tat io.netty.util.concurrent.AbstractEventExecutor.safeExecute(AbstractEventExecutor.java:164)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor.runAllTasks(SingleThreadEventExecutor.java:472)\n",
      "\tat io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:497)\n",
      "\tat io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:989)\n",
      "\tat io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74)\n",
      "\tat io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30)\n",
      "\tat java.lang.Thread.run(Thread.java:750)\n",
      "                                                                                \r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>geo_coordinates</th>\n",
       "      <th>user_name</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>verified_user</th>\n",
       "      <th>user_location</th>\n",
       "      <th>user_description</th>\n",
       "      <th>reply_count</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>retweeted_status</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Sat Oct 29 18:36:18 +0000 2022</td>\n",
       "      <td>1586426715760459778</td>\n",
       "      <td>None</td>\n",
       "      <td>TitanUp Ty</td>\n",
       "      <td>2256</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>#LGM #TITANS #GBO🍊 #FJB #NJ #USA 🇺🇸</td>\n",
       "      <td>5</td>\n",
       "      <td>51</td>\n",
       "      <td>(None, Sat Oct 29 13:22:46 +0000 2022, [0, 82], ([Row(indices=[49, 53], text='SEC'), Row(indices=[54, 70], text='CollegeFootball'), Row(indices=[71, 75], text='cfb'), Row(indices=[76, 82], text='NCAAF')], [Row(additional_media_info=None, description=None, display_url='pic.twitter.com/ra3EkYetqu', expanded_url='https://twitter.com/JWPSports/status/1586347814967132163/photo/1', id=1586347810932154368, id_str='1586347810932154368', indices=[83, 106], media_url='http://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', media_url_https='https://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', sizes=Row(large=Row(h=1159, resize='fit', w=1172), medium=Row(h=1159, resize='fit', w=1172), small=Row(h=672, resize='fit', w=680), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='photo', url='https://t.co/ra3EkYetqu')], [], [], []), ([Row(additional_media_info=None, description=None, display_url='pic.twitter.com/ra3EkYetqu', expanded_url='https://twitter.com/JWPSports/status/1586347814967132163/photo/1', id=1586347810932154368, id_str='1586347810932154368', indices=[83, 106], media_url='http://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', media_url_https='https://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', sizes=Row(large=Row(h=1159, resize='fit', w=1172), medium=Row(h=1159, resize='fit', w=1172), small=Row(h=672, resize='fit', w=680), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='photo', url='https://t.co/ra3EkYetqu', video_info=None)],), None, 402, False, low, None, 1586347814967132163, 1586347814967132163, None, None, None, None, None, False, en, None, False, 3, None, None, None, None, 5, 51, False, None, &lt;a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\"&gt;Twitter for iPhone&lt;/a&gt;, Your current SEC College Football QBR leaders.👀\\n\\n#SEC #CollegeFootball #cfb #NCAAF https://t.co/ra3EkYetqu, False, (False, Fri Jan 31 15:24:06 +0000 2020, True, False, JWP Sports! Be sure to find me on Instagram @jwp_sports | @cfbalerts only on Instagram. Connect with me on all platforms ⬇️, 1935, 4414, 1857, True, 1223265683565416448, 1223265683565416448, False, 6, None, JWP Sports , F5F8FA, , , False, None, http://pbs.twimg.com/profile_images/1225762639957106688/k75frUmv_normal.jpg, https://pbs.twimg.com/profile_images/1225762639957106688/k75frUmv_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, JWPSports, 1623, none, https://linktr.ee/jwpsports, False, None, []), None)</td>\n",
       "      <td>your current sec college football qbr leaders.👀\\n\\n#sec #collegefootball #cfb #ncaaf https://t.co/ra3ekyetqu</td>\n",
       "      <td>RT @JWPSports: Your current SEC College Football QBR leaders.👀\\n\\n#SEC #CollegeFootball #cfb #NCAAF https://t.co/ra3EkYetqu</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Sat Oct 29 18:36:18 +0000 2022</td>\n",
       "      <td>1586426719090589696</td>\n",
       "      <td>None</td>\n",
       "      <td>SickYooo</td>\n",
       "      <td>88</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>..</td>\n",
       "      <td>3</td>\n",
       "      <td>168</td>\n",
       "      <td>(None, Sun Feb 13 19:58:23 +0000 2022, [0, 140], ([Row(indices=[52, 66], text='collegebaddie'), Row(indices=[67, 75], text='college'), Row(indices=[76, 87], text='latinahead'), Row(indices=[88, 102], text='latinaexposed')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1492951306398355456', indices=[104, 127], url='https://t.co/tLxG8VA9ih')], []), None, ([0, 155], ([Row(indices=[52, 66], text='collegebaddie'), Row(indices=[67, 75], text='college'), Row(indices=[76, 87], text='latinahead'), Row(indices=[88, 102], text='latinaexposed'), Row(indices=[103, 116], text='LatinaBeauty'), Row(indices=[117, 130], text='sloppythroat'), Row(indices=[131, 147], text='sloppyheadGIVER'), Row(indices=[148, 155], text='blowie')], [Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/GhG6abUn99', expanded_url='https://twitter.com/pxrnstachee/status/1492951306398355456/video/1', id=1492951256318431232, id_str='1492951256318431232', indices=[156, 179], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=732), medium=Row(h=1200, resize='fit', w=686), small=Row(h=680, resize='fit', w=389), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/GhG6abUn99', video_info=Row(aspect_ratio=[183, 320], duration_millis=45001, variants=[Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/320x558/hDDxsZ1Kgt_l7sg3.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/pl/np_2fWZzuV1YYGWs.m3u8?tag=12&amp;container=fmp4'), Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/720x1258/OSWvqvJs35tJrRRC.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/480x838/ZqGt4Qzk4jREnItM.mp4?tag=12')]))], [], [], []), ([Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/GhG6abUn99', expanded_url='https://twitter.com/pxrnstachee/status/1492951306398355456/video/1', id=1492951256318431232, id_str='1492951256318431232', indices=[156, 179], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=732), medium=Row(h=1200, resize='fit', w=686), small=Row(h=680, resize='fit', w=389), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/GhG6abUn99', video_info=Row(aspect_ratio=[183, 320], duration_millis=45001, variants=[Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/320x558/hDDxsZ1Kgt_l7sg3.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/pl/np_2fWZzuV1YYGWs.m3u8?tag=12&amp;container=fmp4'), Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/720x1258/OSWvqvJs35tJrRRC.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/480x838/ZqGt4Qzk4jREnItM.mp4?tag=12')]))],), Latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed #LatinaBeauty #sloppythroat #sloppyheadGIVER #blowie https://t.co/GhG6abUn99), 1275, False, low, None, 1492951306398355456, 1492951306398355456, None, None, None, None, None, False, en, None, True, 0, None, None, None, None, 3, 168, False, None, &lt;a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\"&gt;Twitter for iPhone&lt;/a&gt;, Latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed… https://t.co/tLxG8VA9ih, True, (False, Thu May 10 01:13:52 +0000 2018, True, False, | 18+ Daily Content | 🌹No Content Is Owned By The Maker Of This Account🌹| Dm for credit/removal no copyright infringement intended, 8017, 64513, 45, False, 994384995740352512, 994384995740352512, False, 216, None, ParadisePxrn (dm for cheap promo), F5F8FA, , , False, https://pbs.twimg.com/profile_banners/994384995740352512/1666345959, http://pbs.twimg.com/profile_images/1583363185045667840/IYjjH-zG_normal.jpg, https://pbs.twimg.com/profile_images/1583363185045667840/IYjjH-zG_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, pxrnstachee, 565, none, https://discord.gg/sullen, False, None, []), None)</td>\n",
       "      <td>latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed #latinabeauty #sloppythroat #sloppyheadgiver #blowie https://t.co/ghg6abun99</td>\n",
       "      <td>RT @pxrnstachee: Latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed #LatinaBeauty #slop…</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Sat Oct 29 18:36:55 +0000 2022</td>\n",
       "      <td>1586426871524343814</td>\n",
       "      <td>None</td>\n",
       "      <td>Saturn</td>\n",
       "      <td>883</td>\n",
       "      <td>False</td>\n",
       "      <td>America</td>\n",
       "      <td>#براندازم #نه_به_جمهوری_اسلامی</td>\n",
       "      <td>0</td>\n",
       "      <td>7</td>\n",
       "      <td>(None, Sat Oct 29 18:23:54 +0000 2022, None, ([Row(indices=[37, 56], text='UniversityofTehran')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586423598645841922', indices=[117, 140], url='https://t.co/J61gZ0Gy9z')], []), None, ([0, 279], ([Row(indices=[37, 56], text='UniversityofTehran'), Row(indices=[267, 279], text='Mahsa_Amini')], None, [], [], []), None, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). Security forces waiting outside arresting and taking students away to undisclosed location #Mahsa_Amini), 11, False, low, None, 1586423598645841922, 1586423598645841922, None, None, None, None, None, True, en, None, None, 0, (None, Sat Oct 29 18:00:59 +0000 2022, None, ([Row(indices=[11, 23], text='دانشکده_فنی')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586417830228267009', indices=[117, 140], url='https://t.co/Fw2dvhchXt')], []), None, ([0, 279], Row(hashtags=[Row(indices=[11, 23], text='دانشکده_فنی'), Row(indices=[268, 279], text='مهسا_امینی')], media=None, symbols=[], urls=[], user_mentions=[]), None, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما مورد حمله نیروهای بسیج دانشجویی قرار گرفتند. نیروهای لباس شخصی به طور غیرقانونی از صبح در دانشگاه مستقر شده بودند و با بسیجی‌ها در این حمله همراه شدند.\\n#مهسا_امینی), 63, False, low, None, 1586417830228267009, 1586417830228267009, None, None, None, None, None, False, fa, None, None, 1, None, None, 1, 25, False, None, &lt;a href=\"https://mobile.twitter.com\" rel=\"nofollow\"&gt;Twitter Web App&lt;/a&gt;, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما… https://t.co/Fw2dvhchXt, True, (False, Tue Aug 11 07:17:35 +0000 2015, False, False, Official account of Iran Human Rights (IHR NGO)- Also visit @iranhr حساب رسمی توییتر سازمان حقوق بشر ایران, 1067, 9849, 284, True, 3414505683, 3414505683, False, 120, Oslo, Norway, Iran Human Rights (IHR NGO), 000000, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/3414505683/1665403153, http://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, https://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, 89C9FA, 000000, 000000, 000000, False, False, IHRights, 8255, none, https://iranhr.net/, True, None, []), None), 1586417830228267009, 1586417830228267009, (twitter.com/ihrights/statu…, https://twitter.com/ihrights/status/1586417830228267009, https://t.co/5MHIgxrjHW), 0, 7, False, None, &lt;a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\"&gt;Twitter for iPhone&lt;/a&gt;, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclo… https://t.co/J61gZ0Gy9z, True, (False, Sat May 02 13:32:40 +0000 2009, True, False, Director of NGO «Iran Human Rights\" @IHRights Professor at the University of Oslo- مدیر سازمان حقوق بشر ایران- پزشک و استاددانشکده پزشکی دانشگاه اسلو, 9660, 5589, 1642, True, 37196329, 37196329, False, 173, Global, Mahmood Amiry-Moghaddam, C0DEED, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/37196329/1425896261, http://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, https://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, iranhr, 11461, none, http://www.iranhr.net, True, None, []), None)</td>\n",
       "      <td>today’s peaceful student protests of #universityoftehran, college of engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). security forces waiting outside arresting and taking students away to undisclosed location #mahsa_amini</td>\n",
       "      <td>RT @iranhr: Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes securit…</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Sat Oct 29 18:38:00 +0000 2022</td>\n",
       "      <td>1586427143302443008</td>\n",
       "      <td>None</td>\n",
       "      <td>💙KOHAKU225💙</td>\n",
       "      <td>127</td>\n",
       "      <td>False</td>\n",
       "      <td>Baton Rouge, LA</td>\n",
       "      <td>None</td>\n",
       "      <td>3</td>\n",
       "      <td>16</td>\n",
       "      <td>(None, Fri Oct 28 23:51:52 +0000 2022, [0, 140], ([Row(indices=[45, 48], text='bj'), Row(indices=[49, 53], text='pyt'), Row(indices=[54, 61], text='baddie'), Row(indices=[62, 69], text='leaked'), Row(indices=[70, 76], text='oussy'), Row(indices=[77, 87], text='backshots'), Row(indices=[88, 103], text='sellingcontent'), Row(indices=[104, 115], text='schoolthot')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586143745078353920', indices=[117, 140], url='https://t.co/Vdd4JmZ8zn')], []), None, ([0, 187], ([Row(indices=[45, 48], text='bj'), Row(indices=[49, 53], text='pyt'), Row(indices=[54, 61], text='baddie'), Row(indices=[62, 69], text='leaked'), Row(indices=[70, 76], text='oussy'), Row(indices=[77, 87], text='backshots'), Row(indices=[88, 103], text='sellingcontent'), Row(indices=[104, 115], text='schoolthot'), Row(indices=[116, 119], text='dm'), Row(indices=[120, 125], text='head'), Row(indices=[126, 137], text='throatgoat'), Row(indices=[138, 142], text='bbc'), Row(indices=[143, 150], text='sloppy'), Row(indices=[151, 157], text='leaks'), Row(indices=[158, 166], text='college'), Row(indices=[167, 172], text='nsfw'), Row(indices=[173, 179], text='ebony'), Row(indices=[180, 187], text='pyteen')], [Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/XKFQqhgNTN', expanded_url='https://twitter.com/freaky_central2/status/1586143745078353920/video/1', id=1586143673661677569, id_str='1586143673661677569', indices=[188, 211], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=592), medium=Row(h=1200, resize='fit', w=555), small=Row(h=680, resize='fit', w=315), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/XKFQqhgNTN', video_info=Row(aspect_ratio=[37, 80], duration_millis=27627, variants=[Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/592x1280/gjcYcZKo5e-ZYGGa.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/pl/LVnxWoY7O9mLNE5G.m3u8?tag=12&amp;container=fmp4'), Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/320x690/8hHnDD2IseH3kEhM.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/480x1036/RLYUVMasvBGnSSvI.mp4?tag=12')]))], [], [], []), ([Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/XKFQqhgNTN', expanded_url='https://twitter.com/freaky_central2/status/1586143745078353920/video/1', id=1586143673661677569, id_str='1586143673661677569', indices=[188, 211], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=592), medium=Row(h=1200, resize='fit', w=555), small=Row(h=680, resize='fit', w=315), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/XKFQqhgNTN', video_info=Row(aspect_ratio=[37, 80], duration_millis=27627, variants=[Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/592x1280/gjcYcZKo5e-ZYGGa.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/pl/LVnxWoY7O9mLNE5G.m3u8?tag=12&amp;container=fmp4'), Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/320x690/8hHnDD2IseH3kEhM.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/480x1036/RLYUVMasvBGnSSvI.mp4?tag=12')]))],), First 100 follows gets a surprise \\nOpen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot #dm #head #throatgoat #bbc #sloppy #leaks #college #nsfw #ebony #pyteen https://t.co/XKFQqhgNTN), 155, False, low, None, 1586143745078353920, 1586143745078353920, None, None, None, None, None, False, en, None, False, 0, None, None, None, None, 3, 16, False, None, &lt;a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\"&gt;Twitter for iPhone&lt;/a&gt;, First 100 follows gets a surprise \\nOpen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot… https://t.co/Vdd4JmZ8zn, True, (False, Tue Oct 25 19:35:23 +0000 2022, True, False, everything u need in one place; DM for credit or removal DAILY POST | FOLLOW FOR MORE| SELLING PERSONAL CONTENT DMS OPEN | DM FOR CHEAP PROMO #promo #ads, 82, 2627, 1, False, 1584991955146121217, 1584991955146121217, False, 12, None, FC🔞, F5F8FA, , , False, https://pbs.twimg.com/profile_banners/1584991955146121217/1666726989, http://pbs.twimg.com/profile_images/1584993944756256799/ZOWKxRRp_normal.jpg, https://pbs.twimg.com/profile_images/1584993944756256799/ZOWKxRRp_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, freaky_central2, 87, none, None, False, None, []), None)</td>\n",
       "      <td>first 100 follows gets a surprise \\nopen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot #dm #head #throatgoat #bbc #sloppy #leaks #college #nsfw #ebony #pyteen https://t.co/xkfqqhgntn</td>\n",
       "      <td>RT @freaky_central2: First 100 follows gets a surprise \\nOpen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot #d…</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sat Oct 29 18:38:43 +0000 2022</td>\n",
       "      <td>1586427325264310273</td>\n",
       "      <td>None</td>\n",
       "      <td>maoooo</td>\n",
       "      <td>6</td>\n",
       "      <td>False</td>\n",
       "      <td>None</td>\n",
       "      <td>None</td>\n",
       "      <td>2</td>\n",
       "      <td>10</td>\n",
       "      <td>(None, Sat Oct 29 18:23:54 +0000 2022, None, ([Row(indices=[37, 56], text='UniversityofTehran')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586423598645841922', indices=[117, 140], url='https://t.co/J61gZ0Gy9z')], []), None, ([0, 279], ([Row(indices=[37, 56], text='UniversityofTehran'), Row(indices=[267, 279], text='Mahsa_Amini')], None, [], [], []), None, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). Security forces waiting outside arresting and taking students away to undisclosed location #Mahsa_Amini), 12, False, low, None, 1586423598645841922, 1586423598645841922, None, None, None, None, None, True, en, None, None, 0, (None, Sat Oct 29 18:00:59 +0000 2022, None, ([Row(indices=[11, 23], text='دانشکده_فنی')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586417830228267009', indices=[117, 140], url='https://t.co/Fw2dvhchXt')], []), None, ([0, 279], Row(hashtags=[Row(indices=[11, 23], text='دانشکده_فنی'), Row(indices=[268, 279], text='مهسا_امینی')], media=None, symbols=[], urls=[], user_mentions=[]), None, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما مورد حمله نیروهای بسیج دانشجویی قرار گرفتند. نیروهای لباس شخصی به طور غیرقانونی از صبح در دانشگاه مستقر شده بودند و با بسیجی‌ها در این حمله همراه شدند.\\n#مهسا_امینی), 64, False, low, None, 1586417830228267009, 1586417830228267009, None, None, None, None, None, False, fa, None, None, 1, None, None, 1, 26, False, None, &lt;a href=\"https://mobile.twitter.com\" rel=\"nofollow\"&gt;Twitter Web App&lt;/a&gt;, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما… https://t.co/Fw2dvhchXt, True, (False, Tue Aug 11 07:17:35 +0000 2015, False, False, Official account of Iran Human Rights (IHR NGO)- Also visit @iranhr حساب رسمی توییتر سازمان حقوق بشر ایران, 1067, 9848, 284, True, 3414505683, 3414505683, False, 119, Oslo, Norway, Iran Human Rights (IHR NGO), 000000, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/3414505683/1665403153, http://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, https://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, 89C9FA, 000000, 000000, 000000, False, False, IHRights, 8255, none, https://iranhr.net/, True, None, []), None), 1586417830228267009, 1586417830228267009, (twitter.com/ihrights/statu…, https://twitter.com/ihrights/status/1586417830228267009, https://t.co/5MHIgxrjHW), 2, 10, False, None, &lt;a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\"&gt;Twitter for iPhone&lt;/a&gt;, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclo… https://t.co/J61gZ0Gy9z, True, (False, Sat May 02 13:32:40 +0000 2009, True, False, Director of NGO «Iran Human Rights\" @IHRights Professor at the University of Oslo- مدیر سازمان حقوق بشر ایران- پزشک و استاددانشکده پزشکی دانشگاه اسلو, 9660, 5590, 1642, True, 37196329, 37196329, False, 173, Global, Mahmood Amiry-Moghaddam, C0DEED, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/37196329/1425896261, http://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, https://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, iranhr, 11461, none, http://www.iranhr.net, True, None, []), None)</td>\n",
       "      <td>today’s peaceful student protests of #universityoftehran, college of engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). security forces waiting outside arresting and taking students away to undisclosed location #mahsa_amini</td>\n",
       "      <td>RT @iranhr: Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes securit…</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       created_at                   id geo_coordinates  \\\n",
       "0  Sat Oct 29 18:36:18 +0000 2022  1586426715760459778            None   \n",
       "1  Sat Oct 29 18:36:18 +0000 2022  1586426719090589696            None   \n",
       "2  Sat Oct 29 18:36:55 +0000 2022  1586426871524343814            None   \n",
       "3  Sat Oct 29 18:38:00 +0000 2022  1586427143302443008            None   \n",
       "4  Sat Oct 29 18:38:43 +0000 2022  1586427325264310273            None   \n",
       "\n",
       "     user_name  followers_count  verified_user    user_location  \\\n",
       "0   TitanUp Ty             2256          False             None   \n",
       "1     SickYooo               88          False             None   \n",
       "2       Saturn              883          False          America   \n",
       "3  💙KOHAKU225💙              127          False  Baton Rouge, LA   \n",
       "4       maoooo                6          False             None   \n",
       "\n",
       "                      user_description  reply_count  retweet_count  \\\n",
       "0  #LGM #TITANS #GBO🍊 #FJB #NJ #USA 🇺🇸            5             51   \n",
       "1                                   ..            3            168   \n",
       "2       #براندازم #نه_به_جمهوری_اسلامی            0              7   \n",
       "3                                 None            3             16   \n",
       "4                                 None            2             10   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     retweeted_status  \\\n",
       "0                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               (None, Sat Oct 29 13:22:46 +0000 2022, [0, 82], ([Row(indices=[49, 53], text='SEC'), Row(indices=[54, 70], text='CollegeFootball'), Row(indices=[71, 75], text='cfb'), Row(indices=[76, 82], text='NCAAF')], [Row(additional_media_info=None, description=None, display_url='pic.twitter.com/ra3EkYetqu', expanded_url='https://twitter.com/JWPSports/status/1586347814967132163/photo/1', id=1586347810932154368, id_str='1586347810932154368', indices=[83, 106], media_url='http://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', media_url_https='https://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', sizes=Row(large=Row(h=1159, resize='fit', w=1172), medium=Row(h=1159, resize='fit', w=1172), small=Row(h=672, resize='fit', w=680), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='photo', url='https://t.co/ra3EkYetqu')], [], [], []), ([Row(additional_media_info=None, description=None, display_url='pic.twitter.com/ra3EkYetqu', expanded_url='https://twitter.com/JWPSports/status/1586347814967132163/photo/1', id=1586347810932154368, id_str='1586347810932154368', indices=[83, 106], media_url='http://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', media_url_https='https://pbs.twimg.com/media/FgPW7eDUYAAPV17.jpg', sizes=Row(large=Row(h=1159, resize='fit', w=1172), medium=Row(h=1159, resize='fit', w=1172), small=Row(h=672, resize='fit', w=680), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='photo', url='https://t.co/ra3EkYetqu', video_info=None)],), None, 402, False, low, None, 1586347814967132163, 1586347814967132163, None, None, None, None, None, False, en, None, False, 3, None, None, None, None, 5, 51, False, None, <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>, Your current SEC College Football QBR leaders.👀\\n\\n#SEC #CollegeFootball #cfb #NCAAF https://t.co/ra3EkYetqu, False, (False, Fri Jan 31 15:24:06 +0000 2020, True, False, JWP Sports! Be sure to find me on Instagram @jwp_sports | @cfbalerts only on Instagram. Connect with me on all platforms ⬇️, 1935, 4414, 1857, True, 1223265683565416448, 1223265683565416448, False, 6, None, JWP Sports , F5F8FA, , , False, None, http://pbs.twimg.com/profile_images/1225762639957106688/k75frUmv_normal.jpg, https://pbs.twimg.com/profile_images/1225762639957106688/k75frUmv_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, JWPSports, 1623, none, https://linktr.ee/jwpsports, False, None, []), None)   \n",
       "1                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                              (None, Sun Feb 13 19:58:23 +0000 2022, [0, 140], ([Row(indices=[52, 66], text='collegebaddie'), Row(indices=[67, 75], text='college'), Row(indices=[76, 87], text='latinahead'), Row(indices=[88, 102], text='latinaexposed')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1492951306398355456', indices=[104, 127], url='https://t.co/tLxG8VA9ih')], []), None, ([0, 155], ([Row(indices=[52, 66], text='collegebaddie'), Row(indices=[67, 75], text='college'), Row(indices=[76, 87], text='latinahead'), Row(indices=[88, 102], text='latinaexposed'), Row(indices=[103, 116], text='LatinaBeauty'), Row(indices=[117, 130], text='sloppythroat'), Row(indices=[131, 147], text='sloppyheadGIVER'), Row(indices=[148, 155], text='blowie')], [Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/GhG6abUn99', expanded_url='https://twitter.com/pxrnstachee/status/1492951306398355456/video/1', id=1492951256318431232, id_str='1492951256318431232', indices=[156, 179], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=732), medium=Row(h=1200, resize='fit', w=686), small=Row(h=680, resize='fit', w=389), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/GhG6abUn99', video_info=Row(aspect_ratio=[183, 320], duration_millis=45001, variants=[Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/320x558/hDDxsZ1Kgt_l7sg3.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/pl/np_2fWZzuV1YYGWs.m3u8?tag=12&container=fmp4'), Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/720x1258/OSWvqvJs35tJrRRC.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/480x838/ZqGt4Qzk4jREnItM.mp4?tag=12')]))], [], [], []), ([Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/GhG6abUn99', expanded_url='https://twitter.com/pxrnstachee/status/1492951306398355456/video/1', id=1492951256318431232, id_str='1492951256318431232', indices=[156, 179], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1492951256318431232/pu/img/PeBtNX6QBgdDQZHX.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=732), medium=Row(h=1200, resize='fit', w=686), small=Row(h=680, resize='fit', w=389), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/GhG6abUn99', video_info=Row(aspect_ratio=[183, 320], duration_millis=45001, variants=[Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/320x558/hDDxsZ1Kgt_l7sg3.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/pl/np_2fWZzuV1YYGWs.m3u8?tag=12&container=fmp4'), Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/720x1258/OSWvqvJs35tJrRRC.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1492951256318431232/pu/vid/480x838/ZqGt4Qzk4jREnItM.mp4?tag=12')]))],), Latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed #LatinaBeauty #sloppythroat #sloppyheadGIVER #blowie https://t.co/GhG6abUn99), 1275, False, low, None, 1492951306398355456, 1492951306398355456, None, None, None, None, None, False, en, None, True, 0, None, None, None, None, 3, 168, False, None, <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>, Latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed… https://t.co/tLxG8VA9ih, True, (False, Thu May 10 01:13:52 +0000 2018, True, False, | 18+ Daily Content | 🌹No Content Is Owned By The Maker Of This Account🌹| Dm for credit/removal no copyright infringement intended, 8017, 64513, 45, False, 994384995740352512, 994384995740352512, False, 216, None, ParadisePxrn (dm for cheap promo), F5F8FA, , , False, https://pbs.twimg.com/profile_banners/994384995740352512/1666345959, http://pbs.twimg.com/profile_images/1583363185045667840/IYjjH-zG_normal.jpg, https://pbs.twimg.com/profile_images/1583363185045667840/IYjjH-zG_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, pxrnstachee, 565, none, https://discord.gg/sullen, False, None, []), None)   \n",
       "2                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                (None, Sat Oct 29 18:23:54 +0000 2022, None, ([Row(indices=[37, 56], text='UniversityofTehran')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586423598645841922', indices=[117, 140], url='https://t.co/J61gZ0Gy9z')], []), None, ([0, 279], ([Row(indices=[37, 56], text='UniversityofTehran'), Row(indices=[267, 279], text='Mahsa_Amini')], None, [], [], []), None, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). Security forces waiting outside arresting and taking students away to undisclosed location #Mahsa_Amini), 11, False, low, None, 1586423598645841922, 1586423598645841922, None, None, None, None, None, True, en, None, None, 0, (None, Sat Oct 29 18:00:59 +0000 2022, None, ([Row(indices=[11, 23], text='دانشکده_فنی')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586417830228267009', indices=[117, 140], url='https://t.co/Fw2dvhchXt')], []), None, ([0, 279], Row(hashtags=[Row(indices=[11, 23], text='دانشکده_فنی'), Row(indices=[268, 279], text='مهسا_امینی')], media=None, symbols=[], urls=[], user_mentions=[]), None, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما مورد حمله نیروهای بسیج دانشجویی قرار گرفتند. نیروهای لباس شخصی به طور غیرقانونی از صبح در دانشگاه مستقر شده بودند و با بسیجی‌ها در این حمله همراه شدند.\\n#مهسا_امینی), 63, False, low, None, 1586417830228267009, 1586417830228267009, None, None, None, None, None, False, fa, None, None, 1, None, None, 1, 25, False, None, <a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما… https://t.co/Fw2dvhchXt, True, (False, Tue Aug 11 07:17:35 +0000 2015, False, False, Official account of Iran Human Rights (IHR NGO)- Also visit @iranhr حساب رسمی توییتر سازمان حقوق بشر ایران, 1067, 9849, 284, True, 3414505683, 3414505683, False, 120, Oslo, Norway, Iran Human Rights (IHR NGO), 000000, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/3414505683/1665403153, http://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, https://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, 89C9FA, 000000, 000000, 000000, False, False, IHRights, 8255, none, https://iranhr.net/, True, None, []), None), 1586417830228267009, 1586417830228267009, (twitter.com/ihrights/statu…, https://twitter.com/ihrights/status/1586417830228267009, https://t.co/5MHIgxrjHW), 0, 7, False, None, <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclo… https://t.co/J61gZ0Gy9z, True, (False, Sat May 02 13:32:40 +0000 2009, True, False, Director of NGO «Iran Human Rights\" @IHRights Professor at the University of Oslo- مدیر سازمان حقوق بشر ایران- پزشک و استاددانشکده پزشکی دانشگاه اسلو, 9660, 5589, 1642, True, 37196329, 37196329, False, 173, Global, Mahmood Amiry-Moghaddam, C0DEED, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/37196329/1425896261, http://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, https://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, iranhr, 11461, none, http://www.iranhr.net, True, None, []), None)   \n",
       "3  (None, Fri Oct 28 23:51:52 +0000 2022, [0, 140], ([Row(indices=[45, 48], text='bj'), Row(indices=[49, 53], text='pyt'), Row(indices=[54, 61], text='baddie'), Row(indices=[62, 69], text='leaked'), Row(indices=[70, 76], text='oussy'), Row(indices=[77, 87], text='backshots'), Row(indices=[88, 103], text='sellingcontent'), Row(indices=[104, 115], text='schoolthot')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586143745078353920', indices=[117, 140], url='https://t.co/Vdd4JmZ8zn')], []), None, ([0, 187], ([Row(indices=[45, 48], text='bj'), Row(indices=[49, 53], text='pyt'), Row(indices=[54, 61], text='baddie'), Row(indices=[62, 69], text='leaked'), Row(indices=[70, 76], text='oussy'), Row(indices=[77, 87], text='backshots'), Row(indices=[88, 103], text='sellingcontent'), Row(indices=[104, 115], text='schoolthot'), Row(indices=[116, 119], text='dm'), Row(indices=[120, 125], text='head'), Row(indices=[126, 137], text='throatgoat'), Row(indices=[138, 142], text='bbc'), Row(indices=[143, 150], text='sloppy'), Row(indices=[151, 157], text='leaks'), Row(indices=[158, 166], text='college'), Row(indices=[167, 172], text='nsfw'), Row(indices=[173, 179], text='ebony'), Row(indices=[180, 187], text='pyteen')], [Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/XKFQqhgNTN', expanded_url='https://twitter.com/freaky_central2/status/1586143745078353920/video/1', id=1586143673661677569, id_str='1586143673661677569', indices=[188, 211], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=592), medium=Row(h=1200, resize='fit', w=555), small=Row(h=680, resize='fit', w=315), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/XKFQqhgNTN', video_info=Row(aspect_ratio=[37, 80], duration_millis=27627, variants=[Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/592x1280/gjcYcZKo5e-ZYGGa.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/pl/LVnxWoY7O9mLNE5G.m3u8?tag=12&container=fmp4'), Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/320x690/8hHnDD2IseH3kEhM.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/480x1036/RLYUVMasvBGnSSvI.mp4?tag=12')]))], [], [], []), ([Row(additional_media_info=Row(description=None, embeddable=None, monetizable=False, title=None), description=None, display_url='pic.twitter.com/XKFQqhgNTN', expanded_url='https://twitter.com/freaky_central2/status/1586143745078353920/video/1', id=1586143673661677569, id_str='1586143673661677569', indices=[188, 211], media_url='http://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', media_url_https='https://pbs.twimg.com/ext_tw_video_thumb/1586143673661677569/pu/img/5hJNsSHYfpGRBKXF.jpg', sizes=Row(large=Row(h=1280, resize='fit', w=592), medium=Row(h=1200, resize='fit', w=555), small=Row(h=680, resize='fit', w=315), thumb=Row(h=150, resize='crop', w=150)), source_status_id=None, source_status_id_str=None, source_user_id=None, source_user_id_str=None, type='video', url='https://t.co/XKFQqhgNTN', video_info=Row(aspect_ratio=[37, 80], duration_millis=27627, variants=[Row(bitrate=2176000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/592x1280/gjcYcZKo5e-ZYGGa.mp4?tag=12'), Row(bitrate=None, content_type='application/x-mpegURL', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/pl/LVnxWoY7O9mLNE5G.m3u8?tag=12&container=fmp4'), Row(bitrate=632000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/320x690/8hHnDD2IseH3kEhM.mp4?tag=12'), Row(bitrate=950000, content_type='video/mp4', url='https://video.twimg.com/ext_tw_video/1586143673661677569/pu/vid/480x1036/RLYUVMasvBGnSSvI.mp4?tag=12')]))],), First 100 follows gets a surprise \\nOpen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot #dm #head #throatgoat #bbc #sloppy #leaks #college #nsfw #ebony #pyteen https://t.co/XKFQqhgNTN), 155, False, low, None, 1586143745078353920, 1586143745078353920, None, None, None, None, None, False, en, None, False, 0, None, None, None, None, 3, 16, False, None, <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>, First 100 follows gets a surprise \\nOpen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot… https://t.co/Vdd4JmZ8zn, True, (False, Tue Oct 25 19:35:23 +0000 2022, True, False, everything u need in one place; DM for credit or removal DAILY POST | FOLLOW FOR MORE| SELLING PERSONAL CONTENT DMS OPEN | DM FOR CHEAP PROMO #promo #ads, 82, 2627, 1, False, 1584991955146121217, 1584991955146121217, False, 12, None, FC🔞, F5F8FA, , , False, https://pbs.twimg.com/profile_banners/1584991955146121217/1666726989, http://pbs.twimg.com/profile_images/1584993944756256799/ZOWKxRRp_normal.jpg, https://pbs.twimg.com/profile_images/1584993944756256799/ZOWKxRRp_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, freaky_central2, 87, none, None, False, None, []), None)   \n",
       "4                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               (None, Sat Oct 29 18:23:54 +0000 2022, None, ([Row(indices=[37, 56], text='UniversityofTehran')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586423598645841922', indices=[117, 140], url='https://t.co/J61gZ0Gy9z')], []), None, ([0, 279], ([Row(indices=[37, 56], text='UniversityofTehran'), Row(indices=[267, 279], text='Mahsa_Amini')], None, [], [], []), None, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). Security forces waiting outside arresting and taking students away to undisclosed location #Mahsa_Amini), 12, False, low, None, 1586423598645841922, 1586423598645841922, None, None, None, None, None, True, en, None, None, 0, (None, Sat Oct 29 18:00:59 +0000 2022, None, ([Row(indices=[11, 23], text='دانشکده_فنی')], None, [], [Row(display_url='twitter.com/i/web/status/1…', expanded_url='https://twitter.com/i/web/status/1586417830228267009', indices=[117, 140], url='https://t.co/Fw2dvhchXt')], []), None, ([0, 279], Row(hashtags=[Row(indices=[11, 23], text='دانشکده_فنی'), Row(indices=[268, 279], text='مهسا_امینی')], media=None, symbols=[], urls=[], user_mentions=[]), None, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما مورد حمله نیروهای بسیج دانشجویی قرار گرفتند. نیروهای لباس شخصی به طور غیرقانونی از صبح در دانشگاه مستقر شده بودند و با بسیجی‌ها در این حمله همراه شدند.\\n#مهسا_امینی), 64, False, low, None, 1586417830228267009, 1586417830228267009, None, None, None, None, None, False, fa, None, None, 1, None, None, 1, 26, False, None, <a href=\"https://mobile.twitter.com\" rel=\"nofollow\">Twitter Web App</a>, ۱/امروز در #دانشکده_فنی دانشگاه تهران، دانشجویان با تجمع مسالمت‌آمیز، شعار دادند و خواسته‌های خود را مطرح کردند اما… https://t.co/Fw2dvhchXt, True, (False, Tue Aug 11 07:17:35 +0000 2015, False, False, Official account of Iran Human Rights (IHR NGO)- Also visit @iranhr حساب رسمی توییتر سازمان حقوق بشر ایران, 1067, 9848, 284, True, 3414505683, 3414505683, False, 119, Oslo, Norway, Iran Human Rights (IHR NGO), 000000, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/3414505683/1665403153, http://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, https://pbs.twimg.com/profile_images/631002652361424896/NWkvQXd-_normal.png, 89C9FA, 000000, 000000, 000000, False, False, IHRights, 8255, none, https://iranhr.net/, True, None, []), None), 1586417830228267009, 1586417830228267009, (twitter.com/ihrights/statu…, https://twitter.com/ihrights/status/1586417830228267009, https://t.co/5MHIgxrjHW), 2, 10, False, None, <a href=\"http://twitter.com/download/iphone\" rel=\"nofollow\">Twitter for iPhone</a>, Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclo… https://t.co/J61gZ0Gy9z, True, (False, Sat May 02 13:32:40 +0000 2009, True, False, Director of NGO «Iran Human Rights\" @IHRights Professor at the University of Oslo- مدیر سازمان حقوق بشر ایران- پزشک و استاددانشکده پزشکی دانشگاه اسلو, 9660, 5590, 1642, True, 37196329, 37196329, False, 173, Global, Mahmood Amiry-Moghaddam, C0DEED, http://abs.twimg.com/images/themes/theme1/bg.png, https://abs.twimg.com/images/themes/theme1/bg.png, False, https://pbs.twimg.com/profile_banners/37196329/1425896261, http://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, https://pbs.twimg.com/profile_images/1306244699716489216/BfQaxpEW_normal.jpg, 1DA1F2, C0DEED, DDEEF6, 333333, True, False, iranhr, 11461, none, http://www.iranhr.net, True, None, []), None)   \n",
       "\n",
       "                                                                                                                                                                                                                                                                                tweet_text  \\\n",
       "0                                                                                                                                                                             your current sec college football qbr leaders.👀\\n\\n#sec #collegefootball #cfb #ncaaf https://t.co/ra3ekyetqu   \n",
       "1                                                                                                      latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed #latinabeauty #sloppythroat #sloppyheadgiver #blowie https://t.co/ghg6abun99   \n",
       "2  today’s peaceful student protests of #universityoftehran, college of engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). security forces waiting outside arresting and taking students away to undisclosed location #mahsa_amini   \n",
       "3                                                                   first 100 follows gets a surprise \\nopen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot #dm #head #throatgoat #bbc #sloppy #leaks #college #nsfw #ebony #pyteen https://t.co/xkfqqhgntn   \n",
       "4  today’s peaceful student protests of #universityoftehran, college of engineering was attacked by basij and plainclothes security forces (who had illegally entered the campus). security forces waiting outside arresting and taking students away to undisclosed location #mahsa_amini   \n",
       "\n",
       "                                                                                                                                              text  \n",
       "0                      RT @JWPSports: Your current SEC College Football QBR leaders.👀\\n\\n#SEC #CollegeFootball #cfb #NCAAF https://t.co/ra3EkYetqu  \n",
       "1     RT @pxrnstachee: Latina baddie wanted to suck me off during a party😫 #collegebaddie #college #latinahead #latinaexposed #LatinaBeauty #slop…  \n",
       "2     RT @iranhr: Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes securit…  \n",
       "3  RT @freaky_central2: First 100 follows gets a surprise \\nOpen dms\\n\\n#bj #pyt #baddie #leaked #oussy #backshots #sellingcontent #schoolthot #d…  \n",
       "4     RT @iranhr: Today’s peaceful student protests of #UniversityofTehran, College of Engineering was attacked by basij and plainclothes securit…  "
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_coords_analysis_df = twitter_df.select([\n",
    "    twitter_df.created_at,\n",
    "    twitter_df.id,\n",
    "    twitter_df.geo.coordinates.alias(\"geo_coordinates\"),\n",
    "    twitter_df.user['name'].alias(\"user_name\"), \n",
    "    twitter_df.user.followers_count.alias(\"followers_count\"), \n",
    "    twitter_df.user.verified.alias(\"verified_user\"),\n",
    "    twitter_df.user.location.alias(\"user_location\"),\n",
    "    twitter_df.user.description.alias(\"user_description\"),\n",
    "    twitter_df.retweeted_status.reply_count.alias(\"reply_count\"),\n",
    "    twitter_df.retweeted_status.retweet_count.alias(\"retweet_count\"),\n",
    "    twitter_df.retweeted_status.alias(\"retweeted_status\"),\n",
    "    twitter_df.tweet_text,\n",
    "    twitter_df.text,\n",
    "])itter\n",
    "\n",
    "df_coords_analysis_df.limit(5).toPandas()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "9d0845ed-b052-4f36-bbe8-b5931c90f187",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    }
   ],
   "source": [
    "#Taking the null count to check for any discrepancies\n",
    "df_nuls_cnt = df_coords_analysis_df.select([\n",
    "                F.count(\n",
    "                    F.when(df_coords_analysis_df[column].isNull(), column)\n",
    "                ).alias(column) for column in df_coords_analysis_df.columns\n",
    "            ]).toPandas()\n",
    "\n",
    "# display.max_columns : int"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "da2788bb-fc3c-4f2f-bd75-8f19939a3dd5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>created_at</th>\n",
       "      <th>id</th>\n",
       "      <th>geo_coordinates</th>\n",
       "      <th>user_name</th>\n",
       "      <th>followers_count</th>\n",
       "      <th>verified_user</th>\n",
       "      <th>user_location</th>\n",
       "      <th>user_description</th>\n",
       "      <th>reply_count</th>\n",
       "      <th>retweet_count</th>\n",
       "      <th>retweeted_status</th>\n",
       "      <th>tweet_text</th>\n",
       "      <th>text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>997584</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>451915</td>\n",
       "      <td>231810</td>\n",
       "      <td>352608</td>\n",
       "      <td>352608</td>\n",
       "      <td>352608</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   created_at  id  geo_coordinates  user_name  followers_count  verified_user  \\\n",
       "0           0   0           997584          0                0              0   \n",
       "\n",
       "   user_location  user_description  reply_count  retweet_count  \\\n",
       "0         451915            231810       352608         352608   \n",
       "\n",
       "   retweeted_status  tweet_text  text  \n",
       "0            352608           0     0  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_nuls_cnt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "ef04a00c-97f1-4abf-92e0-7e90c7cf169b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "                                                                                \r"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CPU times: user 240 ms, sys: 30.4 ms, total: 270 ms\n",
      "Wall time: 3min 46s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "df_coords_analysis_df.write.mode(\"overwrite\").parquet(\"gs://msca-bdp-students-bucket/shared_data/kishorkumarreddy/df_coords_analysis_df/\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d0c0339e-bc48-482a-9210-b6bb2fa54272",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}