scrape_pushshift.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "toxic-frontier",
   "metadata": {},
   "source": [
    "Reference: https://medium.com/swlh/how-to-scrape-large-amounts-of-reddit-data-using-pushshift-1d33bde9286\n",
    "\n",
    "preresiquite: `!pip install pmaw`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "leading-opening",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pmaw import PushshiftAPI\n",
    "from scrape_reddit import *\n",
    "\n",
    "api = PushshiftAPI()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "victorian-commitment",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total:: Success Rate: 50.00% - Requests: 20 - Batches: 2 - Items Remaining: 0\n",
      "Retrieved 1000 submissions from Pushshift\n"
     ]
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 1000\n",
    "\n",
    "before_date = '2021-02-01'\n",
    "after_date = '2021-01-01'\n",
    "\n",
    "submissions_df_test = scrape(subreddit, limit, before_date, after_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "responsible-contamination",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(selected_columns(submissions_df_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "unusual-friendship",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "victorian-adoption",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102 results available in Pushshift\n",
      "Total:: Success Rate: 100.00% - Requests: 10 - Batches: 1 - Items Remaining: 0\n",
      "Retrieved 102 submissions from Pushshift\n"
     ]
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2009-01-01'\n",
    "before_date = '2010-01-01'\n",
    "\n",
    "\n",
    "submissions_df_test = scrape(subreddit, limit, before_date, after_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "wired-tracker",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Eliminate Your Debt using the Snowball Method</td>\n",
       "      <td>drbonemonkey</td>\n",
       "      <td></td>\n",
       "      <td>http://www.infobarrel.com/How_To_Use_the_Snowb...</td>\n",
       "      <td>5</td>\n",
       "      <td>1240169990</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Social Worker saves 1.4 million then donates i...</td>\n",
       "      <td>gen3ric</td>\n",
       "      <td></td>\n",
       "      <td>http://www.stltoday.com/stltoday/news/stories....</td>\n",
       "      <td>2</td>\n",
       "      <td>1239819290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ways &amp;amp; Means</td>\n",
       "      <td>gen3ric</td>\n",
       "      <td></td>\n",
       "      <td>http://www.getrichslowly.org/blog/2009/04/13/t...</td>\n",
       "      <td>1</td>\n",
       "      <td>1239654613</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Debt Settlement Process</td>\n",
       "      <td>SES07</td>\n",
       "      <td></td>\n",
       "      <td>http://firstchoicedebtresolution.wordpress.com...</td>\n",
       "      <td>1</td>\n",
       "      <td>1239647671</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Bankruptcy Means Test</td>\n",
       "      <td>SES07</td>\n",
       "      <td></td>\n",
       "      <td>http://firstchoicedebtresolution.wordpress.com...</td>\n",
       "      <td>1</td>\n",
       "      <td>1239111926</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>97</th>\n",
       "      <td>CCDB - Credit Card Database</td>\n",
       "      <td>mitya001</td>\n",
       "      <td></td>\n",
       "      <td>http://www.ccdb.com/</td>\n",
       "      <td>0</td>\n",
       "      <td>1261530736</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>How To: Save Money on Holiday Shopping with Yo...</td>\n",
       "      <td>kevinweb55</td>\n",
       "      <td></td>\n",
       "      <td>http://creditshout.com/how-to-save-money-on-ho...</td>\n",
       "      <td>0</td>\n",
       "      <td>1259725319</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>Llegan Los Préstamos Rápidos</td>\n",
       "      <td>maxpow01</td>\n",
       "      <td></td>\n",
       "      <td>http://www.articuloz.com/prestamos-articulos/l...</td>\n",
       "      <td>0</td>\n",
       "      <td>1259659213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>Staying Out of Debt for the Holidays</td>\n",
       "      <td>mgagen</td>\n",
       "      <td></td>\n",
       "      <td>http://www.selfgrowthengine.com/content/stayin...</td>\n",
       "      <td>0</td>\n",
       "      <td>1259597059</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>A Guide to Surviving Black Friday 2009</td>\n",
       "      <td>kevinweb55</td>\n",
       "      <td></td>\n",
       "      <td>http://creditshout.com/a-guide-to-surviving-bl...</td>\n",
       "      <td>0</td>\n",
       "      <td>1259175906</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>102 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 title        author selftext  \\\n",
       "0        Eliminate Your Debt using the Snowball Method  drbonemonkey            \n",
       "1    Social Worker saves 1.4 million then donates i...       gen3ric            \n",
       "2                                     Ways &amp; Means       gen3ric            \n",
       "3                              Debt Settlement Process         SES07            \n",
       "4                                Bankruptcy Means Test         SES07            \n",
       "..                                                 ...           ...      ...   \n",
       "97                         CCDB - Credit Card Database      mitya001            \n",
       "98   How To: Save Money on Holiday Shopping with Yo...    kevinweb55            \n",
       "99                        Llegan Los Préstamos Rápidos      maxpow01            \n",
       "100               Staying Out of Debt for the Holidays        mgagen            \n",
       "101             A Guide to Surviving Black Friday 2009    kevinweb55            \n",
       "\n",
       "                                                   url  score  created_utc  \n",
       "0    http://www.infobarrel.com/How_To_Use_the_Snowb...      5   1240169990  \n",
       "1    http://www.stltoday.com/stltoday/news/stories....      2   1239819290  \n",
       "2    http://www.getrichslowly.org/blog/2009/04/13/t...      1   1239654613  \n",
       "3    http://firstchoicedebtresolution.wordpress.com...      1   1239647671  \n",
       "4    http://firstchoicedebtresolution.wordpress.com...      1   1239111926  \n",
       "..                                                 ...    ...          ...  \n",
       "97                                http://www.ccdb.com/      0   1261530736  \n",
       "98   http://creditshout.com/how-to-save-money-on-ho...      0   1259725319  \n",
       "99   http://www.articuloz.com/prestamos-articulos/l...      0   1259659213  \n",
       "100  http://www.selfgrowthengine.com/content/stayin...      0   1259597059  \n",
       "101  http://creditshout.com/a-guide-to-surviving-bl...      0   1259175906  \n",
       "\n",
       "[102 rows x 6 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns(submissions_df_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "considerable-refrigerator",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['author', 'author_created_utc', 'author_flair_background_color',\n",
       "       'author_flair_css_class', 'author_flair_richtext', 'author_flair_text',\n",
       "       'author_flair_text_color', 'author_flair_type', 'author_fullname',\n",
       "       'brand_safe', 'contest_mode', 'created_utc', 'domain', 'full_link',\n",
       "       'gilded', 'id', 'is_crosspostable', 'is_reddit_media_domain', 'is_self',\n",
       "       'is_video', 'link_flair_richtext', 'link_flair_text_color',\n",
       "       'link_flair_type', 'locked', 'media_embed', 'no_follow', 'num_comments',\n",
       "       'num_crossposts', 'over_18', 'parent_whitelist_status', 'permalink',\n",
       "       'retrieved_on', 'rte_mode', 'score', 'secure_media_embed', 'selftext',\n",
       "       'send_replies', 'spoiler', 'stickied', 'subreddit', 'subreddit_id',\n",
       "       'subreddit_type', 'suggested_sort', 'thumbnail', 'title', 'url',\n",
       "       'whitelist_status'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submissions_df_test.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "every-cardiff",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0      https://www.reddit.com/r/personalfinance/comme...\n",
       "1      https://www.reddit.com/r/personalfinance/comme...\n",
       "2      https://www.reddit.com/r/personalfinance/comme...\n",
       "3      https://www.reddit.com/r/personalfinance/comme...\n",
       "4      https://www.reddit.com/r/personalfinance/comme...\n",
       "                             ...                        \n",
       "97     https://www.reddit.com/r/personalfinance/comme...\n",
       "98     https://www.reddit.com/r/personalfinance/comme...\n",
       "99     https://www.reddit.com/r/personalfinance/comme...\n",
       "100    https://www.reddit.com/r/personalfinance/comme...\n",
       "101    https://www.reddit.com/r/personalfinance/comme...\n",
       "Name: full_link, Length: 102, dtype: object"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submissions_df_test['full_link']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "subsequent-executive",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "967 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 31.58% - Requests: 38 - Batches: 10 - Items Remaining: 78\n",
      "Total:: Success Rate: 33.33% - Requests: 39 - Batches: 11 - Items Remaining: 0\n",
      "Retrieved 967 submissions from Pushshift\n"
     ]
    }
   ],
   "source": [
    "subreddit = \"investing\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2009-01-01'\n",
    "before_date = '2010-01-01'\n",
    "\n",
    "\n",
    "submissions_investing_2009 = scrape(subreddit, limit, before_date, after_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "multiple-drinking",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Hey investing reddit, what's your best investm...</td>\n",
       "      <td>EggplantWizard</td>\n",
       "      <td></td>\n",
       "      <td>https://www.reddit.com/r/investing/comments/7v...</td>\n",
       "      <td>8</td>\n",
       "      <td>1233938258</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Pre-Market Stock News Alerts : NASDAQ</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>http://stock-promoter.thesubway.com/thesubway-...</td>\n",
       "      <td>0</td>\n",
       "      <td>1233839415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Best of Investing School - Jan 2009</td>\n",
       "      <td>dning</td>\n",
       "      <td></td>\n",
       "      <td>http://investing-school.com/best-of/best-of-in...</td>\n",
       "      <td>6</td>\n",
       "      <td>1233770455</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Buy and Hold Equates to Buy and Hope</td>\n",
       "      <td>joka86</td>\n",
       "      <td></td>\n",
       "      <td>http://stock-promoter.thesubway.com/market-101...</td>\n",
       "      <td>2</td>\n",
       "      <td>1233765977</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>How to Create And Test a Trading Strategy, Par...</td>\n",
       "      <td>strafefire</td>\n",
       "      <td></td>\n",
       "      <td>http://yousuckattrading.com/2009/02/how-to-cre...</td>\n",
       "      <td>2</td>\n",
       "      <td>1233713737</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>962</th>\n",
       "      <td>More examples of how suspended 401(k) match or...</td>\n",
       "      <td>itsonlymoney</td>\n",
       "      <td></td>\n",
       "      <td>http://blog.oregonlive.com/finance/2009/07/how...</td>\n",
       "      <td>0</td>\n",
       "      <td>1246891158</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>963</th>\n",
       "      <td>What happens to your nest egg if your pension ...</td>\n",
       "      <td>itsonlymoney</td>\n",
       "      <td></td>\n",
       "      <td>http://blog.oregonlive.com/finance/2009/07/_th...</td>\n",
       "      <td>0</td>\n",
       "      <td>1246889262</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>964</th>\n",
       "      <td>Graceway Pharmaceuticals Acquires Early-stage ...</td>\n",
       "      <td>Sevenduolux</td>\n",
       "      <td></td>\n",
       "      <td>http://www.equitygroups.com/nyse/pfe/messages/...</td>\n",
       "      <td>0</td>\n",
       "      <td>1246889203</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>965</th>\n",
       "      <td>Have You Ever Made A Bad Investment? | Investm...</td>\n",
       "      <td>kakahelmy</td>\n",
       "      <td></td>\n",
       "      <td>http://investmentstax.com/have-you-ever-made-a...</td>\n",
       "      <td>1</td>\n",
       "      <td>1246711464</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>966</th>\n",
       "      <td>A Commodity Warning Sign</td>\n",
       "      <td>democracy101</td>\n",
       "      <td></td>\n",
       "      <td>http://dailyreckoning.com/a-commodity-warning-...</td>\n",
       "      <td>1</td>\n",
       "      <td>1246648794</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>967 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 title          author  \\\n",
       "0    Hey investing reddit, what's your best investm...  EggplantWizard   \n",
       "1               Pre-Market Stock News Alerts : NASDAQ        [deleted]   \n",
       "2                  Best of Investing School - Jan 2009           dning   \n",
       "3                 Buy and Hold Equates to Buy and Hope          joka86   \n",
       "4    How to Create And Test a Trading Strategy, Par...      strafefire   \n",
       "..                                                 ...             ...   \n",
       "962  More examples of how suspended 401(k) match or...    itsonlymoney   \n",
       "963  What happens to your nest egg if your pension ...    itsonlymoney   \n",
       "964  Graceway Pharmaceuticals Acquires Early-stage ...     Sevenduolux   \n",
       "965  Have You Ever Made A Bad Investment? | Investm...       kakahelmy   \n",
       "966                           A Commodity Warning Sign    democracy101   \n",
       "\n",
       "      selftext                                                url  score  \\\n",
       "0               https://www.reddit.com/r/investing/comments/7v...      8   \n",
       "1    [deleted]  http://stock-promoter.thesubway.com/thesubway-...      0   \n",
       "2               http://investing-school.com/best-of/best-of-in...      6   \n",
       "3               http://stock-promoter.thesubway.com/market-101...      2   \n",
       "4               http://yousuckattrading.com/2009/02/how-to-cre...      2   \n",
       "..         ...                                                ...    ...   \n",
       "962             http://blog.oregonlive.com/finance/2009/07/how...      0   \n",
       "963             http://blog.oregonlive.com/finance/2009/07/_th...      0   \n",
       "964             http://www.equitygroups.com/nyse/pfe/messages/...      0   \n",
       "965             http://investmentstax.com/have-you-ever-made-a...      1   \n",
       "966             http://dailyreckoning.com/a-commodity-warning-...      1   \n",
       "\n",
       "     created_utc  \n",
       "0     1233938258  \n",
       "1     1233839415  \n",
       "2     1233770455  \n",
       "3     1233765977  \n",
       "4     1233713737  \n",
       "..           ...  \n",
       "962   1246891158  \n",
       "963   1246889262  \n",
       "964   1246889203  \n",
       "965   1246711464  \n",
       "966   1246648794  \n",
       "\n",
       "[967 rows x 6 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns(submissions_investing_2009)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "fifteen-easter",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "259 results available in Pushshift\n",
      "Total:: Success Rate: 100.00% - Requests: 10 - Batches: 1 - Items Remaining: 0\n",
      "Retrieved 259 submissions from Pushshift\n"
     ]
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2010-01-01'\n",
    "before_date = '2011-01-01'\n",
    "\n",
    "\n",
    "submissions_pf_2010 = scrape(subreddit, limit, before_date, after_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "nearby-billion",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>What are your best personal finance videos ?</td>\n",
       "      <td>killermouse0</td>\n",
       "      <td></td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>5</td>\n",
       "      <td>1271604886</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>He got out of debt... You can too !</td>\n",
       "      <td>killermouse0</td>\n",
       "      <td></td>\n",
       "      <td>http://www.getrichslowly.org/blog/2010/04/11/r...</td>\n",
       "      <td>3</td>\n",
       "      <td>1271009625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How I Made the Most Money I've Ever Made in My...</td>\n",
       "      <td>crappycap</td>\n",
       "      <td></td>\n",
       "      <td>http://www.stopbuyingcrap.com/even-more-rambli...</td>\n",
       "      <td>3</td>\n",
       "      <td>1270646837</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What Are Your Rights as a Credit Card Customer?</td>\n",
       "      <td>kevinweb55</td>\n",
       "      <td></td>\n",
       "      <td>http://creditshout.com/what-are-your-rights-as...</td>\n",
       "      <td>3</td>\n",
       "      <td>1270146984</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Young Will Pay Up To 17% More In Premiums For ...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>http://www.msnbc.msn.com/id/36087653/ns/health...</td>\n",
       "      <td>0</td>\n",
       "      <td>1270140616</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>254</th>\n",
       "      <td>10 Ways to Ruin Your Retirement</td>\n",
       "      <td>mattymomostl</td>\n",
       "      <td></td>\n",
       "      <td>http://finance.yahoo.com/news/10-Ways-to-Ruin-...</td>\n",
       "      <td>5</td>\n",
       "      <td>1281448075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>255</th>\n",
       "      <td>Spending less will make you happier</td>\n",
       "      <td>mattymomostl</td>\n",
       "      <td></td>\n",
       "      <td>http://finance.yahoo.com/family-home/article/1...</td>\n",
       "      <td>2</td>\n",
       "      <td>1281446316</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>256</th>\n",
       "      <td>75 Phone Apps + 10 Tips that Will Save You Money</td>\n",
       "      <td>financialhighway</td>\n",
       "      <td></td>\n",
       "      <td>http://financialhighway.com/75-phone-apps-10-t...</td>\n",
       "      <td>1</td>\n",
       "      <td>1281439942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>257</th>\n",
       "      <td>Insurance Help for the Chronically Ill: A new ...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td></td>\n",
       "      <td>http://www.kiplinger.com/magazine/archives/ins...</td>\n",
       "      <td>4</td>\n",
       "      <td>1281387924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>258</th>\n",
       "      <td>Financial Products That Are a Waste of Money</td>\n",
       "      <td>mattymomostl</td>\n",
       "      <td></td>\n",
       "      <td>http://finance.yahoo.com/banking-budgeting/art...</td>\n",
       "      <td>4</td>\n",
       "      <td>1281378031</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>259 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 title            author  \\\n",
       "0        What are your best personal finance videos ?       killermouse0   \n",
       "1                  He got out of debt... You can too !      killermouse0   \n",
       "2    How I Made the Most Money I've Ever Made in My...         crappycap   \n",
       "3     What Are Your Rights as a Credit Card Customer?         kevinweb55   \n",
       "4    Young Will Pay Up To 17% More In Premiums For ...         [deleted]   \n",
       "..                                                 ...               ...   \n",
       "254                    10 Ways to Ruin Your Retirement      mattymomostl   \n",
       "255                Spending less will make you happier      mattymomostl   \n",
       "256   75 Phone Apps + 10 Tips that Will Save You Money  financialhighway   \n",
       "257  Insurance Help for the Chronically Ill: A new ...         [deleted]   \n",
       "258       Financial Products That Are a Waste of Money      mattymomostl   \n",
       "\n",
       "      selftext                                                url  score  \\\n",
       "0               https://www.reddit.com/r/personalfinance/comme...      5   \n",
       "1               http://www.getrichslowly.org/blog/2010/04/11/r...      3   \n",
       "2               http://www.stopbuyingcrap.com/even-more-rambli...      3   \n",
       "3               http://creditshout.com/what-are-your-rights-as...      3   \n",
       "4    [deleted]  http://www.msnbc.msn.com/id/36087653/ns/health...      0   \n",
       "..         ...                                                ...    ...   \n",
       "254             http://finance.yahoo.com/news/10-Ways-to-Ruin-...      5   \n",
       "255             http://finance.yahoo.com/family-home/article/1...      2   \n",
       "256             http://financialhighway.com/75-phone-apps-10-t...      1   \n",
       "257             http://www.kiplinger.com/magazine/archives/ins...      4   \n",
       "258             http://finance.yahoo.com/banking-budgeting/art...      4   \n",
       "\n",
       "     created_utc  \n",
       "0     1271604886  \n",
       "1     1271009625  \n",
       "2     1270646837  \n",
       "3     1270146984  \n",
       "4     1270140616  \n",
       "..           ...  \n",
       "254   1281448075  \n",
       "255   1281446316  \n",
       "256   1281439942  \n",
       "257   1281387924  \n",
       "258   1281378031  \n",
       "\n",
       "[259 rows x 6 columns]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns(submissions_pf_2010)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "contained-riding",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4413 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 46.39% - Requests: 97 - Batches: 10 - Items Remaining: 311\n",
      "Total:: Success Rate: 45.76% - Requests: 118 - Batches: 14 - Items Remaining: 0\n",
      "Retrieved 4413 submissions from Pushshift\n"
     ]
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2011-01-01'\n",
    "before_date = '2012-01-01'\n",
    "\n",
    "\n",
    "submissions_pf_2011 = scrape(subreddit, limit, before_date, after_date)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "hundred-speaking",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A good credit card for a law student wanting t...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>I'm currently a financially responsible law st...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>4</td>\n",
       "      <td>1303317487</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Longterm IRA advice</td>\n",
       "      <td>knightzend</td>\n",
       "      <td>I'm 27 and maxing out both my 401k and Roth IR...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>9</td>\n",
       "      <td>1303309211</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How ETFs Have Reshaped Investing - Exchange-tr...</td>\n",
       "      <td>vints1</td>\n",
       "      <td></td>\n",
       "      <td>http://online.wsj.com/article/SB10001424052748...</td>\n",
       "      <td>11</td>\n",
       "      <td>1303306488</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How to Build Good Credit (And Improve Your Cre...</td>\n",
       "      <td>jregenstein</td>\n",
       "      <td></td>\n",
       "      <td>http://www.rainstonefinancial.com/financial-pl...</td>\n",
       "      <td>1</td>\n",
       "      <td>1303303773</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Anybody concerned about inflation yet? Latest ...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td></td>\n",
       "      <td>http://buildnetworth.blogspot.com/2011/04/anyb...</td>\n",
       "      <td>1</td>\n",
       "      <td>1303276636</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4408</th>\n",
       "      <td>Ask: I'm 31, I put 30% of my 401k into S&amp;amp;P...</td>\n",
       "      <td>fldash</td>\n",
       "      <td></td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>5</td>\n",
       "      <td>1312827806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4409</th>\n",
       "      <td>Any Book Recommendations?</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>Any suggestions on books about the stock marke...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>7</td>\n",
       "      <td>1312827237</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4410</th>\n",
       "      <td>As a student still in college with no job, I j...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>I thought it would be a good idea to begin to ...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>4</td>\n",
       "      <td>1312825541</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4411</th>\n",
       "      <td>Doing well, but not sure what to do with the m...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>I have a 401k matched to company's contributio...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>8</td>\n",
       "      <td>1312812593</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4412</th>\n",
       "      <td>Where should you put your money after the down...</td>\n",
       "      <td>cochedejmu</td>\n",
       "      <td></td>\n",
       "      <td>http://www.dailyfinance.com/2011/08/05/after-t...</td>\n",
       "      <td>1</td>\n",
       "      <td>1312800547</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>4413 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                  title       author  \\\n",
       "0     A good credit card for a law student wanting t...    [deleted]   \n",
       "1                                   Longterm IRA advice   knightzend   \n",
       "2     How ETFs Have Reshaped Investing - Exchange-tr...       vints1   \n",
       "3     How to Build Good Credit (And Improve Your Cre...  jregenstein   \n",
       "4     Anybody concerned about inflation yet? Latest ...    [deleted]   \n",
       "...                                                 ...          ...   \n",
       "4408  Ask: I'm 31, I put 30% of my 401k into S&amp;P...       fldash   \n",
       "4409                          Any Book Recommendations?    [deleted]   \n",
       "4410  As a student still in college with no job, I j...    [deleted]   \n",
       "4411  Doing well, but not sure what to do with the m...    [deleted]   \n",
       "4412  Where should you put your money after the down...   cochedejmu   \n",
       "\n",
       "                                               selftext  \\\n",
       "0     I'm currently a financially responsible law st...   \n",
       "1     I'm 27 and maxing out both my 401k and Roth IR...   \n",
       "2                                                         \n",
       "3                                                         \n",
       "4                                                         \n",
       "...                                                 ...   \n",
       "4408                                                      \n",
       "4409  Any suggestions on books about the stock marke...   \n",
       "4410  I thought it would be a good idea to begin to ...   \n",
       "4411  I have a 401k matched to company's contributio...   \n",
       "4412                                                      \n",
       "\n",
       "                                                    url  score  created_utc  \n",
       "0     http://www.reddit.com/r/personalfinance/commen...      4   1303317487  \n",
       "1     http://www.reddit.com/r/personalfinance/commen...      9   1303309211  \n",
       "2     http://online.wsj.com/article/SB10001424052748...     11   1303306488  \n",
       "3     http://www.rainstonefinancial.com/financial-pl...      1   1303303773  \n",
       "4     http://buildnetworth.blogspot.com/2011/04/anyb...      1   1303276636  \n",
       "...                                                 ...    ...          ...  \n",
       "4408  http://www.reddit.com/r/personalfinance/commen...      5   1312827806  \n",
       "4409  http://www.reddit.com/r/personalfinance/commen...      7   1312827237  \n",
       "4410  http://www.reddit.com/r/personalfinance/commen...      4   1312825541  \n",
       "4411  http://www.reddit.com/r/personalfinance/commen...      8   1312812593  \n",
       "4412  http://www.dailyfinance.com/2011/08/05/after-t...      1   1312800547  \n",
       "\n",
       "[4413 rows x 6 columns]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selected_columns(submissions_pf_2011)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "duplicate-diagram",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11888 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 96.00% - Requests: 100 - Batches: 10 - Items Remaining: 2567\n",
      "Total:: Success Rate: 91.78% - Requests: 146 - Batches: 16 - Items Remaining: 0\n",
      "Retrieved 11888 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>So I called BoA the other day...</td>\n",
       "      <td>lifeinsector4</td>\n",
       "      <td>I know, I know, BoA sucks but I'm stuck with t...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>12</td>\n",
       "      <td>1328555860</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>I have 4 credit cards (soon to be 5) and I was...</td>\n",
       "      <td>backstab</td>\n",
       "      <td>I prefer to use one card over the others becau...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>5</td>\n",
       "      <td>1328555008</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How does one increase their credit score &amp;gt; ...</td>\n",
       "      <td>a-ko</td>\n",
       "      <td>I have a really good credit score for my age. ...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>9</td>\n",
       "      <td>1328554924</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Converting a 547 Deferred Compensation Plan to...</td>\n",
       "      <td>Ngawa</td>\n",
       "      <td>I am inheriting (and splitting) a 457 Deferred...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>3</td>\n",
       "      <td>1328554215</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Repair or replace my car?  What say Reddit?</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>What makes better financial sense in this case...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>3</td>\n",
       "      <td>1328553233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11883</th>\n",
       "      <td>spreadsheet for calculating income tax on pay ...</td>\n",
       "      <td>touristoflife</td>\n",
       "      <td>is there a spreadsheet or a way to calculate e...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>15</td>\n",
       "      <td>1349648465</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11884</th>\n",
       "      <td>How much rent can I afford?</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>My partner is breaking up with me and I need a...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>19</td>\n",
       "      <td>1349647626</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11885</th>\n",
       "      <td>I'm currently paying into a mandatory 403b and...</td>\n",
       "      <td>gsf4lyfe</td>\n",
       "      <td>I work full time for a non-profit and just qua...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>2</td>\n",
       "      <td>1349642441</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11886</th>\n",
       "      <td>When is the optimal time to buy a new car?</td>\n",
       "      <td>zanzibuz</td>\n",
       "      <td>My Accord is 12+ years old, 170k miles.  I owe...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>29</td>\n",
       "      <td>1349640892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11887</th>\n",
       "      <td>Hi!. I have 150 (apparently) tonne of pigs off...</td>\n",
       "      <td>SteveJEO</td>\n",
       "      <td></td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>0</td>\n",
       "      <td>1349639232</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11888 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title         author  \\\n",
       "0                       So I called BoA the other day...  lifeinsector4   \n",
       "1      I have 4 credit cards (soon to be 5) and I was...       backstab   \n",
       "2      How does one increase their credit score &gt; ...           a-ko   \n",
       "3      Converting a 547 Deferred Compensation Plan to...          Ngawa   \n",
       "4            Repair or replace my car?  What say Reddit?      [deleted]   \n",
       "...                                                  ...            ...   \n",
       "11883  spreadsheet for calculating income tax on pay ...  touristoflife   \n",
       "11884                        How much rent can I afford?      [deleted]   \n",
       "11885  I'm currently paying into a mandatory 403b and...       gsf4lyfe   \n",
       "11886         When is the optimal time to buy a new car?       zanzibuz   \n",
       "11887  Hi!. I have 150 (apparently) tonne of pigs off...       SteveJEO   \n",
       "\n",
       "                                                selftext  \\\n",
       "0      I know, I know, BoA sucks but I'm stuck with t...   \n",
       "1      I prefer to use one card over the others becau...   \n",
       "2      I have a really good credit score for my age. ...   \n",
       "3      I am inheriting (and splitting) a 457 Deferred...   \n",
       "4      What makes better financial sense in this case...   \n",
       "...                                                  ...   \n",
       "11883  is there a spreadsheet or a way to calculate e...   \n",
       "11884  My partner is breaking up with me and I need a...   \n",
       "11885  I work full time for a non-profit and just qua...   \n",
       "11886  My Accord is 12+ years old, 170k miles.  I owe...   \n",
       "11887                                                      \n",
       "\n",
       "                                                     url  score  created_utc  \n",
       "0      http://www.reddit.com/r/personalfinance/commen...     12   1328555860  \n",
       "1      http://www.reddit.com/r/personalfinance/commen...      5   1328555008  \n",
       "2      http://www.reddit.com/r/personalfinance/commen...      9   1328554924  \n",
       "3      http://www.reddit.com/r/personalfinance/commen...      3   1328554215  \n",
       "4      http://www.reddit.com/r/personalfinance/commen...      3   1328553233  \n",
       "...                                                  ...    ...          ...  \n",
       "11883  http://www.reddit.com/r/personalfinance/commen...     15   1349648465  \n",
       "11884  http://www.reddit.com/r/personalfinance/commen...     19   1349647626  \n",
       "11885  http://www.reddit.com/r/personalfinance/commen...      2   1349642441  \n",
       "11886  http://www.reddit.com/r/personalfinance/commen...     29   1349640892  \n",
       "11887  http://www.reddit.com/r/personalfinance/commen...      0   1349639232  \n",
       "\n",
       "[11888 rows x 6 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2012-01-01'\n",
    "before_date = '2013-01-01'\n",
    "\n",
    "\n",
    "submissions_pf_2012 = scrape(subreddit, limit, before_date, after_date)\n",
    "selected_columns(submissions_pf_2012)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "finished-victorian",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "25642 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 94.00% - Requests: 100 - Batches: 10 - Items Remaining: 16242\n",
      "Checkpoint:: Success Rate: 94.50% - Requests: 200 - Batches: 20 - Items Remaining: 7068\n",
      "Checkpoint:: Success Rate: 90.00% - Requests: 300 - Batches: 30 - Items Remaining: 1260\n",
      "Total:: Success Rate: 90.94% - Requests: 331 - Batches: 34 - Items Remaining: 0\n",
      "Retrieved 25642 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Credit Card Miles. Help!</td>\n",
       "      <td>Picklepreacher</td>\n",
       "      <td>I am traveling to the Netherlands this fall an...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1363323241</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Is this illegal in any way, or just unfair?</td>\n",
       "      <td>VinceVega77</td>\n",
       "      <td>Dear Reddit,\\nI am an 18 year old male from th...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>6</td>\n",
       "      <td>1363320212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>From a financial perspective, what would happe...</td>\n",
       "      <td>Commandolam</td>\n",
       "      <td>I just want to begin by saying they're not gon...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>4</td>\n",
       "      <td>1363320114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Employer not allowing disbursement from 403b?</td>\n",
       "      <td>moatbuilder</td>\n",
       "      <td>My dad's employer says he cannot withdraw from...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>3</td>\n",
       "      <td>1363318655</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Loans vs. Nest Egg...keep saving or start chip...</td>\n",
       "      <td>Cypa</td>\n",
       "      <td>Important Bits:\\n\\n* 24yrs - Gross $55K last y...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>3</td>\n",
       "      <td>1363318172</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25637</th>\n",
       "      <td>What's the lowest expense ratio for retirement...</td>\n",
       "      <td>nomii</td>\n",
       "      <td>Everyone says that the fees should be \"as low ...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>7</td>\n",
       "      <td>1387324325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25638</th>\n",
       "      <td>Investing through an HSA with sub-optimal fees...</td>\n",
       "      <td>Feyndude</td>\n",
       "      <td>I am a 24 year old with no debt, an emergency ...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>17</td>\n",
       "      <td>1387323456</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25639</th>\n",
       "      <td>I need some tax advice RE: sale of converted s...</td>\n",
       "      <td>sketchyvanguy</td>\n",
       "      <td>Specifically, the sale of WMIH, which was purc...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>4</td>\n",
       "      <td>1387321227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25640</th>\n",
       "      <td>Taboo Investing (x-post from r/investing)</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>Hello all,\\n\\nI am here to get your opinion on...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>5</td>\n",
       "      <td>1387320706</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25641</th>\n",
       "      <td>IRA or compound interest savings?</td>\n",
       "      <td>PsychedelicBumblebee</td>\n",
       "      <td>I'm a young person looking to start investing ...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>6</td>\n",
       "      <td>1387318459</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>25642 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title  \\\n",
       "0                               Credit Card Miles. Help!   \n",
       "1            Is this illegal in any way, or just unfair?   \n",
       "2      From a financial perspective, what would happe...   \n",
       "3          Employer not allowing disbursement from 403b?   \n",
       "4      Loans vs. Nest Egg...keep saving or start chip...   \n",
       "...                                                  ...   \n",
       "25637  What's the lowest expense ratio for retirement...   \n",
       "25638  Investing through an HSA with sub-optimal fees...   \n",
       "25639  I need some tax advice RE: sale of converted s...   \n",
       "25640          Taboo Investing (x-post from r/investing)   \n",
       "25641                  IRA or compound interest savings?   \n",
       "\n",
       "                     author  \\\n",
       "0            Picklepreacher   \n",
       "1               VinceVega77   \n",
       "2               Commandolam   \n",
       "3               moatbuilder   \n",
       "4                      Cypa   \n",
       "...                     ...   \n",
       "25637                 nomii   \n",
       "25638              Feyndude   \n",
       "25639         sketchyvanguy   \n",
       "25640             [deleted]   \n",
       "25641  PsychedelicBumblebee   \n",
       "\n",
       "                                                selftext  \\\n",
       "0      I am traveling to the Netherlands this fall an...   \n",
       "1      Dear Reddit,\\nI am an 18 year old male from th...   \n",
       "2      I just want to begin by saying they're not gon...   \n",
       "3      My dad's employer says he cannot withdraw from...   \n",
       "4      Important Bits:\\n\\n* 24yrs - Gross $55K last y...   \n",
       "...                                                  ...   \n",
       "25637  Everyone says that the fees should be \"as low ...   \n",
       "25638  I am a 24 year old with no debt, an emergency ...   \n",
       "25639  Specifically, the sale of WMIH, which was purc...   \n",
       "25640  Hello all,\\n\\nI am here to get your opinion on...   \n",
       "25641  I'm a young person looking to start investing ...   \n",
       "\n",
       "                                                     url  score  created_utc  \n",
       "0      http://www.reddit.com/r/personalfinance/commen...      1   1363323241  \n",
       "1      http://www.reddit.com/r/personalfinance/commen...      6   1363320212  \n",
       "2      http://www.reddit.com/r/personalfinance/commen...      4   1363320114  \n",
       "3      http://www.reddit.com/r/personalfinance/commen...      3   1363318655  \n",
       "4      http://www.reddit.com/r/personalfinance/commen...      3   1363318172  \n",
       "...                                                  ...    ...          ...  \n",
       "25637  http://www.reddit.com/r/personalfinance/commen...      7   1387324325  \n",
       "25638  http://www.reddit.com/r/personalfinance/commen...     17   1387323456  \n",
       "25639  http://www.reddit.com/r/personalfinance/commen...      4   1387321227  \n",
       "25640  http://www.reddit.com/r/personalfinance/commen...      5   1387320706  \n",
       "25641  http://www.reddit.com/r/personalfinance/commen...      6   1387318459  \n",
       "\n",
       "[25642 rows x 6 columns]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2013-01-01'\n",
    "before_date = '2014-01-01'\n",
    "\n",
    "submissions_pf_2013 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2013)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "velvet-mineral",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "62372 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 100.00% - Requests: 100 - Batches: 10 - Items Remaining: 52372\n",
      "Checkpoint:: Success Rate: 94.50% - Requests: 200 - Batches: 20 - Items Remaining: 43472\n",
      "Checkpoint:: Success Rate: 92.33% - Requests: 300 - Batches: 30 - Items Remaining: 34672\n",
      "Checkpoint:: Success Rate: 86.75% - Requests: 400 - Batches: 40 - Items Remaining: 29462\n",
      "Checkpoint:: Success Rate: 84.00% - Requests: 500 - Batches: 50 - Items Remaining: 23100\n",
      "Checkpoint:: Success Rate: 82.00% - Requests: 600 - Batches: 60 - Items Remaining: 16043\n",
      "Checkpoint:: Success Rate: 80.00% - Requests: 700 - Batches: 70 - Items Remaining: 9786\n",
      "Checkpoint:: Success Rate: 78.75% - Requests: 800 - Batches: 80 - Items Remaining: 6033\n",
      "Checkpoint:: Success Rate: 79.44% - Requests: 900 - Batches: 90 - Items Remaining: 1052\n",
      "Total:: Success Rate: 78.96% - Requests: 941 - Batches: 95 - Items Remaining: 4\n",
      "Total:: Success Rate: 78.98% - Requests: 942 - Batches: 96 - Items Remaining: 0\n",
      "Retrieved 62372 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Request: Help budget my new salary for when I ...</td>\n",
       "      <td>luckylad5391</td>\n",
       "      <td>Hi fiscally responsible redditors, I have foun...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>5</td>\n",
       "      <td>1394858978</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>19 years old on the work, sleep, repeat schedu...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>Hello reddit, first time posting so please exc...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>17</td>\n",
       "      <td>1394858545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Listing parents as dependents &amp;amp; Obamacare</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>Hi all, \\n\\nI make about 57k a year and I am t...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1394858518</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Community College kid wanting to invest (CA)</td>\n",
       "      <td>slobodan48</td>\n",
       "      <td>Third year full community college kid studying...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>4</td>\n",
       "      <td>1394857042</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>$30,000 to invest as a 19 year-old</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>3 years ago my father gave me $4k worth of sto...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1394856476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62367</th>\n",
       "      <td>Is it possible to live to live without financi...</td>\n",
       "      <td>Theworst2013</td>\n",
       "      <td>I am a senior in high school, almost done with...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>2</td>\n",
       "      <td>1416170416</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62368</th>\n",
       "      <td>Should I claim additional allowances for 2014?</td>\n",
       "      <td>Xzanthos</td>\n",
       "      <td>This is my first year in my new position. I us...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>2</td>\n",
       "      <td>1391704930</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62369</th>\n",
       "      <td>What do I need to bring to a tax professional?</td>\n",
       "      <td>zsimmons</td>\n",
       "      <td>I quit one job and started another in the midd...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>0</td>\n",
       "      <td>1391704707</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62370</th>\n",
       "      <td>Will a brief tenure at a job with a higher sal...</td>\n",
       "      <td>SwampRabbit</td>\n",
       "      <td>My husband is the Assistant Manager on a blue-...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>52</td>\n",
       "      <td>1391704695</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>62371</th>\n",
       "      <td>Need some help with my student loan interest 1...</td>\n",
       "      <td>carloadoffarts</td>\n",
       "      <td>I received one form a few weeks ago saying I h...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>2</td>\n",
       "      <td>1391704526</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>62372 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title          author  \\\n",
       "0      Request: Help budget my new salary for when I ...    luckylad5391   \n",
       "1      19 years old on the work, sleep, repeat schedu...       [deleted]   \n",
       "2          Listing parents as dependents &amp; Obamacare       [deleted]   \n",
       "3           Community College kid wanting to invest (CA)      slobodan48   \n",
       "4                     $30,000 to invest as a 19 year-old       [deleted]   \n",
       "...                                                  ...             ...   \n",
       "62367  Is it possible to live to live without financi...    Theworst2013   \n",
       "62368     Should I claim additional allowances for 2014?        Xzanthos   \n",
       "62369     What do I need to bring to a tax professional?        zsimmons   \n",
       "62370  Will a brief tenure at a job with a higher sal...     SwampRabbit   \n",
       "62371  Need some help with my student loan interest 1...  carloadoffarts   \n",
       "\n",
       "                                                selftext  \\\n",
       "0      Hi fiscally responsible redditors, I have foun...   \n",
       "1      Hello reddit, first time posting so please exc...   \n",
       "2      Hi all, \\n\\nI make about 57k a year and I am t...   \n",
       "3      Third year full community college kid studying...   \n",
       "4      3 years ago my father gave me $4k worth of sto...   \n",
       "...                                                  ...   \n",
       "62367  I am a senior in high school, almost done with...   \n",
       "62368  This is my first year in my new position. I us...   \n",
       "62369  I quit one job and started another in the midd...   \n",
       "62370  My husband is the Assistant Manager on a blue-...   \n",
       "62371  I received one form a few weeks ago saying I h...   \n",
       "\n",
       "                                                     url  score  created_utc  \n",
       "0      http://www.reddit.com/r/personalfinance/commen...      5   1394858978  \n",
       "1      http://www.reddit.com/r/personalfinance/commen...     17   1394858545  \n",
       "2      http://www.reddit.com/r/personalfinance/commen...      1   1394858518  \n",
       "3      http://www.reddit.com/r/personalfinance/commen...      4   1394857042  \n",
       "4      http://www.reddit.com/r/personalfinance/commen...      1   1394856476  \n",
       "...                                                  ...    ...          ...  \n",
       "62367  http://www.reddit.com/r/personalfinance/commen...      2   1416170416  \n",
       "62368  http://www.reddit.com/r/personalfinance/commen...      2   1391704930  \n",
       "62369  http://www.reddit.com/r/personalfinance/commen...      0   1391704707  \n",
       "62370  http://www.reddit.com/r/personalfinance/commen...     52   1391704695  \n",
       "62371  http://www.reddit.com/r/personalfinance/commen...      2   1391704526  \n",
       "\n",
       "[62372 rows x 6 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2014-01-01'\n",
    "before_date = '2015-01-01'\n",
    "\n",
    "submissions_pf_2014 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2014)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "careful-meter",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "116789 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 72.00% - Requests: 100 - Batches: 10 - Items Remaining: 109589\n",
      "Checkpoint:: Success Rate: 74.50% - Requests: 200 - Batches: 20 - Items Remaining: 101889\n",
      "Checkpoint:: Success Rate: 75.67% - Requests: 300 - Batches: 30 - Items Remaining: 94089\n",
      "Checkpoint:: Success Rate: 78.00% - Requests: 400 - Batches: 40 - Items Remaining: 85589\n",
      "Checkpoint:: Success Rate: 79.40% - Requests: 500 - Batches: 50 - Items Remaining: 77089\n",
      "Checkpoint:: Success Rate: 82.00% - Requests: 600 - Batches: 60 - Items Remaining: 67589\n",
      "Checkpoint:: Success Rate: 83.00% - Requests: 700 - Batches: 70 - Items Remaining: 58689\n",
      "Checkpoint:: Success Rate: 83.75% - Requests: 800 - Batches: 80 - Items Remaining: 50056\n",
      "Checkpoint:: Success Rate: 83.89% - Requests: 900 - Batches: 90 - Items Remaining: 42114\n",
      "Checkpoint:: Success Rate: 84.30% - Requests: 1000 - Batches: 100 - Items Remaining: 34090\n",
      "Checkpoint:: Success Rate: 84.64% - Requests: 1100 - Batches: 110 - Items Remaining: 26228\n",
      "Checkpoint:: Success Rate: 84.33% - Requests: 1200 - Batches: 120 - Items Remaining: 18874\n",
      "Checkpoint:: Success Rate: 85.00% - Requests: 1300 - Batches: 130 - Items Remaining: 12538\n",
      "Checkpoint:: Success Rate: 85.36% - Requests: 1400 - Batches: 140 - Items Remaining: 7901\n",
      "Checkpoint:: Success Rate: 85.53% - Requests: 1500 - Batches: 150 - Items Remaining: 3542\n",
      "Checkpoint:: Success Rate: 85.58% - Requests: 1588 - Batches: 160 - Items Remaining: 4\n",
      "Total:: Success Rate: 85.58% - Requests: 1588 - Batches: 160 - Items Remaining: 4\n",
      "Total:: Success Rate: 85.59% - Requests: 1589 - Batches: 161 - Items Remaining: 0\n",
      "Retrieved 116789 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>TurboTax: where does last year's federal refun...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>Last year's state/local tax returns automatica...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>0</td>\n",
       "      <td>1426394770</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Anyone know a good retirement calculator that ...</td>\n",
       "      <td>dstokes</td>\n",
       "      <td>I'm looking for something that can calculate t...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>0</td>\n",
       "      <td>1426393705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Is there any reason to split up my money among...</td>\n",
       "      <td>PhonyHoldenCaulfield</td>\n",
       "      <td>Or can I put most of my money in one bank? The...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1426393705</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Do you count paying down debts in your savings...</td>\n",
       "      <td>seandealan</td>\n",
       "      <td>If I save 20% of my take home but then put ano...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1426393402</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Currently looking to take out a personal loan.</td>\n",
       "      <td>lakaihc</td>\n",
       "      <td>I'm currently looking to take out a personal l...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>0</td>\n",
       "      <td>1426392985</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116784</th>\n",
       "      <td>Retiring mid to late thirties</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>\"A man is a success if he gets up in the morni...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1428427224</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116785</th>\n",
       "      <td>Should I be doing more with my finances?</td>\n",
       "      <td>RadialThrowaway</td>\n",
       "      <td>I'm currently a college sophomore who lives at...</td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>5</td>\n",
       "      <td>1435856372</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116786</th>\n",
       "      <td>Cotinthian colleges lawsuit</td>\n",
       "      <td>dontthink19</td>\n",
       "      <td></td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1435856151</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116787</th>\n",
       "      <td>What are some things to consider when you want...</td>\n",
       "      <td>elvar_10</td>\n",
       "      <td></td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1435856120</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>116788</th>\n",
       "      <td>Is there a database or website that's shows wh...</td>\n",
       "      <td>thebrownfossil</td>\n",
       "      <td></td>\n",
       "      <td>http://www.reddit.com/r/personalfinance/commen...</td>\n",
       "      <td>1</td>\n",
       "      <td>1435856067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>116789 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title  \\\n",
       "0       TurboTax: where does last year's federal refun...   \n",
       "1       Anyone know a good retirement calculator that ...   \n",
       "2       Is there any reason to split up my money among...   \n",
       "3       Do you count paying down debts in your savings...   \n",
       "4          Currently looking to take out a personal loan.   \n",
       "...                                                   ...   \n",
       "116784                      Retiring mid to late thirties   \n",
       "116785           Should I be doing more with my finances?   \n",
       "116786                        Cotinthian colleges lawsuit   \n",
       "116787  What are some things to consider when you want...   \n",
       "116788  Is there a database or website that's shows wh...   \n",
       "\n",
       "                      author  \\\n",
       "0                  [deleted]   \n",
       "1                    dstokes   \n",
       "2       PhonyHoldenCaulfield   \n",
       "3                 seandealan   \n",
       "4                    lakaihc   \n",
       "...                      ...   \n",
       "116784             [deleted]   \n",
       "116785       RadialThrowaway   \n",
       "116786           dontthink19   \n",
       "116787              elvar_10   \n",
       "116788        thebrownfossil   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0       Last year's state/local tax returns automatica...   \n",
       "1       I'm looking for something that can calculate t...   \n",
       "2       Or can I put most of my money in one bank? The...   \n",
       "3       If I save 20% of my take home but then put ano...   \n",
       "4       I'm currently looking to take out a personal l...   \n",
       "...                                                   ...   \n",
       "116784  \"A man is a success if he gets up in the morni...   \n",
       "116785  I'm currently a college sophomore who lives at...   \n",
       "116786                                                      \n",
       "116787                                                      \n",
       "116788                                                      \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       http://www.reddit.com/r/personalfinance/commen...      0   1426394770  \n",
       "1       http://www.reddit.com/r/personalfinance/commen...      0   1426393705  \n",
       "2       http://www.reddit.com/r/personalfinance/commen...      1   1426393705  \n",
       "3       http://www.reddit.com/r/personalfinance/commen...      1   1426393402  \n",
       "4       http://www.reddit.com/r/personalfinance/commen...      0   1426392985  \n",
       "...                                                   ...    ...          ...  \n",
       "116784  http://www.reddit.com/r/personalfinance/commen...      1   1428427224  \n",
       "116785  http://www.reddit.com/r/personalfinance/commen...      5   1435856372  \n",
       "116786  http://www.reddit.com/r/personalfinance/commen...      1   1435856151  \n",
       "116787  http://www.reddit.com/r/personalfinance/commen...      1   1435856120  \n",
       "116788  http://www.reddit.com/r/personalfinance/commen...      1   1435856067  \n",
       "\n",
       "[116789 rows x 6 columns]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2015-01-01'\n",
    "before_date = '2016-01-01'\n",
    "\n",
    "submissions_pf_2015 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2015)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "hispanic-expansion",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "138627 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 81.00% - Requests: 100 - Batches: 10 - Items Remaining: 130527\n",
      "Checkpoint:: Success Rate: 80.50% - Requests: 200 - Batches: 20 - Items Remaining: 122527\n",
      "Checkpoint:: Success Rate: 83.33% - Requests: 300 - Batches: 30 - Items Remaining: 113627\n",
      "Checkpoint:: Success Rate: 82.25% - Requests: 400 - Batches: 40 - Items Remaining: 105727\n",
      "Checkpoint:: Success Rate: 82.40% - Requests: 500 - Batches: 50 - Items Remaining: 97427\n",
      "Checkpoint:: Success Rate: 84.33% - Requests: 600 - Batches: 60 - Items Remaining: 88027\n",
      "Checkpoint:: Success Rate: 83.71% - Requests: 700 - Batches: 70 - Items Remaining: 80072\n",
      "Checkpoint:: Success Rate: 83.75% - Requests: 800 - Batches: 80 - Items Remaining: 71982\n",
      "Checkpoint:: Success Rate: 83.00% - Requests: 900 - Batches: 90 - Items Remaining: 64646\n",
      "Checkpoint:: Success Rate: 81.80% - Requests: 1000 - Batches: 100 - Items Remaining: 57709\n",
      "Checkpoint:: Success Rate: 82.09% - Requests: 1100 - Batches: 110 - Items Remaining: 49539\n",
      "Checkpoint:: Success Rate: 81.42% - Requests: 1200 - Batches: 120 - Items Remaining: 42714\n",
      "Checkpoint:: Success Rate: 81.54% - Requests: 1300 - Batches: 130 - Items Remaining: 34902\n",
      "Checkpoint:: Success Rate: 81.57% - Requests: 1400 - Batches: 140 - Items Remaining: 28152\n",
      "Checkpoint:: Success Rate: 82.33% - Requests: 1500 - Batches: 150 - Items Remaining: 21622\n",
      "Checkpoint:: Success Rate: 82.50% - Requests: 1600 - Batches: 160 - Items Remaining: 15720\n",
      "Checkpoint:: Success Rate: 82.47% - Requests: 1700 - Batches: 170 - Items Remaining: 11465\n",
      "Checkpoint:: Success Rate: 82.89% - Requests: 1800 - Batches: 180 - Items Remaining: 6320\n",
      "Checkpoint:: Success Rate: 82.79% - Requests: 1900 - Batches: 190 - Items Remaining: 2043\n",
      "Total:: Success Rate: 82.98% - Requests: 1956 - Batches: 196 - Items Remaining: 6\n",
      "Total:: Success Rate: 82.98% - Requests: 1957 - Batches: 197 - Items Remaining: 0\n",
      "Retrieved 138627 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Setting up a Vanguard account overseas?</td>\n",
       "      <td>1yawaworhtsdrawkcab</td>\n",
       "      <td>I am relocating for work. My bank account and ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1467435325</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>fellow goodyear workers: eli5 i cant get the s...</td>\n",
       "      <td>jacob1642</td>\n",
       "      <td>the Goodyear Tire &amp;amp; Rubber Company Employe...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1467435229</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Leaving Full Time Job for Grad School - Advice?</td>\n",
       "      <td>kingsalm0n</td>\n",
       "      <td>I'm planning on quitting my job and going to g...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1467434722</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>[Credit] 21 years old, currently a student, ma...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1467434233</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Was suggested to post this here, so here goes....</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1467433967</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138622</th>\n",
       "      <td>Medical residency + supporting a family</td>\n",
       "      <td>whattodowhattodo12</td>\n",
       "      <td>Not talking about spouse + child...talking abo...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1454786271</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138623</th>\n",
       "      <td>Should I cash out my IRA for my new family</td>\n",
       "      <td>PF_throwaway2616</td>\n",
       "      <td>I am married, we are both 30. We have 11k in a...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1454786137</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138624</th>\n",
       "      <td>Should I pay 7K to get rid of my PMI or invest...</td>\n",
       "      <td>Spinnerhead</td>\n",
       "      <td>I apologize if this has been posted before but...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1454786050</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138625</th>\n",
       "      <td>Information to keep on hand in case of an audit?</td>\n",
       "      <td>cyclist36</td>\n",
       "      <td>Because of it being tax season, I was just cur...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1454785892</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>138626</th>\n",
       "      <td>Subtracting commision from hourly wages</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1454785865</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>138627 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title  \\\n",
       "0                 Setting up a Vanguard account overseas?   \n",
       "1       fellow goodyear workers: eli5 i cant get the s...   \n",
       "2         Leaving Full Time Job for Grad School - Advice?   \n",
       "3       [Credit] 21 years old, currently a student, ma...   \n",
       "4       Was suggested to post this here, so here goes....   \n",
       "...                                                   ...   \n",
       "138622            Medical residency + supporting a family   \n",
       "138623         Should I cash out my IRA for my new family   \n",
       "138624  Should I pay 7K to get rid of my PMI or invest...   \n",
       "138625   Information to keep on hand in case of an audit?   \n",
       "138626            Subtracting commision from hourly wages   \n",
       "\n",
       "                     author  \\\n",
       "0       1yawaworhtsdrawkcab   \n",
       "1                 jacob1642   \n",
       "2                kingsalm0n   \n",
       "3                 [deleted]   \n",
       "4                 [deleted]   \n",
       "...                     ...   \n",
       "138622   whattodowhattodo12   \n",
       "138623     PF_throwaway2616   \n",
       "138624          Spinnerhead   \n",
       "138625            cyclist36   \n",
       "138626            [deleted]   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0       I am relocating for work. My bank account and ...   \n",
       "1       the Goodyear Tire &amp; Rubber Company Employe...   \n",
       "2       I'm planning on quitting my job and going to g...   \n",
       "3                                               [deleted]   \n",
       "4                                               [removed]   \n",
       "...                                                   ...   \n",
       "138622  Not talking about spouse + child...talking abo...   \n",
       "138623  I am married, we are both 30. We have 11k in a...   \n",
       "138624  I apologize if this has been posted before but...   \n",
       "138625  Because of it being tax season, I was just cur...   \n",
       "138626                                          [removed]   \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       https://www.reddit.com/r/personalfinance/comme...      2   1467435325  \n",
       "1       https://www.reddit.com/r/personalfinance/comme...      0   1467435229  \n",
       "2       https://www.reddit.com/r/personalfinance/comme...      2   1467434722  \n",
       "3       https://www.reddit.com/r/personalfinance/comme...      1   1467434233  \n",
       "4       https://www.reddit.com/r/personalfinance/comme...      1   1467433967  \n",
       "...                                                   ...    ...          ...  \n",
       "138622  https://www.reddit.com/r/personalfinance/comme...      1   1454786271  \n",
       "138623  https://www.reddit.com/r/personalfinance/comme...      0   1454786137  \n",
       "138624  https://www.reddit.com/r/personalfinance/comme...      1   1454786050  \n",
       "138625  https://www.reddit.com/r/personalfinance/comme...      2   1454785892  \n",
       "138626  https://www.reddit.com/r/personalfinance/comme...      1   1454785865  \n",
       "\n",
       "[138627 rows x 6 columns]"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2016-01-01'\n",
    "before_date = '2017-01-01'\n",
    "\n",
    "submissions_pf_2016 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2016)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "hindu-montana",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "165224 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 85.00% - Requests: 100 - Batches: 10 - Items Remaining: 156724\n",
      "Checkpoint:: Success Rate: 76.50% - Requests: 200 - Batches: 20 - Items Remaining: 149924\n",
      "Checkpoint:: Success Rate: 77.33% - Requests: 300 - Batches: 30 - Items Remaining: 142024\n",
      "Checkpoint:: Success Rate: 75.00% - Requests: 400 - Batches: 40 - Items Remaining: 135224\n",
      "Checkpoint:: Success Rate: 74.80% - Requests: 500 - Batches: 50 - Items Remaining: 127824\n",
      "Checkpoint:: Success Rate: 72.83% - Requests: 600 - Batches: 60 - Items Remaining: 121524\n",
      "Checkpoint:: Success Rate: 71.86% - Requests: 700 - Batches: 70 - Items Remaining: 114924\n",
      "Checkpoint:: Success Rate: 69.00% - Requests: 800 - Batches: 80 - Items Remaining: 110074\n",
      "Checkpoint:: Success Rate: 67.44% - Requests: 900 - Batches: 90 - Items Remaining: 104586\n",
      "Checkpoint:: Success Rate: 67.50% - Requests: 1000 - Batches: 100 - Items Remaining: 97984\n",
      "Checkpoint:: Success Rate: 67.82% - Requests: 1100 - Batches: 110 - Items Remaining: 91008\n",
      "Checkpoint:: Success Rate: 68.00% - Requests: 1200 - Batches: 120 - Items Remaining: 84140\n",
      "Checkpoint:: Success Rate: 68.54% - Requests: 1300 - Batches: 130 - Items Remaining: 76915\n",
      "Checkpoint:: Success Rate: 69.36% - Requests: 1400 - Batches: 140 - Items Remaining: 69272\n",
      "Checkpoint:: Success Rate: 69.47% - Requests: 1500 - Batches: 150 - Items Remaining: 62438\n",
      "Checkpoint:: Success Rate: 71.06% - Requests: 1600 - Batches: 160 - Items Remaining: 53799\n",
      "Checkpoint:: Success Rate: 71.76% - Requests: 1700 - Batches: 170 - Items Remaining: 46203\n",
      "Checkpoint:: Success Rate: 72.61% - Requests: 1800 - Batches: 180 - Items Remaining: 39996\n",
      "Checkpoint:: Success Rate: 72.32% - Requests: 1900 - Batches: 190 - Items Remaining: 34391\n",
      "Checkpoint:: Success Rate: 72.55% - Requests: 2000 - Batches: 200 - Items Remaining: 27987\n",
      "Checkpoint:: Success Rate: 72.95% - Requests: 2100 - Batches: 210 - Items Remaining: 22482\n",
      "Checkpoint:: Success Rate: 73.05% - Requests: 2200 - Batches: 220 - Items Remaining: 17898\n",
      "Checkpoint:: Success Rate: 73.17% - Requests: 2300 - Batches: 230 - Items Remaining: 13373\n",
      "Checkpoint:: Success Rate: 73.71% - Requests: 2400 - Batches: 240 - Items Remaining: 9250\n",
      "Checkpoint:: Success Rate: 73.84% - Requests: 2500 - Batches: 250 - Items Remaining: 3949\n",
      "Checkpoint:: Success Rate: 73.69% - Requests: 2600 - Batches: 260 - Items Remaining: 958\n",
      "Total:: Success Rate: 73.93% - Requests: 2624 - Batches: 263 - Items Remaining: 12\n",
      "Total:: Success Rate: 73.83% - Requests: 2629 - Batches: 264 - Items Remaining: 0\n",
      "Retrieved 165224 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Long time reader, first time poster on this pa...</td>\n",
       "      <td>VoteKvothe4SithLord</td>\n",
       "      <td></td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1499014611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Recent grad buying first car. Looking for a us...</td>\n",
       "      <td>PM-TITTIES-2-ME</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1499014575</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>How to adjust to one income?</td>\n",
       "      <td>coffeeandhorror</td>\n",
       "      <td>My husband recently left me and our son, and s...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>6</td>\n",
       "      <td>1499014185</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Medical debt on my credit report?</td>\n",
       "      <td>obeygoosey87</td>\n",
       "      <td>I just turned 30 and I really want to start fi...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1499014048</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Why is it so hard to get a job after Graduation?</td>\n",
       "      <td>milakunis22</td>\n",
       "      <td>I have been applying online since March. Unfor...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1499014007</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165219</th>\n",
       "      <td>Should I try to buy a house or rent an apartment?</td>\n",
       "      <td>suddenlyinternet2</td>\n",
       "      <td>My SO and I make about $3500 a month combined ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1486399826</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165220</th>\n",
       "      <td>I made $23,000CAD selling my home. Where shoul...</td>\n",
       "      <td>partyc4t</td>\n",
       "      <td>I don't want to just leave it in my bank accou...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1486399799</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165221</th>\n",
       "      <td>Received a call from an ‘accounts receivable m...</td>\n",
       "      <td>shinsukato</td>\n",
       "      <td>I went to community college for 2 years in the...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1486399719</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165222</th>\n",
       "      <td>Little over 5$ in personal bank account - need...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1486399614</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165223</th>\n",
       "      <td>Cars - Buying or Leasing</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1486399599</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165224 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title  \\\n",
       "0       Long time reader, first time poster on this pa...   \n",
       "1       Recent grad buying first car. Looking for a us...   \n",
       "2                            How to adjust to one income?   \n",
       "3                       Medical debt on my credit report?   \n",
       "4        Why is it so hard to get a job after Graduation?   \n",
       "...                                                   ...   \n",
       "165219  Should I try to buy a house or rent an apartment?   \n",
       "165220  I made $23,000CAD selling my home. Where shoul...   \n",
       "165221  Received a call from an ‘accounts receivable m...   \n",
       "165222  Little over 5$ in personal bank account - need...   \n",
       "165223                           Cars - Buying or Leasing   \n",
       "\n",
       "                     author  \\\n",
       "0       VoteKvothe4SithLord   \n",
       "1           PM-TITTIES-2-ME   \n",
       "2           coffeeandhorror   \n",
       "3              obeygoosey87   \n",
       "4               milakunis22   \n",
       "...                     ...   \n",
       "165219    suddenlyinternet2   \n",
       "165220             partyc4t   \n",
       "165221           shinsukato   \n",
       "165222            [deleted]   \n",
       "165223            [deleted]   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0                                                           \n",
       "1                                               [removed]   \n",
       "2       My husband recently left me and our son, and s...   \n",
       "3       I just turned 30 and I really want to start fi...   \n",
       "4       I have been applying online since March. Unfor...   \n",
       "...                                                   ...   \n",
       "165219  My SO and I make about $3500 a month combined ...   \n",
       "165220  I don't want to just leave it in my bank accou...   \n",
       "165221  I went to community college for 2 years in the...   \n",
       "165222                                          [deleted]   \n",
       "165223                                          [deleted]   \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       https://www.reddit.com/r/personalfinance/comme...      1   1499014611  \n",
       "1       https://www.reddit.com/r/personalfinance/comme...      0   1499014575  \n",
       "2       https://www.reddit.com/r/personalfinance/comme...      6   1499014185  \n",
       "3       https://www.reddit.com/r/personalfinance/comme...      1   1499014048  \n",
       "4       https://www.reddit.com/r/personalfinance/comme...      1   1499014007  \n",
       "...                                                   ...    ...          ...  \n",
       "165219  https://www.reddit.com/r/personalfinance/comme...      1   1486399826  \n",
       "165220  https://www.reddit.com/r/personalfinance/comme...      0   1486399799  \n",
       "165221  https://www.reddit.com/r/personalfinance/comme...      1   1486399719  \n",
       "165222  https://www.reddit.com/r/personalfinance/comme...      2   1486399614  \n",
       "165223  https://www.reddit.com/r/personalfinance/comme...      2   1486399599  \n",
       "\n",
       "[165224 rows x 6 columns]"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2017-01-01'\n",
    "before_date = '2018-01-01'\n",
    "\n",
    "submissions_pf_2017 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "synthetic-context",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "165224 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 49.00% - Requests: 100 - Batches: 10 - Items Remaining: 160324\n",
      "Checkpoint:: Success Rate: 54.50% - Requests: 200 - Batches: 20 - Items Remaining: 154324\n",
      "Checkpoint:: Success Rate: 62.00% - Requests: 300 - Batches: 30 - Items Remaining: 146624\n",
      "Checkpoint:: Success Rate: 65.75% - Requests: 400 - Batches: 40 - Items Remaining: 138924\n",
      "Checkpoint:: Success Rate: 64.20% - Requests: 500 - Batches: 50 - Items Remaining: 133124\n",
      "Checkpoint:: Success Rate: 65.83% - Requests: 600 - Batches: 60 - Items Remaining: 125724\n",
      "Checkpoint:: Success Rate: 66.57% - Requests: 700 - Batches: 70 - Items Remaining: 118624\n",
      "Checkpoint:: Success Rate: 69.50% - Requests: 800 - Batches: 80 - Items Remaining: 109668\n",
      "Checkpoint:: Success Rate: 71.00% - Requests: 900 - Batches: 90 - Items Remaining: 101465\n",
      "Checkpoint:: Success Rate: 71.90% - Requests: 1000 - Batches: 100 - Items Remaining: 93688\n",
      "Checkpoint:: Success Rate: 72.82% - Requests: 1100 - Batches: 110 - Items Remaining: 85657\n",
      "Checkpoint:: Success Rate: 72.92% - Requests: 1200 - Batches: 120 - Items Remaining: 78424\n",
      "Checkpoint:: Success Rate: 73.31% - Requests: 1300 - Batches: 130 - Items Remaining: 70893\n",
      "Checkpoint:: Success Rate: 73.64% - Requests: 1400 - Batches: 140 - Items Remaining: 63450\n",
      "Checkpoint:: Success Rate: 73.60% - Requests: 1500 - Batches: 150 - Items Remaining: 56510\n",
      "Checkpoint:: Success Rate: 73.81% - Requests: 1600 - Batches: 160 - Items Remaining: 49692\n",
      "Checkpoint:: Success Rate: 74.53% - Requests: 1700 - Batches: 170 - Items Remaining: 42234\n",
      "Checkpoint:: Success Rate: 75.00% - Requests: 1800 - Batches: 180 - Items Remaining: 36288\n",
      "Checkpoint:: Success Rate: 75.58% - Requests: 1900 - Batches: 190 - Items Remaining: 28928\n",
      "Checkpoint:: Success Rate: 75.85% - Requests: 2000 - Batches: 200 - Items Remaining: 23277\n",
      "Checkpoint:: Success Rate: 75.95% - Requests: 2100 - Batches: 210 - Items Remaining: 17996\n",
      "Checkpoint:: Success Rate: 75.18% - Requests: 2200 - Batches: 220 - Items Remaining: 14642\n",
      "Checkpoint:: Success Rate: 74.17% - Requests: 2300 - Batches: 230 - Items Remaining: 11252\n",
      "Checkpoint:: Success Rate: 73.62% - Requests: 2400 - Batches: 240 - Items Remaining: 7554\n",
      "Checkpoint:: Success Rate: 73.20% - Requests: 2500 - Batches: 250 - Items Remaining: 4694\n",
      "Checkpoint:: Success Rate: 72.88% - Requests: 2600 - Batches: 260 - Items Remaining: 2061\n",
      "Total:: Success Rate: 72.82% - Requests: 2664 - Batches: 268 - Items Remaining: 12\n",
      "Total:: Success Rate: 72.83% - Requests: 2665 - Batches: 269 - Items Remaining: 0\n",
      "Retrieved 165224 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>22yr old assist wz looking for car insurance</td>\n",
       "      <td>010101111x</td>\n",
       "      <td>Hello reddit!\\nI'm 22 years old living in Cali...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1489553952</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Help with fund choices and allocating between ...</td>\n",
       "      <td>through_her_skull</td>\n",
       "      <td>Hello PF, I'm entering the workforce at a some...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1489553885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Credit card fraudulent charges, but i am trave...</td>\n",
       "      <td>captnaimerica</td>\n",
       "      <td>My Chase Sapphire Reserve credit card has thre...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>5</td>\n",
       "      <td>1489553777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>My dad recently passed away. My mom and I need...</td>\n",
       "      <td>smexyfish</td>\n",
       "      <td>My dad was our main source of income through S...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1489553766</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Pay down chunk of personal loan with ESPP funds?</td>\n",
       "      <td>zyz479</td>\n",
       "      <td>Hi, first time poster here (on reddit ever), l...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1489553652</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165219</th>\n",
       "      <td>The world is at my feet and I have no idea wha...</td>\n",
       "      <td>[deleted]</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1489553256</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165220</th>\n",
       "      <td>I'm 17 and for the first time, had a job over ...</td>\n",
       "      <td>notmenotmenotme123</td>\n",
       "      <td>Hi all, so as I was saying, I'm 17 and turn 18...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1489553125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165221</th>\n",
       "      <td>When to get new credit card?</td>\n",
       "      <td>TaxThatAssMaster</td>\n",
       "      <td>Hey all,  just a bit of background... relative...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1489552916</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165222</th>\n",
       "      <td>Applied for 2 BoA CCs; Ticked Resident Alien t...</td>\n",
       "      <td>reubTV</td>\n",
       "      <td>TLDR: Its late at night, I was applying for 2 ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1489552227</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>165223</th>\n",
       "      <td>Does opening credit cards affect mortgage rate?</td>\n",
       "      <td>11v3t</td>\n",
       "      <td>I'm building a house and opened a credit card,...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1489551940</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>165224 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title              author  \\\n",
       "0            22yr old assist wz looking for car insurance          010101111x   \n",
       "1       Help with fund choices and allocating between ...   through_her_skull   \n",
       "2       Credit card fraudulent charges, but i am trave...       captnaimerica   \n",
       "3       My dad recently passed away. My mom and I need...           smexyfish   \n",
       "4        Pay down chunk of personal loan with ESPP funds?              zyz479   \n",
       "...                                                   ...                 ...   \n",
       "165219  The world is at my feet and I have no idea wha...           [deleted]   \n",
       "165220  I'm 17 and for the first time, had a job over ...  notmenotmenotme123   \n",
       "165221                       When to get new credit card?    TaxThatAssMaster   \n",
       "165222  Applied for 2 BoA CCs; Ticked Resident Alien t...              reubTV   \n",
       "165223    Does opening credit cards affect mortgage rate?               11v3t   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0       Hello reddit!\\nI'm 22 years old living in Cali...   \n",
       "1       Hello PF, I'm entering the workforce at a some...   \n",
       "2       My Chase Sapphire Reserve credit card has thre...   \n",
       "3       My dad was our main source of income through S...   \n",
       "4       Hi, first time poster here (on reddit ever), l...   \n",
       "...                                                   ...   \n",
       "165219                                          [removed]   \n",
       "165220  Hi all, so as I was saying, I'm 17 and turn 18...   \n",
       "165221  Hey all,  just a bit of background... relative...   \n",
       "165222  TLDR: Its late at night, I was applying for 2 ...   \n",
       "165223  I'm building a house and opened a credit card,...   \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       https://www.reddit.com/r/personalfinance/comme...      1   1489553952  \n",
       "1       https://www.reddit.com/r/personalfinance/comme...      1   1489553885  \n",
       "2       https://www.reddit.com/r/personalfinance/comme...      5   1489553777  \n",
       "3       https://www.reddit.com/r/personalfinance/comme...      2   1489553766  \n",
       "4       https://www.reddit.com/r/personalfinance/comme...      1   1489553652  \n",
       "...                                                   ...    ...          ...  \n",
       "165219  https://www.reddit.com/r/personalfinance/comme...      1   1489553256  \n",
       "165220  https://www.reddit.com/r/personalfinance/comme...      1   1489553125  \n",
       "165221  https://www.reddit.com/r/personalfinance/comme...      2   1489552916  \n",
       "165222  https://www.reddit.com/r/personalfinance/comme...      2   1489552227  \n",
       "165223  https://www.reddit.com/r/personalfinance/comme...      1   1489551940  \n",
       "\n",
       "[165224 rows x 6 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2017-01-01'\n",
    "before_date = '2018-01-01'\n",
    "\n",
    "submissions_pf_2017 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2017)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "operational-medline",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "164684 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 78.00% - Requests: 100 - Batches: 10 - Items Remaining: 156884\n",
      "Checkpoint:: Success Rate: 81.50% - Requests: 200 - Batches: 20 - Items Remaining: 148384\n",
      "Checkpoint:: Success Rate: 78.00% - Requests: 300 - Batches: 30 - Items Remaining: 141314\n",
      "Checkpoint:: Success Rate: 80.00% - Requests: 400 - Batches: 40 - Items Remaining: 132714\n",
      "Checkpoint:: Success Rate: 80.40% - Requests: 500 - Batches: 50 - Items Remaining: 124514\n",
      "Checkpoint:: Success Rate: 81.17% - Requests: 600 - Batches: 60 - Items Remaining: 116064\n",
      "Checkpoint:: Success Rate: 82.43% - Requests: 700 - Batches: 70 - Items Remaining: 107198\n",
      "Checkpoint:: Success Rate: 82.88% - Requests: 800 - Batches: 80 - Items Remaining: 98725\n",
      "Checkpoint:: Success Rate: 82.56% - Requests: 900 - Batches: 90 - Items Remaining: 90922\n",
      "Checkpoint:: Success Rate: 82.50% - Requests: 1000 - Batches: 100 - Items Remaining: 82832\n",
      "Checkpoint:: Success Rate: 82.36% - Requests: 1100 - Batches: 110 - Items Remaining: 75028\n",
      "Checkpoint:: Success Rate: 82.00% - Requests: 1200 - Batches: 120 - Items Remaining: 67557\n",
      "Checkpoint:: Success Rate: 81.77% - Requests: 1300 - Batches: 130 - Items Remaining: 60104\n",
      "Checkpoint:: Success Rate: 82.43% - Requests: 1400 - Batches: 140 - Items Remaining: 52696\n",
      "Checkpoint:: Success Rate: 81.80% - Requests: 1500 - Batches: 150 - Items Remaining: 47460\n",
      "Checkpoint:: Success Rate: 82.06% - Requests: 1600 - Batches: 160 - Items Remaining: 40644\n",
      "Checkpoint:: Success Rate: 82.06% - Requests: 1700 - Batches: 170 - Items Remaining: 33328\n",
      "Checkpoint:: Success Rate: 81.89% - Requests: 1800 - Batches: 180 - Items Remaining: 26916\n",
      "Checkpoint:: Success Rate: 81.84% - Requests: 1900 - Batches: 190 - Items Remaining: 20811\n",
      "Checkpoint:: Success Rate: 82.10% - Requests: 2000 - Batches: 200 - Items Remaining: 15469\n",
      "Checkpoint:: Success Rate: 81.95% - Requests: 2100 - Batches: 210 - Items Remaining: 11666\n",
      "Checkpoint:: Success Rate: 82.18% - Requests: 2200 - Batches: 220 - Items Remaining: 6566\n",
      "Checkpoint:: Success Rate: 81.91% - Requests: 2300 - Batches: 230 - Items Remaining: 2794\n",
      "Total:: Success Rate: 81.99% - Requests: 2371 - Batches: 238 - Items Remaining: 790\n",
      "Total:: Success Rate: 82.02% - Requests: 2380 - Batches: 239 - Items Remaining: 0\n",
      "Retrieved 164684 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Need guidance. One of my employers did not ded...</td>\n",
       "      <td>hitenken</td>\n",
       "      <td>I am college student. In 2017 I worked 3 part ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1517936127</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Does anyone use their car for work all the tim...</td>\n",
       "      <td>Marinomelissa2</td>\n",
       "      <td></td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1517936117</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Signed a job offer and trying to begin budgeting.</td>\n",
       "      <td>dayyumson</td>\n",
       "      <td>Hello, I just signed a job offer for $40k base...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1517936114</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>What to do with $100,000 I don't need</td>\n",
       "      <td>Thr0wAway23445</td>\n",
       "      <td>Hello,\\n\\nI am 21 and in college. I receive di...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1517936098</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Success story: goodwill letter to Synchrony</td>\n",
       "      <td>ragnar05</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1517935904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164679</th>\n",
       "      <td>Anyone know of credit cards with same-day tran...</td>\n",
       "      <td>autoshag</td>\n",
       "      <td>Sorry if this is the wrong place to post this....</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1543135019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164680</th>\n",
       "      <td>If I kill myself does it void my life insuranc...</td>\n",
       "      <td>Anonymenace</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1543134699</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164681</th>\n",
       "      <td>Need help, moving out with nothing but a car a...</td>\n",
       "      <td>newlifeplshelp</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1543134403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164682</th>\n",
       "      <td>I (21m) have a little over $1,000 in credit ca...</td>\n",
       "      <td>BiggerestGreen</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1543134267</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>164683</th>\n",
       "      <td>Student Loans, yuck.</td>\n",
       "      <td>viatone</td>\n",
       "      <td>Here’s the situation. My dad took out loans fo...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1543134158</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>164684 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title          author  \\\n",
       "0       Need guidance. One of my employers did not ded...        hitenken   \n",
       "1       Does anyone use their car for work all the tim...  Marinomelissa2   \n",
       "2       Signed a job offer and trying to begin budgeting.       dayyumson   \n",
       "3                   What to do with $100,000 I don't need  Thr0wAway23445   \n",
       "4             Success story: goodwill letter to Synchrony        ragnar05   \n",
       "...                                                   ...             ...   \n",
       "164679  Anyone know of credit cards with same-day tran...        autoshag   \n",
       "164680  If I kill myself does it void my life insuranc...     Anonymenace   \n",
       "164681  Need help, moving out with nothing but a car a...  newlifeplshelp   \n",
       "164682  I (21m) have a little over $1,000 in credit ca...  BiggerestGreen   \n",
       "164683                               Student Loans, yuck.         viatone   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0       I am college student. In 2017 I worked 3 part ...   \n",
       "1                                                           \n",
       "2       Hello, I just signed a job offer for $40k base...   \n",
       "3       Hello,\\n\\nI am 21 and in college. I receive di...   \n",
       "4                                               [removed]   \n",
       "...                                                   ...   \n",
       "164679  Sorry if this is the wrong place to post this....   \n",
       "164680                                          [removed]   \n",
       "164681                                          [removed]   \n",
       "164682                                          [removed]   \n",
       "164683  Here’s the situation. My dad took out loans fo...   \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       https://www.reddit.com/r/personalfinance/comme...      0   1517936127  \n",
       "1       https://www.reddit.com/r/personalfinance/comme...      1   1517936117  \n",
       "2       https://www.reddit.com/r/personalfinance/comme...      1   1517936114  \n",
       "3       https://www.reddit.com/r/personalfinance/comme...      0   1517936098  \n",
       "4       https://www.reddit.com/r/personalfinance/comme...      1   1517935904  \n",
       "...                                                   ...    ...          ...  \n",
       "164679  https://www.reddit.com/r/personalfinance/comme...      1   1543135019  \n",
       "164680  https://www.reddit.com/r/personalfinance/comme...      1   1543134699  \n",
       "164681  https://www.reddit.com/r/personalfinance/comme...      1   1543134403  \n",
       "164682  https://www.reddit.com/r/personalfinance/comme...      1   1543134267  \n",
       "164683  https://www.reddit.com/r/personalfinance/comme...      1   1543134158  \n",
       "\n",
       "[164684 rows x 6 columns]"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2018-01-01'\n",
    "before_date = '2019-01-01'\n",
    "\n",
    "submissions_pf_2018 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "regional-constitution",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "169378 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 86.00% - Requests: 100 - Batches: 10 - Items Remaining: 160778\n",
      "Checkpoint:: Success Rate: 83.50% - Requests: 200 - Batches: 20 - Items Remaining: 152678\n",
      "Checkpoint:: Success Rate: 79.67% - Requests: 300 - Batches: 30 - Items Remaining: 145478\n",
      "Checkpoint:: Success Rate: 77.50% - Requests: 400 - Batches: 40 - Items Remaining: 138378\n",
      "Checkpoint:: Success Rate: 76.40% - Requests: 500 - Batches: 50 - Items Remaining: 131178\n",
      "Checkpoint:: Success Rate: 77.00% - Requests: 600 - Batches: 60 - Items Remaining: 123178\n",
      "Checkpoint:: Success Rate: 75.57% - Requests: 700 - Batches: 70 - Items Remaining: 116478\n",
      "Checkpoint:: Success Rate: 75.75% - Requests: 800 - Batches: 80 - Items Remaining: 108778\n",
      "Checkpoint:: Success Rate: 74.22% - Requests: 900 - Batches: 90 - Items Remaining: 102645\n",
      "Checkpoint:: Success Rate: 72.70% - Requests: 1000 - Batches: 100 - Items Remaining: 96919\n",
      "Checkpoint:: Success Rate: 72.27% - Requests: 1100 - Batches: 110 - Items Remaining: 90416\n",
      "Checkpoint:: Success Rate: 73.25% - Requests: 1200 - Batches: 120 - Items Remaining: 82290\n",
      "Checkpoint:: Success Rate: 73.85% - Requests: 1300 - Batches: 130 - Items Remaining: 74344\n",
      "Checkpoint:: Success Rate: 74.43% - Requests: 1400 - Batches: 140 - Items Remaining: 66379\n",
      "Checkpoint:: Success Rate: 74.73% - Requests: 1500 - Batches: 150 - Items Remaining: 58669\n",
      "Checkpoint:: Success Rate: 75.00% - Requests: 1600 - Batches: 160 - Items Remaining: 51066\n",
      "Checkpoint:: Success Rate: 74.35% - Requests: 1700 - Batches: 170 - Items Remaining: 45581\n",
      "Checkpoint:: Success Rate: 74.00% - Requests: 1800 - Batches: 180 - Items Remaining: 40119\n",
      "Checkpoint:: Success Rate: 74.21% - Requests: 1900 - Batches: 190 - Items Remaining: 34145\n",
      "Checkpoint:: Success Rate: 73.65% - Requests: 2000 - Batches: 200 - Items Remaining: 29740\n",
      "Checkpoint:: Success Rate: 73.52% - Requests: 2100 - Batches: 210 - Items Remaining: 24497\n",
      "Checkpoint:: Success Rate: 73.45% - Requests: 2200 - Batches: 220 - Items Remaining: 19378\n",
      "Checkpoint:: Success Rate: 72.87% - Requests: 2300 - Batches: 230 - Items Remaining: 15657\n",
      "Checkpoint:: Success Rate: 72.42% - Requests: 2400 - Batches: 240 - Items Remaining: 12116\n",
      "Checkpoint:: Success Rate: 72.56% - Requests: 2500 - Batches: 250 - Items Remaining: 7600\n",
      "Checkpoint:: Success Rate: 71.77% - Requests: 2600 - Batches: 260 - Items Remaining: 5740\n",
      "Checkpoint:: Success Rate: 72.15% - Requests: 2700 - Batches: 270 - Items Remaining: 2870\n",
      "Checkpoint:: Success Rate: 72.30% - Requests: 2791 - Batches: 280 - Items Remaining: 9\n",
      "Total:: Success Rate: 72.30% - Requests: 2791 - Batches: 280 - Items Remaining: 9\n",
      "Total:: Success Rate: 72.31% - Requests: 2792 - Batches: 281 - Items Remaining: 0\n",
      "Retrieved 169378 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Juat a thsnk you to this aubreddit. Never post...</td>\n",
       "      <td>CoolHandRK1</td>\n",
       "      <td>I just got my latest credit score report. For ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1549472353</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Let's retire millionaires</td>\n",
       "      <td>Nat643</td>\n",
       "      <td>https://www.facebook.com/groups/34277696299261...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1549472346</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Housing trends for Austin: Timing and what to ...</td>\n",
       "      <td>WildWolfOfMibu</td>\n",
       "      <td>I recently got my first Regular full-time job ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1549472118</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Paying off mortgage early with new tax laws.</td>\n",
       "      <td>Flo_Evans</td>\n",
       "      <td>It seems like with the new higher standard ded...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1549471777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Sep ira- main employer/ freelance</td>\n",
       "      <td>TimDeQuatro</td>\n",
       "      <td>Hi everyone, so my main employer matches conti...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1549471689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169373</th>\n",
       "      <td>How does non-dealer financing for a car work?</td>\n",
       "      <td>SelectTadpole</td>\n",
       "      <td>I recently went to go purchase a vehicle (wow ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>2</td>\n",
       "      <td>1558930891</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169374</th>\n",
       "      <td>Collections for $30.96</td>\n",
       "      <td>keklmao123</td>\n",
       "      <td>I just received a letter in the mail from a co...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1558930607</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169375</th>\n",
       "      <td>Chronic insomniac trying to save up enough mon...</td>\n",
       "      <td>dhsbbbx</td>\n",
       "      <td>Hi all,\\nI’m 30 years old, and have a chronic ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1558930430</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169376</th>\n",
       "      <td>Buy a house out of college or move into bigger...</td>\n",
       "      <td>AndersonEm</td>\n",
       "      <td>In exactly a year, I will be graduating with a...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>0</td>\n",
       "      <td>1558930429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>169377</th>\n",
       "      <td>Switching to Ally Bank, leave Capital One 360 ...</td>\n",
       "      <td>MagicJMarker</td>\n",
       "      <td>I currently have savings and checking accounts...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1558929987</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>169378 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title          author  \\\n",
       "0       Juat a thsnk you to this aubreddit. Never post...     CoolHandRK1   \n",
       "1                               Let's retire millionaires          Nat643   \n",
       "2       Housing trends for Austin: Timing and what to ...  WildWolfOfMibu   \n",
       "3            Paying off mortgage early with new tax laws.       Flo_Evans   \n",
       "4                       Sep ira- main employer/ freelance     TimDeQuatro   \n",
       "...                                                   ...             ...   \n",
       "169373      How does non-dealer financing for a car work?   SelectTadpole   \n",
       "169374                             Collections for $30.96      keklmao123   \n",
       "169375  Chronic insomniac trying to save up enough mon...         dhsbbbx   \n",
       "169376  Buy a house out of college or move into bigger...      AndersonEm   \n",
       "169377  Switching to Ally Bank, leave Capital One 360 ...    MagicJMarker   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0       I just got my latest credit score report. For ...   \n",
       "1       https://www.facebook.com/groups/34277696299261...   \n",
       "2       I recently got my first Regular full-time job ...   \n",
       "3       It seems like with the new higher standard ded...   \n",
       "4       Hi everyone, so my main employer matches conti...   \n",
       "...                                                   ...   \n",
       "169373  I recently went to go purchase a vehicle (wow ...   \n",
       "169374  I just received a letter in the mail from a co...   \n",
       "169375  Hi all,\\nI’m 30 years old, and have a chronic ...   \n",
       "169376  In exactly a year, I will be graduating with a...   \n",
       "169377  I currently have savings and checking accounts...   \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       https://www.reddit.com/r/personalfinance/comme...      1   1549472353  \n",
       "1       https://www.reddit.com/r/personalfinance/comme...      1   1549472346  \n",
       "2       https://www.reddit.com/r/personalfinance/comme...      1   1549472118  \n",
       "3       https://www.reddit.com/r/personalfinance/comme...      1   1549471777  \n",
       "4       https://www.reddit.com/r/personalfinance/comme...      1   1549471689  \n",
       "...                                                   ...    ...          ...  \n",
       "169373  https://www.reddit.com/r/personalfinance/comme...      2   1558930891  \n",
       "169374  https://www.reddit.com/r/personalfinance/comme...      1   1558930607  \n",
       "169375  https://www.reddit.com/r/personalfinance/comme...      0   1558930430  \n",
       "169376  https://www.reddit.com/r/personalfinance/comme...      0   1558930429  \n",
       "169377  https://www.reddit.com/r/personalfinance/comme...      1   1558929987  \n",
       "\n",
       "[169378 rows x 6 columns]"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 10000000\n",
    "\n",
    "after_date = '2019-01-01'\n",
    "before_date = '2020-01-01'\n",
    "\n",
    "submissions_pf_2019 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2019)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "figured-albany",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "101768 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 77.00% - Requests: 100 - Batches: 10 - Items Remaining: 94068\n",
      "Checkpoint:: Success Rate: 79.00% - Requests: 200 - Batches: 20 - Items Remaining: 85968\n",
      "Checkpoint:: Success Rate: 79.67% - Requests: 300 - Batches: 30 - Items Remaining: 77868\n",
      "Checkpoint:: Success Rate: 79.25% - Requests: 400 - Batches: 40 - Items Remaining: 70090\n",
      "Checkpoint:: Success Rate: 80.00% - Requests: 500 - Batches: 50 - Items Remaining: 61855\n",
      "Checkpoint:: Success Rate: 78.17% - Requests: 600 - Batches: 60 - Items Remaining: 54980\n",
      "Checkpoint:: Success Rate: 79.86% - Requests: 700 - Batches: 70 - Items Remaining: 45980\n",
      "Checkpoint:: Success Rate: 81.00% - Requests: 800 - Batches: 80 - Items Remaining: 37569\n",
      "Checkpoint:: Success Rate: 81.33% - Requests: 900 - Batches: 90 - Items Remaining: 30265\n",
      "Checkpoint:: Success Rate: 82.40% - Requests: 1000 - Batches: 100 - Items Remaining: 22563\n",
      "Checkpoint:: Success Rate: 83.27% - Requests: 1100 - Batches: 110 - Items Remaining: 15000\n",
      "Checkpoint:: Success Rate: 83.25% - Requests: 1200 - Batches: 120 - Items Remaining: 8948\n",
      "Checkpoint:: Success Rate: 83.15% - Requests: 1300 - Batches: 130 - Items Remaining: 5504\n",
      "Checkpoint:: Success Rate: 83.71% - Requests: 1400 - Batches: 140 - Items Remaining: 1718\n",
      "Total:: Success Rate: 83.98% - Requests: 1454 - Batches: 146 - Items Remaining: 7\n",
      "Total:: Success Rate: 83.76% - Requests: 1459 - Batches: 147 - Items Remaining: 0\n",
      "Retrieved 101768 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>My GF is really bad with her money</td>\n",
       "      <td>Anonymous20010</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1584143253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Trade in car or keep?</td>\n",
       "      <td>crosstitchchampion</td>\n",
       "      <td>So we have a Jeep Grand Cherokee and we averag...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1584143188</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>About to purchase my first home, should I cont...</td>\n",
       "      <td>mechanicallyinsane</td>\n",
       "      <td>I am about to purchase my first home, the clos...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1584142847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>How much should my SO charge me in rent?</td>\n",
       "      <td>newgradneedsjob</td>\n",
       "      <td>I'm currently living with roommates, paying $8...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1584142824</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Trump has waived student loan interest in resp...</td>\n",
       "      <td>sanc13579</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1584142600</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101763</th>\n",
       "      <td>Should I even pay collections?</td>\n",
       "      <td>throwaway1949374829</td>\n",
       "      <td>I got stuck with a few thousand dollar collect...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1579424471</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101764</th>\n",
       "      <td>Is it worth doing a strategic default on a hig...</td>\n",
       "      <td>TheDessert_Fox</td>\n",
       "      <td>Or should I avoid the credit hit and just pay ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1579424044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101765</th>\n",
       "      <td>Clean slate - with which brokerage would you p...</td>\n",
       "      <td>kodridrocl</td>\n",
       "      <td>I just moved all my brokerage from Ameritrade ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1579423549</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101766</th>\n",
       "      <td>Clean slate - with which brokerage would you p...</td>\n",
       "      <td>kodridrocl</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1579422543</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101767</th>\n",
       "      <td>$30,000 as a TEENAGER. What to do now??</td>\n",
       "      <td>kobelow</td>\n",
       "      <td>Hey reddit I need some life advice. I’m 19 and...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1579421648</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>101768 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    title  \\\n",
       "0                      My GF is really bad with her money   \n",
       "1                                   Trade in car or keep?   \n",
       "2       About to purchase my first home, should I cont...   \n",
       "3                How much should my SO charge me in rent?   \n",
       "4       Trump has waived student loan interest in resp...   \n",
       "...                                                   ...   \n",
       "101763                     Should I even pay collections?   \n",
       "101764  Is it worth doing a strategic default on a hig...   \n",
       "101765  Clean slate - with which brokerage would you p...   \n",
       "101766  Clean slate - with which brokerage would you p...   \n",
       "101767            $30,000 as a TEENAGER. What to do now??   \n",
       "\n",
       "                     author  \\\n",
       "0            Anonymous20010   \n",
       "1        crosstitchchampion   \n",
       "2        mechanicallyinsane   \n",
       "3           newgradneedsjob   \n",
       "4                 sanc13579   \n",
       "...                     ...   \n",
       "101763  throwaway1949374829   \n",
       "101764       TheDessert_Fox   \n",
       "101765           kodridrocl   \n",
       "101766           kodridrocl   \n",
       "101767              kobelow   \n",
       "\n",
       "                                                 selftext  \\\n",
       "0                                               [removed]   \n",
       "1       So we have a Jeep Grand Cherokee and we averag...   \n",
       "2       I am about to purchase my first home, the clos...   \n",
       "3       I'm currently living with roommates, paying $8...   \n",
       "4                                               [removed]   \n",
       "...                                                   ...   \n",
       "101763  I got stuck with a few thousand dollar collect...   \n",
       "101764  Or should I avoid the credit hit and just pay ...   \n",
       "101765  I just moved all my brokerage from Ameritrade ...   \n",
       "101766                                          [removed]   \n",
       "101767  Hey reddit I need some life advice. I’m 19 and...   \n",
       "\n",
       "                                                      url  score  created_utc  \n",
       "0       https://www.reddit.com/r/personalfinance/comme...      1   1584143253  \n",
       "1       https://www.reddit.com/r/personalfinance/comme...      1   1584143188  \n",
       "2       https://www.reddit.com/r/personalfinance/comme...      1   1584142847  \n",
       "3       https://www.reddit.com/r/personalfinance/comme...      1   1584142824  \n",
       "4       https://www.reddit.com/r/personalfinance/comme...      1   1584142600  \n",
       "...                                                   ...    ...          ...  \n",
       "101763  https://www.reddit.com/r/personalfinance/comme...      1   1579424471  \n",
       "101764  https://www.reddit.com/r/personalfinance/comme...      1   1579424044  \n",
       "101765  https://www.reddit.com/r/personalfinance/comme...      1   1579423549  \n",
       "101766  https://www.reddit.com/r/personalfinance/comme...      1   1579422543  \n",
       "101767  https://www.reddit.com/r/personalfinance/comme...      1   1579421648  \n",
       "\n",
       "[101768 rows x 6 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 1000000000\n",
    "\n",
    "after_date = '2020-01-01'\n",
    "before_date = '2020-07-01'\n",
    "\n",
    "submissions_pf_2020_1 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2020_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "nuclear-impression",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "78531 results available in Pushshift\n",
      "Checkpoint:: Success Rate: 82.00% - Requests: 100 - Batches: 10 - Items Remaining: 70331\n",
      "Checkpoint:: Success Rate: 78.00% - Requests: 200 - Batches: 20 - Items Remaining: 62931\n",
      "Checkpoint:: Success Rate: 77.00% - Requests: 300 - Batches: 30 - Items Remaining: 55484\n",
      "Checkpoint:: Success Rate: 76.00% - Requests: 400 - Batches: 40 - Items Remaining: 48244\n",
      "Checkpoint:: Success Rate: 72.20% - Requests: 500 - Batches: 50 - Items Remaining: 42729\n",
      "Checkpoint:: Success Rate: 72.33% - Requests: 600 - Batches: 60 - Items Remaining: 35577\n",
      "Checkpoint:: Success Rate: 72.71% - Requests: 700 - Batches: 70 - Items Remaining: 28773\n",
      "Checkpoint:: Success Rate: 72.62% - Requests: 800 - Batches: 80 - Items Remaining: 22918\n",
      "Checkpoint:: Success Rate: 72.44% - Requests: 900 - Batches: 90 - Items Remaining: 17010\n",
      "Checkpoint:: Success Rate: 71.80% - Requests: 1000 - Batches: 100 - Items Remaining: 11194\n",
      "Checkpoint:: Success Rate: 71.45% - Requests: 1100 - Batches: 110 - Items Remaining: 6753\n",
      "Checkpoint:: Success Rate: 70.92% - Requests: 1200 - Batches: 120 - Items Remaining: 3994\n",
      "Checkpoint:: Success Rate: 70.54% - Requests: 1300 - Batches: 130 - Items Remaining: 741\n",
      "Total:: Success Rate: 70.95% - Requests: 1322 - Batches: 133 - Items Remaining: 2\n",
      "Total:: Success Rate: 70.81% - Requests: 1326 - Batches: 134 - Items Remaining: 0\n",
      "Retrieved 78531 submissions from Pushshift\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>author</th>\n",
       "      <th>selftext</th>\n",
       "      <th>url</th>\n",
       "      <th>score</th>\n",
       "      <th>created_utc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Is buying a house for the short term a good idea?</td>\n",
       "      <td>vortexmak</td>\n",
       "      <td>I like the condo building I'm renting in. But ...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1596756112</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>What to say if you have a car loan and need an...</td>\n",
       "      <td>MinMaxRex</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1596756012</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Discrepancy between Loan payment amounts</td>\n",
       "      <td>zilfondel</td>\n",
       "      <td>Currently shopping for a car, did some estimat...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1596756009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Looking for the best way to take over Direct P...</td>\n",
       "      <td>awkwardandalkaline</td>\n",
       "      <td>On mobile, first post to sub, etc. This is a l...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1596755763</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HELP!! Recent grad having trouble wrapping my ...</td>\n",
       "      <td>yesokay1</td>\n",
       "      <td>Hello! I'm 23, a recent grad, and will begin m...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1596755657</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78526</th>\n",
       "      <td>Recieved a letter from creditor</td>\n",
       "      <td>The_Texidian</td>\n",
       "      <td>My mom died last year. She left an estate of $...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1599683686</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78527</th>\n",
       "      <td>Explain a discretionary profit sharing 401(k).</td>\n",
       "      <td>BigMickBorger</td>\n",
       "      <td>I’m about to switch to a private company after...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1599683589</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78528</th>\n",
       "      <td>Tax Deferred Saving?</td>\n",
       "      <td>TheNicestRedditor</td>\n",
       "      <td>(26 years old)\\n\\nSalary: $55,000/yr \\n\\nSavin...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1599683527</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78529</th>\n",
       "      <td>85k Miles on my entry level luxury car - Shoul...</td>\n",
       "      <td>Tentcell2</td>\n",
       "      <td>[removed]</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1607886972</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>78530</th>\n",
       "      <td>X-posted: Roth Conversion with excess contribu...</td>\n",
       "      <td>Yousmith</td>\n",
       "      <td>I was laid off from my previous job in May and...</td>\n",
       "      <td>https://www.reddit.com/r/personalfinance/comme...</td>\n",
       "      <td>1</td>\n",
       "      <td>1607886938</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>78531 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                   title              author  \\\n",
       "0      Is buying a house for the short term a good idea?           vortexmak   \n",
       "1      What to say if you have a car loan and need an...           MinMaxRex   \n",
       "2               Discrepancy between Loan payment amounts           zilfondel   \n",
       "3      Looking for the best way to take over Direct P...  awkwardandalkaline   \n",
       "4      HELP!! Recent grad having trouble wrapping my ...            yesokay1   \n",
       "...                                                  ...                 ...   \n",
       "78526                    Recieved a letter from creditor        The_Texidian   \n",
       "78527     Explain a discretionary profit sharing 401(k).       BigMickBorger   \n",
       "78528                               Tax Deferred Saving?   TheNicestRedditor   \n",
       "78529  85k Miles on my entry level luxury car - Shoul...           Tentcell2   \n",
       "78530  X-posted: Roth Conversion with excess contribu...            Yousmith   \n",
       "\n",
       "                                                selftext  \\\n",
       "0      I like the condo building I'm renting in. But ...   \n",
       "1                                              [removed]   \n",
       "2      Currently shopping for a car, did some estimat...   \n",
       "3      On mobile, first post to sub, etc. This is a l...   \n",
       "4      Hello! I'm 23, a recent grad, and will begin m...   \n",
       "...                                                  ...   \n",
       "78526  My mom died last year. She left an estate of $...   \n",
       "78527  I’m about to switch to a private company after...   \n",
       "78528  (26 years old)\\n\\nSalary: $55,000/yr \\n\\nSavin...   \n",
       "78529                                          [removed]   \n",
       "78530  I was laid off from my previous job in May and...   \n",
       "\n",
       "                                                     url  score  created_utc  \n",
       "0      https://www.reddit.com/r/personalfinance/comme...      1   1596756112  \n",
       "1      https://www.reddit.com/r/personalfinance/comme...      1   1596756012  \n",
       "2      https://www.reddit.com/r/personalfinance/comme...      1   1596756009  \n",
       "3      https://www.reddit.com/r/personalfinance/comme...      1   1596755763  \n",
       "4      https://www.reddit.com/r/personalfinance/comme...      1   1596755657  \n",
       "...                                                  ...    ...          ...  \n",
       "78526  https://www.reddit.com/r/personalfinance/comme...      1   1599683686  \n",
       "78527  https://www.reddit.com/r/personalfinance/comme...      1   1599683589  \n",
       "78528  https://www.reddit.com/r/personalfinance/comme...      1   1599683527  \n",
       "78529  https://www.reddit.com/r/personalfinance/comme...      1   1607886972  \n",
       "78530  https://www.reddit.com/r/personalfinance/comme...      1   1607886938  \n",
       "\n",
       "[78531 rows x 6 columns]"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit = \"personalfinance\"\n",
    "limit = 1000000000\n",
    "\n",
    "after_date = '2020-07-01'\n",
    "before_date = '2021-01-01'\n",
    "\n",
    "submissions_pf_2020_2 = scrape(subreddit, limit, before_date, after_date)\n",
    "\n",
    "selected_columns(submissions_pf_2020_2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "understanding-performer",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}