hurriyet_daily_news_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import  requests \n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "from urllib.error import URLError, HTTPError\n",
    "from urllib3.exceptions import SSLError\n",
    "\n",
    "from requests.exceptions import HTTPError\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlopen, Request\n",
    "from socket import timeout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.hurriyetdailynews.com/search/thy?p=\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.hurriyetdailynews.com/search/'\n",
    "# keyword = 'akbank'\n",
    "keyword = 'thy'\n",
    "\n",
    "keyword2 = '%20'.join(keyword.split())\n",
    "keyword3 = '_'.join(keyword.split())\n",
    "\n",
    "url = url + keyword2 + '?p='\n",
    "print(url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total news:  53\n",
      "1.0392156862745099\n"
     ]
    }
   ],
   "source": [
    "try: \n",
    "    \n",
    "    url2 = url + '1'\n",
    "    \n",
    "    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "    reg_url = url2\n",
    "    req = Request(url=reg_url, headers=headers) \n",
    "    html = urlopen(req).read()  \n",
    "\n",
    "    soup = BeautifulSoup(html, \"html.parser\")\n",
    "\n",
    "    data = soup.find('p',{'class':'search-result-summary'})\n",
    "    n_news = int(data.text.split()[1]) \n",
    "    print('Total news: ',n_news)\n",
    "        \n",
    "            \n",
    "except HTTPError as http_err:\n",
    "    print(f'HTTP error occurred: {http_err}')  \n",
    "    pass\n",
    "except Exception as err:\n",
    "    print(f'Other1 error occurred: {err}')  \n",
    "    pass\n",
    "\n",
    "print(n_news/51)\n",
    "if str(n_news/51).split('.')[1][0]  != '0':\n",
    "    pages = int(n_news/51)+2\n",
    "else:\n",
    "    pages = int(n_news/51)+1\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getArticle(link):\n",
    "    \n",
    "    try: \n",
    "        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "        reg_url = link\n",
    "        req = Request(url=reg_url, headers=headers) \n",
    "        html = urlopen(req).read()  \n",
    "\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "\n",
    "        data = soup.find('li',{'class':'date'})\n",
    "        date = data.text\n",
    "        data = soup.findAll('p')\n",
    "        temp = []\n",
    "        for p in data:\n",
    "            temp.append(p.text)\n",
    "        text = ' '.join(temp)\n",
    "        \n",
    "        return [text,date]\n",
    "            \n",
    "    except HTTPError as http_err:\n",
    "#         print(f'HTTP error occurred: {http_err}')  \n",
    "        pass\n",
    "    except Exception as err:\n",
    "#         print(f'Other1 error occurred: {err}')  \n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1\n",
      "n of news on current page:  51\n",
      "12\n"
     ]
    }
   ],
   "source": [
    "try: \n",
    "    headlines = []\n",
    "    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "    \n",
    "    for i in range(1,pages,1):\n",
    "        print(i)\n",
    "        url3 = url+str(i)\n",
    "#         print()\n",
    "        reg_url = url3\n",
    "        req = Request(url=reg_url, headers=headers) \n",
    "        html = urlopen(req).read()  \n",
    "\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "\n",
    "        data = soup.findAll('div',{'class':'col-md-4 col-sm-4'})\n",
    "        news_found = len(data)\n",
    "    \n",
    "        print('n of news on current page: ',news_found)\n",
    "    \n",
    "        for i in range(0,news_found,1):\n",
    "            headline = data[i].find('h3').text\n",
    "            link = str(data[i].find('a'))\n",
    "            link = 'https://www.hurriyetdailynews.com/'+link.split('href=\"')[1].split('\"')[0]\n",
    "            article, date = getArticle(link)\n",
    "            headlines.append([headline,article,date,link])\n",
    "\n",
    "            \n",
    "except HTTPError as http_err:\n",
    "#     print(f'HTTP error occurred: {http_err}')  \n",
    "    pass\n",
    "except Exception as err:\n",
    "#     print(f'Other1 error occurred: {err}')  \n",
    "    pass\n",
    "    \n",
    "    \n",
    "print(len(headlines))\n",
    "    \n",
    "df = pd.DataFrame(headlines, columns =['headline','article','date','article_link'])\n",
    "df.to_csv(f'apis_datasets/{keyword3}_DailyNews_hurriet.csv', encoding='utf-8', index=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>headline</th>\n",
       "      <th>article</th>\n",
       "      <th>date</th>\n",
       "      <th>article_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Joseph Lowery, civil rights leader and MLK aid...</td>\n",
       "      <td>In this May 1970 file photo, Leonard Woodcock,...</td>\n",
       "      <td>March 28 2020 11:09:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//joseph-lowe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>THY weathers headwinds in tough year</td>\n",
       "      <td>Turkey’s flag carrier Turkish Airlines (THY) h...</td>\n",
       "      <td>October 23 2019 09:00:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//thy-weather...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Turkish Airlines should move to the new airpor...</td>\n",
       "      <td>Turkish Airlines (THY) will move to the new ai...</td>\n",
       "      <td>October 18 2018</td>\n",
       "      <td>https://www.hurriyetdailynews.com//opinion/ugu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Gaziantep cuisine sets eye on food Oscar</td>\n",
       "      <td>Thousands of asylum seekers on the Greek isla...</td>\n",
       "      <td>May 16 2017 00:01:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//gaziantep-c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Life across Turkey remains on hold amid contin...</td>\n",
       "      <td>AFP photo  Thousands of asylum seekers on the ...</td>\n",
       "      <td>January 10 2017 10:45:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//life-across...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Turkish Airlines flies to the Seychelles</td>\n",
       "      <td>Thousands of asylum seekers on the Greek isla...</td>\n",
       "      <td>November 04 2016 00:01:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//turkish-air...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Turkish Airlines CEO takes over as head of Aer...</td>\n",
       "      <td>Thousands of asylum seekers on the Greek isla...</td>\n",
       "      <td>October 21 2016 09:44:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//turkish-air...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Flights as usual after failed coup attempt, Tu...</td>\n",
       "      <td>REUTERS photo  Thousands of asylum seekers on ...</td>\n",
       "      <td>July 23 2016 00:01:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//flights-as-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>‘Motherland’ wins big at Ankara Film Fest</td>\n",
       "      <td>AA photo  Thousands of asylum seekers on the G...</td>\n",
       "      <td>May 09 2016 12:03:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//motherland-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Turkish Airlines ‘closing 10 pct gap from Russ...</td>\n",
       "      <td>CİHAN photo  Thousands of asylum seekers on th...</td>\n",
       "      <td>April 27 2016 14:07:00</td>\n",
       "      <td>https://www.hurriyetdailynews.com//turkish-air...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            headline  \\\n",
       "0  Joseph Lowery, civil rights leader and MLK aid...   \n",
       "1               THY weathers headwinds in tough year   \n",
       "2  Turkish Airlines should move to the new airpor...   \n",
       "3           Gaziantep cuisine sets eye on food Oscar   \n",
       "4  Life across Turkey remains on hold amid contin...   \n",
       "5           Turkish Airlines flies to the Seychelles   \n",
       "6  Turkish Airlines CEO takes over as head of Aer...   \n",
       "7  Flights as usual after failed coup attempt, Tu...   \n",
       "8          ‘Motherland’ wins big at Ankara Film Fest   \n",
       "9  Turkish Airlines ‘closing 10 pct gap from Russ...   \n",
       "\n",
       "                                             article  \\\n",
       "0  In this May 1970 file photo, Leonard Woodcock,...   \n",
       "1  Turkey’s flag carrier Turkish Airlines (THY) h...   \n",
       "2  Turkish Airlines (THY) will move to the new ai...   \n",
       "3   Thousands of asylum seekers on the Greek isla...   \n",
       "4  AFP photo  Thousands of asylum seekers on the ...   \n",
       "5   Thousands of asylum seekers on the Greek isla...   \n",
       "6   Thousands of asylum seekers on the Greek isla...   \n",
       "7  REUTERS photo  Thousands of asylum seekers on ...   \n",
       "8  AA photo  Thousands of asylum seekers on the G...   \n",
       "9  CİHAN photo  Thousands of asylum seekers on th...   \n",
       "\n",
       "                        date  \\\n",
       "0     March 28 2020 11:09:00   \n",
       "1   October 23 2019 09:00:00   \n",
       "2            October 18 2018   \n",
       "3       May 16 2017 00:01:00   \n",
       "4   January 10 2017 10:45:00   \n",
       "5  November 04 2016 00:01:00   \n",
       "6   October 21 2016 09:44:00   \n",
       "7      July 23 2016 00:01:00   \n",
       "8       May 09 2016 12:03:00   \n",
       "9     April 27 2016 14:07:00   \n",
       "\n",
       "                                        article_link  \n",
       "0  https://www.hurriyetdailynews.com//joseph-lowe...  \n",
       "1  https://www.hurriyetdailynews.com//thy-weather...  \n",
       "2  https://www.hurriyetdailynews.com//opinion/ugu...  \n",
       "3  https://www.hurriyetdailynews.com//gaziantep-c...  \n",
       "4  https://www.hurriyetdailynews.com//life-across...  \n",
       "5  https://www.hurriyetdailynews.com//turkish-air...  \n",
       "6  https://www.hurriyetdailynews.com//turkish-air...  \n",
       "7  https://www.hurriyetdailynews.com//flights-as-...  \n",
       "8  https://www.hurriyetdailynews.com//motherland-...  \n",
       "9  https://www.hurriyetdailynews.com//turkish-air...  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}