daily_sabah_news_api.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import  requests \n",
    "import time\n",
    "import pandas as pd\n",
    "\n",
    "from urllib.error import URLError, HTTPError\n",
    "from urllib3.exceptions import SSLError\n",
    "\n",
    "from requests.exceptions import HTTPError\n",
    "from bs4 import BeautifulSoup\n",
    "from urllib.request import urlopen, Request\n",
    "from socket import timeout"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getArticle(link):\n",
    "    \n",
    "    try: \n",
    "        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "        reg_url = link\n",
    "        req = Request(url=reg_url, headers=headers) \n",
    "        html = urlopen(req).read()  \n",
    "\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "#         data = soup.findAll('div',{'class':'article_top_slogan'})\n",
    "#         slogan = data[0].text\n",
    "        data = soup.findAll('div',{'class':'article_body'})\n",
    "        article = data[0].text\n",
    "        \n",
    "        return article\n",
    "\n",
    "    except HTTPError as http_err:\n",
    "#         print(f'HTTP error occurred: {http_err}')  \n",
    "        pass\n",
    "    except Exception as err:\n",
    "#         print(f'Other1 error occurred: {err}')  \n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.dailysabah.com/search?query=turkish+airlines&qlimit=by_fifty&pgno=\n",
      "https://www.dailysabah.com/search?query=turkish+airlines&qlimit=by_fifty&qsort=oldest&pgno=\n"
     ]
    }
   ],
   "source": [
    "search_input = 'Turkish Airlines'\n",
    "search_string = '+'.join(str(e) for e in search_input.lower().split())\n",
    "\n",
    "## each search page has number of results pages with a limit of 50 news article in each one.\n",
    "\n",
    "url1 = f'https://www.dailysabah.com/search?query={search_string}&qlimit=by_fifty&pgno='\n",
    "url2 = f'https://www.dailysabah.com/search?query={search_string}&qlimit=by_fifty&qsort=oldest&pgno='\n",
    "urls = [url1,url2]\n",
    "print(url1)\n",
    "print(url2)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "102\n",
      "https://www.dailysabah.com/search?query=turkish+airlines&qlimit=by_fifty&pgno=1\n",
      "https://www.dailysabah.com/search?query=turkish+airlines&qlimit=by_fifty&qsort=oldest&pgno=51\n"
     ]
    }
   ],
   "source": [
    "list_of_links = []\n",
    "\n",
    "for url in urls:\n",
    "    numbers = 1\n",
    "    try: \n",
    "    \n",
    "        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "        reg_url = url\n",
    "        req = Request(url=reg_url, headers=headers) \n",
    "        html = urlopen(req).read()  \n",
    "\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "    \n",
    "        data = soup.findAll('div',{'class':'pages_list'})\n",
    "        data2 = str(data).split('pgno=')\n",
    "        number = int(data2[-2].split('\">')[0])\n",
    "        \n",
    "        for i in range(1,number+1,1):\n",
    "            list_of_links.append(url+str(i))\n",
    "            \n",
    "    except HTTPError as http_err:\n",
    "        print(f'HTTP error occurred: {http_err}')  \n",
    "        pass\n",
    "    except Exception as err:\n",
    "        print(f'Other1 error occurred: {err}')  \n",
    "        pass\n",
    "\n",
    "    \n",
    "print(len(list_of_links))\n",
    "print(list_of_links[0])\n",
    "print(list_of_links[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "headlines = []\n",
    "for url in list_of_links:\n",
    "    try: \n",
    "    \n",
    "        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.3'}\n",
    "        reg_url = url\n",
    "        req = Request(url=reg_url, headers=headers) \n",
    "        html = urlopen(req).read()  \n",
    "\n",
    "        soup = BeautifulSoup(html, \"html.parser\")\n",
    "    \n",
    "        data = soup.findAll('ul',{'class':'items_list'})\n",
    "        data2 = str(data[0]).split('<div class=\"image_holder\">')\n",
    "\n",
    "        for i in range(1,len(data2),2):\n",
    "            link = data2[i].split('<a href=\"')[1].split('\">')[0]\n",
    "            date =  data2[i].split('<div class=\"date_text\">')[1].split('</div')[0].strip()\n",
    "            headline = data2[i].split('<h3>')[1].split('</a>')[0].split('\">')[1].strip()\n",
    "#             print(link)\n",
    "            article = getArticle(link)\n",
    "            \n",
    "            headlines.append([headline,date,article,link])\n",
    "            \n",
    "    except HTTPError as http_err:\n",
    "        print(f'HTTP error occurred: {http_err}')  \n",
    "        pass\n",
    "    except Exception as err:\n",
    "        print(f'Other1 error occurred: {err}')  \n",
    "        pass\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[\"Turkey's exports expected to bounce back this month\", 'Sep 03, 2015', \"\\n                    The slowdown in the rate of the fall of Turkey's exports in August has corroborated Turkish Exporters Assembly (TİM) President Mehmet Büyükekşi's argument that exports will start to rise in September. Exports dropped from 13 percent in July to 5 percent in August.\\n\\r\\nTurkey's exports decreased to $10.5 billion with a 4.9 percent fall in August when compared to the same month last year. This decline was around 13 percent in July. Turkey's exports dropped 8.9 percent to $95.135 billion between January 2015 and August 2015.\\n\\r\\nBüyükekşi announced export figures for this period and said the decline in exports stemmed from negative developments in foreign markets and the fall in euro-dollar parity. The fall in the parity caused a loss of $8.9 billion in the first eight months of the year, according to Büyükekşi.\\n\\r\\nWarning companies against speculative exchange rate movements, Büyükekşi said exporters should care about making a profit from their businesses rather than from speculative exchange rate fluctuations.\\n\\r\\nTİM guides exporters to new markets, such as Iran and African countries, to make up for the loss, Büyükekşi said. A total of 63 percent of companies believe that exports to Iran will improve following the nuclear deal that was reached between Iran and the P5+1 countries - the U.S., U.K., France, Russia and China plus Germany. TİM will send pioneering exploratory and commercial delegations to 33 African countries to which Turkish Airlines offers flights, including Nigeria, Botswana and Namibia in the upcoming period. Between Oct. 5 and Oct. 8, the assembly will also send a delegation to Iran, which Büyükekşi said offers great opportunities. Agricultural and industrial exports increased 5.8 percent on a kilogram basis in August, while total exports rose 11.7 percent on a kilogram basis in the same month. Turkey's exports to Jordan skyrocketed 223 percent, making the country rank first among the top 30 countries to which Turkey carried out the greatest number of exports.\\n\\nTurkish exports to neighboring countries down 31 pct\\n\\r\\nTurkish exports to neighboring countries are gradually decreasing. Although Turkey conducted close to 20 percent of its foreign trade with bordering countries in 2013, the share of exports to such countries dropped to 13.8 percent in the first seven months of 2015, a decrease of 31 percent.\\n\\r\\nNeighboring countries accounted for 9.98 percent of Turkish exports in 2005, and this figure rose to 10.34 percent and 10.83 percent in 2006 and 2007, respectively. When the global financial crisis hit in 2008, the share of exports to neighboring countries stayed at 10.83 percent. When the crisis intensified in 2009, their share of exports rose to 13.46. It soared to 14.22 percent and 14.71 percent in 2010 and 2011, respectively. Although it reached the highest level of 18.47 percent in 2012, this figure gradually decreased in the following periods. It fell to 16.32 percent and 15.52 percent in 2013 and 2014, respectively. Bordering countries share of Turkish exports further decreased to 14.7 percent between January 2014 and June 2014 and dropped by approximately one point to 13.82 percent in the same period this year.\\n\\r\\nDuring the period when Turkey began pursuing a policy of no problems with its neighbors, it also improved its commercial relations with seven out of its eight neighbors: Greece, Bulgaria, Georgia, Azerbaijan, Syria, Iraq and Iran - with the exception of Armenia. Turkey exported $85.5 billion worth of goods and services in 2005. In the following 10 years until the end of 2014, Turkish exports rose to $157.6 billion. The country aims to reach $500 billion in exports by 2023. In the same period, exports to neighboring countries surged from $8.5 billion to $24.4 billion, marking a larger increase when compared to overall exports and, by extension, the share of Turkish exports to neighboring countries gradually increased.\\n\\r\\nMeanwhile, Iran has felt relatively relieved in terms of trade with the removal of embargoes following a nuclear deal that was reached with the P5+1 in July. This has had an impact on its commercial relations with Turkey as well. Turkey suffered a decline in exports to all of its neighbors during the January-July period this year with the exception of Iran. Turkish exports to Azerbaijan, Bulgaria, Iraq, Georgia and Syria dropped by 24.1 percent, 21.9 percent, 20.8 percent, 19.77 percent and 15 percent, respectively, while exports to Iran soared by 22 percent in the same period.\\n\\r\\nAccording to data from the Turkish Exporters Assembly (TİM), although exports to Iran decreased by 9.3 percent in August, they experienced an 18.4 percent upsurge overall between January 2015 and August 2015 to $2.476 billion.\\n\\nForeign trade deficit down 41 pct in Aug\\n\\r\\nAccording to preliminary data released by the Customs and Trade Ministry, the foreign trade deficit plummeted 40.95 percent to $4.88 billion in August. A written statement from the ministry said exports declined 1.45 percent to $11.07 billion and imports fell 18.2 percent to $15.95 billion in August. The export-import coverage ratio soared from 57.6 percent in August 2014 to 69.4 percent for the same month this year. The greatest number of exports was realized in the motor land vehicles sector with $1.05 billion in August, followed by boilers, machinery and equipment, tools and parts with $951 billion and knitwear and its accessorizes with $824 billion. The greatest number of imports was in the mineral fuels sector, mineral oils and mineral preparations with $2.788 billion.\\n\", 'https://www.dailysabah.com/business/2015/09/03/turkeys-exports-expected-to-bounce-back-this-month']\n",
      "2526\n"
     ]
    }
   ],
   "source": [
    "print(headlines[-3])\n",
    "\n",
    "print(len(headlines))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "df = pd.DataFrame(headlines, columns =['text','date','article','article_link'])\n",
    "df.to_csv(f'apis_datasets/{search_input}_DailySabah.csv', encoding='utf-8', index=False)\n",
    "\n",
    "# df.head()  \n",
    "# df.describe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>text</th>\n",
       "      <th>date</th>\n",
       "      <th>article</th>\n",
       "      <th>article_link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Turkish airports host 9.5M passengers in Augus...</td>\n",
       "      <td>Sep 10, 2020</td>\n",
       "      <td>\\nThe Turkish civil aviation sector has surviv...</td>\n",
       "      <td>https://www.dailysabah.com/business/transporta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Performance tests to start on high-speed metro...</td>\n",
       "      <td>Sep 07, 2020</td>\n",
       "      <td>\\nThe production of the first subway car of th...</td>\n",
       "      <td>https://www.dailysabah.com/business/transporta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Ukrainian woman returning from Antalya takes w...</td>\n",
       "      <td>Sep 04, 2020</td>\n",
       "      <td>\\nA woman arriving back from a vacation in Tur...</td>\n",
       "      <td>https://www.dailysabah.com/world/europe/ukrain...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Turkey-Iraq flights suspended until Oct. 1</td>\n",
       "      <td>Sep 01, 2020</td>\n",
       "      <td>\\nFlights between Turkey and Iraq have been su...</td>\n",
       "      <td>https://www.dailysabah.com/business/transporta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Companies traded on Borsa Istanbul post $4.7B ...</td>\n",
       "      <td>Aug 26, 2020</td>\n",
       "      <td>\\nThe net profits of companies traded on the B...</td>\n",
       "      <td>https://www.dailysabah.com/business/finance/co...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                text          date  \\\n",
       "0  Turkish airports host 9.5M passengers in Augus...  Sep 10, 2020   \n",
       "1  Performance tests to start on high-speed metro...  Sep 07, 2020   \n",
       "2  Ukrainian woman returning from Antalya takes w...  Sep 04, 2020   \n",
       "3         Turkey-Iraq flights suspended until Oct. 1  Sep 01, 2020   \n",
       "4  Companies traded on Borsa Istanbul post $4.7B ...  Aug 26, 2020   \n",
       "\n",
       "                                             article  \\\n",
       "0  \\nThe Turkish civil aviation sector has surviv...   \n",
       "1  \\nThe production of the first subway car of th...   \n",
       "2  \\nA woman arriving back from a vacation in Tur...   \n",
       "3  \\nFlights between Turkey and Iraq have been su...   \n",
       "4  \\nThe net profits of companies traded on the B...   \n",
       "\n",
       "                                        article_link  \n",
       "0  https://www.dailysabah.com/business/transporta...  \n",
       "1  https://www.dailysabah.com/business/transporta...  \n",
       "2  https://www.dailysabah.com/world/europe/ukrain...  \n",
       "3  https://www.dailysabah.com/business/transporta...  \n",
       "4  https://www.dailysabah.com/business/finance/co...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}