main.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import requests\n",
    "\n",
    "import os\n",
    "import collections"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_excel(\"data/buzzsumo_fakenews.xlsx\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Source\"].value_counts()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df[\"Source\"] == \"Lapada Lapada – Informações em Tempo Real\", \"Source\"] = \"Lapada Lapada\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"Source\"].value_counts()[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "agregando todas notícias do UOL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df[\"Source\"].str.contains(\"UOL\"), \"Source\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df[\"Source\"].str.contains(\"Uol\"), \"Source\"].value_counts()\n",
    "df.loc[df[\"Source\"].str.contains(\"UOL\"), \"Source\"].value_counts()\n",
    "df.loc[df[\"Source\"].str.contains(\"uol\"), \"Source\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.loc[df[\"Source\"].str.contains(\"Uol\"), \"Source\"] = \"UOL\"\n",
    "df.loc[df[\"Source\"].str.contains(\"UOL\"), \"Source\"] = \"UOL\"\n",
    "df.loc[df[\"Source\"].str.contains(\"uol\"), \"Source\"] = \"UOL\""
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "selecionando top 10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_list = list(df[\"Source\"].value_counts()[:10].index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "headers = {'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.5 Safari/605.1.15'}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "news_error = []\n",
    "for news in new_news_list:\n",
    "    news_urls = df[df[\"Source\"] == news][\"URL\"]\n",
    "\n",
    "    try:\n",
    "        os.mkdir(\"data/\"+news)\n",
    "    except FileExistsError:\n",
    "        print(f\"Pasta data/{news} já criada.\")\n",
    "    \n",
    "    for url in news_urls:\n",
    "        response = requests.get(url, headers=headers)\n",
    "        if response.ok:\n",
    "            file_name_list = url.rsplit(\"/\", 2)\n",
    "            if file_name_list[2]:\n",
    "                file_name = file_name_list[2]\n",
    "            else:\n",
    "                file_name = file_name_list[1]\n",
    "            \n",
    "            with open(f\"data/{news}/{file_name}\", \"w\") as file:\n",
    "                file.write(response.text)\n",
    "        else:\n",
    "            news_error.append([url, response.status_code])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_news_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len([error[0] for error in news_error if \"documento\" in error[0]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "collections.Counter([error[1] for error in news_error])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_news_df = df.loc[(df[\"Source\"] != \"Overidico\") & (df[\"Source\"] != \"Poliarquia\"), \"Source\"].value_counts()[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_news_list = list(new_news_df.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_news_list"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('venv': venv)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "c6b7bb539a62f04173d918f783f15ea703dc61b861d6d9dd40d0ba3f6066be45"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}