basic news scraping and summary working for ap and reuters

Author: Sean McIlroy

scrape_news/scrape_ap.py

@@ -0,0 +1,25 @@
+from bs4 import BeautifulSoup
+import requests
+from utils import get_summary
+
+print('==========================================================')
+
+site = 'https://www.apnews.com'
+
+source = requests.get(site).text
+
+soup = BeautifulSoup(source, 'lxml')
+
+first_story = soup.find('a', class_="headline")
+
+print(first_story.text)
+print(get_summary(site + first_story['href']))
+print('==========================================================')
+
+second_story_container = soup.find('div', class_="RelatedStory")
+second_story_link = second_story_container.a
+second_story_title = second_story_container.find('div', class_="headline")
+
+print(second_story_title.text)
+print(get_summary(site + second_story_link['href']))
+print('==========================================================')

scrape_news/scrape_reuters.py

@@ -0,0 +1,26 @@
+from bs4 import BeautifulSoup
+import requests
+from utils import get_summary
+
+print('==========================================================')
+
+site = 'https://www.reuters.com'
+
+source = requests.get(site).text
+
+soup = BeautifulSoup(source, 'lxml')
+
+first_story_container = soup.find('h2', class_="story-title")
+first_story = first_story_container.a
+
+print(first_story.text)
+print(get_summary(site + first_story['href']))
+print('==========================================================')
+
+second_story_container = soup.find('div', class_="news-headline-list")
+second_story_link = second_story_container.find('a')
+second_story_title = second_story_container.find('h3', class_="story-title")
+
+print(second_story_title.text.strip())
+print(get_summary(site + second_story_link['href']))
+print('==========================================================')

scrape_news/utils.py

@@ -0,0 +1,9 @@
+from  newspaper import Article
+
+def get_summary(url):
+    article = Article(url)
+    article.download()
+    article.parse()
+    article.nlp()
+    return article.summary
+