sfbrigade · CraigMariani · Dec 3, 2020 · Dec 17, 2020
diff --git a/__pycache__/secrets.cpython-38.pyc b/__pycache__/secrets.cpython-38.pyc
diff --git a/s3_uploader.py b/s3_uploader.py
@@ -0,0 +1,14 @@
+import boto3
+import os
+
+
+if __name__ == '__main__':
+    client = boto3.client('s3')
+    pathlib.Path().absolute()
+    # os.chdir('data')
+    for file in os.listdir():
+        if '.csv' in file:
+            print(file)
+            upload_file_bucket = 'twitter-data-01'
+            upload_file_key = 'csv_twitter/' + str(file)
+            client.upload_file(file, upload_file_bucket, upload_file_key)
diff --git a/secrets.py b/secrets.py
@@ -0,0 +1,5 @@
+api_key = ''
+api_secret = ''
+
+con_key = ''
+con_secret = ''
diff --git a/twitter_extract.py b/twitter_extract.py
@@ -0,0 +1,71 @@
+import tweepy
+import preprocessing as p 
+import csv
+import pandas as pd 
+import pprint
+import json
+from secrets import api_key, api_secret, con_key, con_secret
+
+class TwitterExtract:
+
+    def __init__(self):
+        # auth = tweepy.OAuthHandler(con_key, con_secret)
+        # auth.set_access_token(api_key, api_secret)
+
+        auth = tweepy.OAuthHandler(api_key, api_secret)
+        auth.set_access_token(con_key, con_secret)
+
+        api = tweepy.API(auth, wait_on_rate_limit=True)
+
+        self.api = api
+
+    def create_csv(self, tweet_data):
+        # tweet_csv = open('tweet_results.csv', 'a')
+        df = pd.DataFrame(tweet_data)
+        df.to_csv('twitter_grant_data.csv')
+
+
+
+    def get_tweets(self):
+        api = self.api
+
+        # search_word = '#sf'
+        search_words = ['#grant' , '#bayarea', '#sf ']
+
+        # new_search = search_word + " -filter:retweets"
+
+        data = {
+            'search_word': [],
+            'date_created': [],
+            'tweet_id' : [],
+            'text' : [],
+            'source' : []
+        }
+
+        for search_word in search_words:
+            for item in tweepy.Cursor(api.search, q=search_word, lang='en').items(10):
+
+                json_item = item._json
+
+                data['search_word'].append(search_word)
+                data['date_created'].append(json_item['created_at'])
+                data['tweet_id'].append(json_item['id'])
+                data['text'].append(json_item['text'].encode('utf-8'))
+                data['source'].append(json_item['source'])
+                # tweet_data = [search_word, json_item['created_at'], json_item['id'], json_item['text'].encode('utf-8'), json_item['source']]
+
+                # print('date')
+                # print(json_item['created_at'])
+                # print('id')
+                # print(json_item['id'])
+                # print('text')
+                # print(json_item['text'])
+                # print('source')
+                # print(json_item['source'])
+                # print(item.entities.get('hashtags'))
+        return data
+
+t = TwitterExtract()
+tweet_data = t.get_tweets()
+t.create_csv(tweet_data)
+# pprint.pprint(tweet_data[0])
diff --git a/twitter_grant_data.csv b/twitter_grant_data.csv
@@ -0,0 +1,31 @@
+,search_word,date_created,tweet_id,text,source
+0,#grant,Thu Dec 17 20:44:50 +0000 2020,1339672928401514496,"b""RT @EM_TCCM: Applications for Fall 2021 are open until February 15th! Don't miss the opportunity to get a full grant to study Theoretical C\xe2\x80\xa6""","<a href=""http://twitter.com/download/android"" rel=""nofollow"">Twitter for Android</a>"
+1,#grant,Thu Dec 17 20:30:12 +0000 2020,1339669245479809034,b'SUNLIGHT receives \xe2\x82\xac50m grant to develop \xe2\x80\x98low-environmental footprint\xe2\x80\x99 cells\n\nGet more insights from FutureBridge at\xe2\x80\xa6 https://t.co/P8R015uulQ',"<a href=""https://www.hootsuite.com"" rel=""nofollow"">Hootsuite Inc.</a>"
+2,#grant,Thu Dec 17 20:24:44 +0000 2020,1339667868162650128,"b'RT @teamSVBS: The last 2020 edition of our #newsletter is out now, featuring an important update on #grant funding and helpful advice on ho\xe2\x80\xa6'","<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+3,#grant,Thu Dec 17 19:48:56 +0000 2020,1339658860190175236,b'RT @EUI_History: What could you do as a PhD researcher @EUI_History? Have a look at our research profile https://t.co/8zz6WwBYRW and apply\xe2\x80\xa6',"<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"
+4,#grant,Thu Dec 17 19:44:35 +0000 2020,1339657763945918467,"b'RT @ChiTrust: Earlier this year, the Trust provided $425,000 in general operating funding to eight Building Pathways to Stability #grant re\xe2\x80\xa6'","<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"
+5,#grant,Thu Dec 17 18:54:12 +0000 2020,1339645084288094208,b'Just found out this week that I got a small #grant from Fredrik &amp; Ingrid Thurings Stiftelse. My first ever grant as\xe2\x80\xa6 https://t.co/og0oOWwT2a',"<a href=""http://twitter.com/#!/download/ipad"" rel=""nofollow"">Twitter for iPad</a>"
+6,#grant,Thu Dec 17 18:23:34 +0000 2020,1339637374876332035,b'RT @JGrapsa: This is your opportunity to apply for an educational #grant for #Echokurs - one of the best clinical-pathology-imaging meeting\xe2\x80\xa6',"<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+7,#grant,Thu Dec 17 18:16:32 +0000 2020,1339635605253308419,"b""RT @GBSLEP: Does your SME operate in the @GBSLEP area's Visitor Economy? Transportation, food &amp; drink, accommodation, sports and fitness, e\xe2\x80\xa6""","<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"
+8,#grant,Thu Dec 17 18:16:08 +0000 2020,1339635507773497346,"b""RT @EM_TCCM: Applications for Fall 2021 are open until February 15th! Don't miss the opportunity to get a full grant to study Theoretical C\xe2\x80\xa6""","<a href=""http://twitter.com/download/android"" rel=""nofollow"">Twitter for Android</a>"
+9,#grant,Thu Dec 17 18:15:07 +0000 2020,1339635250972995585,"b'Earlier this year, the Trust provided $425,000 in general operating funding to eight Building Pathways to Stability\xe2\x80\xa6 https://t.co/mQwp7GjIf7'","<a href=""https://sproutsocial.com"" rel=""nofollow"">Sprout Social</a>"
+10,#bayarea,Thu Dec 17 20:54:33 +0000 2020,1339675374863540224,"b'RT @KPIXtv: The travel #quarantine order strongly discourages non-essential travel of any kind and within any distance, including from one\xe2\x80\xa6'","<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+11,#bayarea,Thu Dec 17 20:54:25 +0000 2020,1339675338398343169,"b""RT @KPIXtv: #SanFrancisco's #quarantine order:\n- Anyone who comes to SF MUST #quarantine for 10 days if they spent any time outside the 9 #\xe2\x80\xa6""","<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+12,#bayarea,Thu Dec 17 20:53:57 +0000 2020,1339675221251379202,b'featuring:\n\n@surfdogteddy\nRusty The Surfing MinPin\nRothstein\nTristan\nCarson (Team Kihei)\n\n#surfingdog\xe2\x80\xa6 https://t.co/VGtSpam6iK',"<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"
+13,#bayarea,Thu Dec 17 20:51:44 +0000 2020,1339674665514459136,"b""RT @KPIXtv: SAN FRANCISCO QUARANTINE ORDER\nStarting at 12:01AM TOMORROW, #SanFrancisco's mandatory #quarantine for those traveling to the c\xe2\x80\xa6""","<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"
+14,#bayarea,Thu Dec 17 20:50:40 +0000 2020,1339674396546408448,b'RT @KPIXtv: #BREAKING:\n#SanFrancisco Mayor @LondonBreed &amp; @SF_DPH Dr. Colfax announce a public health order that places a mandatory #quaran\xe2\x80\xa6',"<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+15,#bayarea,Thu Dec 17 20:50:15 +0000 2020,1339674291114110977,"b""RT @KPIXtv: SAN FRANCISCO QUARANTINE ORDER\nStarting at 12:01AM TOMORROW, #SanFrancisco's mandatory #quarantine for those traveling to the c\xe2\x80\xa6""","<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"
+16,#bayarea,Thu Dec 17 20:47:26 +0000 2020,1339673582998343683,"b""RT @KPIXtv: SAN FRANCISCO QUARANTINE ORDER\nStarting at 12:01AM TOMORROW, #SanFrancisco's mandatory #quarantine for those traveling to the c\xe2\x80\xa6""","<a href=""http://www.socialnewsdesk.com"" rel=""nofollow"">SocialNewsDesk</a>"
+17,#bayarea,Thu Dec 17 20:47:26 +0000 2020,1339673582629228547,"b""RT @KPIXtv: SAN FRANCISCO QUARANTINE ORDER\nStarting at 12:01AM TOMORROW, #SanFrancisco's mandatory #quarantine for those traveling to the c\xe2\x80\xa6""","<a href=""http://www.socialnewsdesk.com"" rel=""nofollow"">SocialNewsDesk</a>"
+18,#bayarea,Thu Dec 17 20:47:26 +0000 2020,1339673582507573248,"b""RT @KPIXtv: SAN FRANCISCO QUARANTINE ORDER\nStarting at 12:01AM TOMORROW, #SanFrancisco's mandatory #quarantine for those traveling to the c\xe2\x80\xa6""","<a href=""http://www.socialnewsdesk.com"" rel=""nofollow"">SocialNewsDesk</a>"
+19,#bayarea,Thu Dec 17 20:47:26 +0000 2020,1339673582503378945,"b""RT @KPIXtv: SAN FRANCISCO QUARANTINE ORDER\nStarting at 12:01AM TOMORROW, #SanFrancisco's mandatory #quarantine for those traveling to the c\xe2\x80\xa6""","<a href=""http://www.socialnewsdesk.com"" rel=""nofollow"">SocialNewsDesk</a>"
+20,#sf ,Thu Dec 17 20:58:23 +0000 2020,1339676338148425729,"b'Thermo Fisher Scientific is looking for teammates like you. See our latest #GeneralScience job openings, including\xe2\x80\xa6 https://t.co/dyUfL2itLU'","<a href=""https://www.careerarc.com"" rel=""nofollow"">CareerArc 2.0</a>"
+21,#sf ,Thu Dec 17 20:58:18 +0000 2020,1339676315113451522,b'RT @shattawalegh: Are we all ready ? \xe2\x9c\x8c\xef\xb8\x8f\xf0\x9f\x98\x8f\n\nDP change for the #GOGALBUM ? \xf0\x9f\x98\xac\n\nG I F T.   O F .  G O D .\xf0\x9f\x98\x87\n\n#SM4LYF \n#SM\n#SF \n#DGC\n#SHAXI\n#SP\n#G\xe2\x80\xa6',"<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+22,#sf ,Thu Dec 17 20:57:54 +0000 2020,1339676216823934976,"b'Can you recommend anyone for this job in #SF, CA? https://t.co/6Zp46Irxd0 #scientist'","<a href=""https://www.careerarc.com"" rel=""nofollow"">CareerArc 2.0</a>"
+23,#sf ,Thu Dec 17 20:57:09 +0000 2020,1339676027933642760,b'RT @FBISanFrancisco: #Breaking: FBI Special Agent in Charge Craig Fair announces reward in #SF church arson. https://t.co/nJ7Nv89U0M https:\xe2\x80\xa6',"<a href=""http://twitter.com/download/android"" rel=""nofollow"">Twitter for Android</a>"
+24,#sf ,Thu Dec 17 20:55:50 +0000 2020,1339675696344416256,b'RT @FBISanFrancisco: #Breaking: FBI Special Agent in Charge Craig Fair announces reward in #SF church arson. https://t.co/nJ7Nv89U0M https:\xe2\x80\xa6',"<a href=""http://twitter.com/download/android"" rel=""nofollow"">Twitter for Android</a>"
+25,#sf ,Thu Dec 17 20:54:42 +0000 2020,1339675411681214464,b'RT @FBISanFrancisco: #Breaking: FBI Special Agent in Charge Craig Fair announces reward in #SF church arson. https://t.co/nJ7Nv89U0M https:\xe2\x80\xa6',"<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+26,#sf ,Thu Dec 17 20:54:25 +0000 2020,1339675338398343169,"b""RT @KPIXtv: #SanFrancisco's #quarantine order:\n- Anyone who comes to SF MUST #quarantine for 10 days if they spent any time outside the 9 #\xe2\x80\xa6""","<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+27,#sf ,Thu Dec 17 20:53:58 +0000 2020,1339675226150301696,b'RT @FBISanFrancisco: #Breaking: FBI Special Agent in Charge Craig Fair announces reward in #SF church arson. https://t.co/nJ7Nv89U0M https:\xe2\x80\xa6',"<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+28,#sf ,Thu Dec 17 20:53:28 +0000 2020,1339675100078075906,b'RT @FBISanFrancisco: #Breaking: FBI Special Agent in Charge Craig Fair announces reward in #SF church arson. https://t.co/nJ7Nv89U0M https:\xe2\x80\xa6',"<a href=""http://twitter.com/download/iphone"" rel=""nofollow"">Twitter for iPhone</a>"
+29,#sf ,Thu Dec 17 20:51:18 +0000 2020,1339674555753742336,b'RT @FBISanFrancisco: #Breaking: FBI Special Agent in Charge Craig Fair announces reward in #SF church arson. https://t.co/nJ7Nv89U0M https:\xe2\x80\xa6',"<a href=""https://mobile.twitter.com"" rel=""nofollow"">Twitter Web App</a>"