script for downloading stamps

Author: doruchan

beehive_scraper.py

@@ -1,7 +1,7 @@
 import json
 import bs4
 from bs4 import BeautifulSoup
-import urllib
+import requests
 
 def parse_single_release(s):
     release = {}
@@ -16,11 +16,11 @@ def parse_single_release(s):
 if __name__ == '__main__':
     for i in range(277): 
         base = 'https://www.beehive.govt.nz'
-        data = urllib.request.urlopen(base + '/releases?page={}'.format(i))
+        data = requests.get(base + '/releases?page={}'.format(i))
         soup = BeautifulSoup(data)
         paths = [s['href'] for s in soup.find_all('a') if '/release/' in s['href']]
         paths = [base+p for i, p in enumerate(paths) if i%2==0]
-        releases = [BeautifulSoup(urllib.request.urlopen(p)) for p in paths]
+        releases = [BeautifulSoup(requests.get(p)) for p in paths]
         parsed_releases = [parse_single_release(s) for s in releases]
         for rel in parsed_releases:
             print('writing {}'.format(rel['title']))

download_stamps.py

@@ -0,0 +1,37 @@
+import requests
+import os
+import shutil
+import sys
+
+def copy_image(image_url, download_path):
+	r = requests.get(image_url, stream=True)
+	if r.status_code == 200:
+	    with open(download_path, 'wb') as f:
+	        r.raw.decode_content = True
+	        shutil.copyfileobj(r.raw, f)   
+
+def get_images(url):
+	if url is None:
+		return []
+	r = requests.get(url)
+	if r.status_code == 200:
+		return [result['large_thumbnail_url'] for result in r.json()["search"]["results"]]
+	return []
+
+api_key = "eJesEUUomq_zGoW9nBAW"
+url = "http://api.digitalnz.org/v3/records.json?api_key={api_key}&text=stamp&and[category][]=Images&per_page=200".format(api_key=api_key)
+page = sys.argv[1]
+
+url += "&page=%s" % page
+
+folder = "image_collection"
+if not os.path.exists(folder):
+	os.mkdir(folder)
+
+for index, thumb in enumerate(get_images(url)):
+	print "downloading:", thumb
+	try:
+		copy_image(thumb, os.path.join(folder, "%s_%s.jpg" % (page, index)))
+	except Exception as error:
+		print "Error downloading image %s:" % thumb, error
+		continue