badlinks filtering

Author: teamoo

musiccrawler/config/badlinks.cfg

@@ -0,0 +1 @@
+http://www.dwmp3.com/wp-content;http://www.dwmp3.com/xmlrpc;http://www.dwmp3.com/wp-content;http://www.dwmp3.com/wp-includes;http://www.dwmp3.com/feed;http://www.dwmp3.com/dmca;http://www.dwmp3.com/2013;http://www.dwmp3.com/tag

musiccrawler/pipelines.py

@@ -84,10 +84,15 @@ def process_item(self, item, spider):
         return item
 
 class BadFilesPipeline(object):
+    def __init__(self):
+        self.badlinks = open(pkg_resources.resource_string('musiccrawler.config', "badlinks.cfg")).read().split(";")
+
     def process_item(self, item, spider):
         if str(item['url']).endswith(".jpg") or str(item['url']).endswith(".png") or str(item['url']).endswith(".gif"):
             log.msg(("Bad Item:" + str(item)), level=log.DEBUG)
             raise DropItem("Bad Link-URL found: %s" % item['url'])
+        elif str(item['url']) in self.badlinks:
+            raise DropItem("Bad Link-URL found: %s" % item['url'])
         else:
             return item
 

setup.py

@@ -6,7 +6,7 @@
     name='musiccrawler',
     version='1.0',
     packages=find_packages(),
-    package_data={'musiccrawler.config': ['*.json']},
+    package_data={'musiccrawler.config': ['*.json','*.cfg']},
     entry_points={'scrapy': ['settings = musiccrawler.settings']},
     zip_safe=False
 )