Merge pull request NodyHub#16 from codders/feat/crawl-and-filter-dates

Improve efficiency of IDMaintainer, extract 'from' dates

Author: codders

flathunter/abstract_crawler.py

@@ -16,4 +16,8 @@ def crawl(self, url, max_pages=None):
         return []
 
     def get_name(self):
-        return type(self).__name__
+        return type(self).__name__
+
+    def get_expose_details(self, expose):
+        # Implement in subclass - extract additional data by processing the expose URL
+        return expose

flathunter/crawl_ebaykleinanzeigen.py

@@ -1,13 +1,28 @@
 import logging
 import requests
 import re
+import datetime
 from bs4 import BeautifulSoup
 from flathunter.abstract_crawler import Crawler
 
 class CrawlEbayKleinanzeigen(Crawler):
     __log__ = logging.getLogger(__name__)
     USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'
     URL_PATTERN = re.compile(r'https://www\.ebay-kleinanzeigen\.de')
+    MONTHS = {
+      "Januar": "01",
+      "Februar": "02",
+      "März": "03",
+      "April": "04",
+      "Mai": "05",
+      "Juni": "06",
+      "Juli": "07",
+      "August": "08",
+      "September": "09",
+      "Oktober": "10",
+      "November": "11",
+      "Dezember": "12"
+    }
 
     def __init__(self):
         logging.getLogger("requests").setLevel(logging.WARNING)
@@ -29,6 +44,17 @@ def get_page(self, search_url):
             self.__log__.error("Got response (%i): %s" % (resp.status_code, resp.content))
         return BeautifulSoup(resp.content, 'html.parser')
 
+    def get_expose_details(self, expose):
+        soup = self.get_page(expose['url'])
+        for detail in soup.find_all('li', { "class": "addetailslist--detail" }):
+            if re.match(r'Verfügbar ab', detail.text):
+                date_string = re.match(r'(\w+) (\d{4})', detail.text)
+                if date_string is not None:
+                    expose['from'] = "01." + self.MONTHS[date_string[1]] + "." + date_string[2]
+        if 'from' not in expose:
+            expose['from'] = datetime.datetime.now().strftime('%02d.%02m.%Y')
+        return expose
+
     def extract_data(self, soup):
         entries = list()
         soup = soup.find(id="srchrslt-adtable")
@@ -57,14 +83,14 @@ def extract_data(self, soup):
             address = address.replace('\n', ' ').replace('\r', '')
             address = " ".join(address.split())
             try:
-                self.__log__.debug(tags[0].text)
-                rooms = tags[0].text
+                self.__log__.debug(tags[1].text)
+                rooms = re.match(r'(\d+)', tags[1].text)[1]
             except IndexError:
                 self.__log__.debug("Keine Zimmeranzahl gegeben")
                 rooms = "Nicht gegeben"
             try:
-                self.__log__.debug(tags[1].text)
-                size = tags[1].text
+                self.__log__.debug(tags[0].text)
+                size = tags[0].text
             except IndexError:
                 size = "Nicht gegeben"
                 self.__log__.debug("Quadratmeter nicht angegeben")

flathunter/crawl_immobilienscout.py

@@ -1,7 +1,9 @@
 import logging
 import requests
 import re
+import datetime
 from bs4 import BeautifulSoup
+
 from flathunter.abstract_crawler import Crawler
 
 class CrawlImmobilienscout(Crawler):
@@ -47,12 +49,24 @@ def get_results(self, search_url, max_pages=None):
             entries.extend(cur_entry)
         return entries
 
-    def get_page(self, search_url, page_no):
-        resp = requests.get(search_url.format(page_no))
+    def get_soup_from_url(self, url):
+        resp = requests.get(url)
         if resp.status_code != 200:
             self.__log__.error("Got response (%i): %s" % (resp.status_code, resp.content))
         return BeautifulSoup(resp.content, 'html.parser')
 
+    def get_page(self, search_url, page_no):
+        return self.get_soup_from_url(search_url.format(page_no))
+
+    def get_expose_details(self, expose):
+        soup = self.get_soup_from_url(expose['url'])
+        date = soup.find('dd', { "class": "is24qa-bezugsfrei-ab" })
+        expose['from'] = datetime.datetime.now().strftime("%2d.%2d.%Y")
+        if date is not None:
+            if not re.match(r'.*sofort.*', date.text):
+                expose['from'] = date.text.strip()
+        return expose
+
     def extract_data(self, soup):
         entries = list()
 
@@ -98,7 +112,7 @@ def extract_data(self, soup):
                     'title': title_el.text.strip().replace('NEU', ''),
                     'price': attr_els[0].text.strip().split(' ')[0].strip(),
                     'size': attr_els[1].text.strip().split(' ')[0].strip() + " qm",
-                    'rooms': attr_els[2].text.strip().split(' ')[0].strip() + " Zi.",
+                    'rooms': attr_els[2].text.strip().split(' ')[0].strip(),
                     'address': address,
                     'crawler': self.get_name()
                 }

flathunter/crawl_immowelt.py

@@ -1,7 +1,9 @@
 import logging
 import requests
 import re
+import datetime
 from bs4 import BeautifulSoup
+
 from flathunter.abstract_crawler import Crawler
 
 class CrawlImmowelt(Crawler):
@@ -28,6 +30,28 @@ def get_page(self, search_url):
             self.__log__.error("Got response (%i): %s" % (resp.status_code, resp.content))
         return BeautifulSoup(resp.content, 'html.parser')
 
+    def get_expose_details(self, expose):
+        soup = self.get_page(expose['url'])
+        immo_div = soup.find("div", { "id": "divImmobilie" })
+        if immo_div is not None:
+            details = immo_div.find_all("div", { "class": "clear" })
+            for detail in details:
+                if detail.find("div", { "class": "iw_left" }) is None:
+                    continue
+                if detail.find("div", { "class": "iw_left" }).text.strip() == 'Die Wohnung':
+                    description_element = detail.find("div", { "class": "iw_right" })
+                    if description_element is None or description_element.find("p") is None:
+                        continue
+                    description = description_element.find("p").text
+                    if re.match(r'.*sofort.*', description, re.MULTILINE|re.DOTALL|re.IGNORECASE):
+                        expose['from'] = datetime.datetime.now().strftime("%2d.%2d.%Y")
+                    date_string = re.match(r'.*(\d{2}.\d{2}.\d{4}).*', description, re.MULTILINE|re.DOTALL)
+                    if date_string is not None:
+                        expose['from'] = date_string[1]
+            if 'from' not in expose:
+                expose['from'] = datetime.datetime.now().strftime("%2d.%2d.%Y")
+        return expose
+
     def extract_data(self, soup):
         entries = list()
         soup = soup.find(id="listItemWrapperFixed")

flathunter/crawl_wggesucht.py

@@ -59,20 +59,26 @@ def extract_data(self, soup):
             numbers_row = row.find("div", { "class": "middle" })
             price = numbers_row.find("div", { "class": "col-xs-3" }).text.strip()
             rooms = re.findall(r'\d Zimmer', details_array[0])[0][:1]
-            date = re.findall(r'\d{2}.\d{2}.\d{4}', numbers_row.find("div", { "class": "text-center" }).text)[0]
+            dates = re.findall(r'\d{2}.\d{2}.\d{4}', numbers_row.find("div", { "class": "text-center" }).text)
             size = re.findall(r'\d{2,4}\sm²', numbers_row.find("div", { "class": "text-right" }).text)[0]
 
             details = {
                 'id': int(url.split('.')[-2]),
                 'image': image,
                 'url': url,
-                'title': "%s ab dem %s" % (title, date),
+                'title': "%s ab dem %s" % (title, dates[0]),
                 'price': price,
                 'size': size,
-                'rooms': rooms + " Zi.",
+                'rooms': rooms,
                 'address': url,
                 'crawler': self.get_name()
             }
+            if len(dates) == 2:
+                details['from'] = dates[0]
+                details['to'] = dates[1]
+            elif len(dates) == 1:
+                details['from'] = dates[0]
+
             entries.append(details)
 
         self.__log__.debug('extracted: ' + str(entries))

flathunter/default_processors.py

@@ -27,6 +27,17 @@ def process_expose(self, expose):
                     break
         return expose
 
+class CrawlExposeDetails(Processor):
+
+    def __init__(self, config):
+        self.config = config
+
+    def process_expose(self, expose):
+        for searcher in self.config.searchers():
+            if re.search(searcher.URL_PATTERN, expose['url']):
+                expose = searcher.get_expose_details(expose)
+        return expose
+
 class LambdaProcessor(Processor):
 
     def __init__(self, config, func):

flathunter/googlecloud_idmaintainer.py

@@ -23,22 +23,27 @@ def mark_processed(self, expose_id):
         self.__log__.debug('mark_processed(' + str(expose_id) + ')')
         self.db.collection(u'processed').document(str(expose_id)).set({ u'id': expose_id })
 
+    def is_processed(self, expose_id):
+        self.__log__.debug('is_processed(' + str(expose_id) + ')')
+        doc = self.db.collection(u'processed').document(str(expose_id))
+        return doc.get().exists
+
     def save_expose(self, expose):
         record = expose.copy()
         record.update({ 'created_at': datetime.datetime.now(), 'created_sort': (0 - datetime.datetime.now().timestamp()) })
         self.db.collection(u'exposes').document(str(expose[u'id'])).set(record)
 
     def get_exposes_since(self, min_datetime):
         res = []
-        for doc in self.db.collection(u'exposes').order_by('created_sort').stream():
+        for doc in self.db.collection(u'exposes').order_by('created_sort').limit(100).stream():
             if doc.to_dict()[u'created_at'] < min_datetime:
                 break
             res.append(doc.to_dict())
         return res
 
     def get_recent_exposes(self, count, filter=None):
         res = []
-        for doc in self.db.collection(u'exposes').order_by('created_sort').stream():
+        for doc in self.db.collection(u'exposes').order_by('created_sort').limit(100).stream():
             expose = doc.to_dict()
             if filter is None or filter.is_interesting_expose(expose):
                 res.append(expose)
@@ -66,15 +71,6 @@ def get_user_filters(self):
                 res.append((int(doc.id), settings['filters']))
         return res
 
-    def get(self):
-        res = []
-        for doc in self.db.collection(u'processed').stream():
-            res.append(doc.to_dict()[u'id'])
-
-        self.__log__.info('already processed: ' + str(len(res)))
-        self.__log__.debug(str(res))
-        return res
-
     def get_last_run_time(self):
         for doc in self.db.collection(u'executions').order_by(u'timestamp', direction=firestore.Query.DESCENDING).limit(1).stream():
             return doc.to_dict()[u'timestamp']

flathunter/idmaintainer.py

@@ -26,12 +26,10 @@ class AlreadySeenFilter:
 
     def __init__(self, id_watch):
         self.id_watch = id_watch
-        self.processed = self.id_watch.get()
 
     def is_interesting(self, expose):
-        if expose['id'] not in self.processed:
+        if not self.id_watch.is_processed(expose['id']):
             self.id_watch.mark_processed(expose['id'])
-            self.processed.append(expose['id'])
             return True
         return False
 
@@ -59,6 +57,13 @@ def get_connection(self):
                 raise e
         return connection
 
+    def is_processed(self, expose_id):
+        self.__log__.debug('is_processed(' + str(expose_id) + ')')
+        cur = self.get_connection().cursor()
+        cur.execute('SELECT id FROM processed WHERE id = ?', (expose_id,))
+        row = cur.fetchone()
+        return (row is not None)
+
     def mark_processed(self, expose_id):
         self.__log__.debug('mark_processed(' + str(expose_id) + ')')
         cur = self.get_connection().cursor()
@@ -111,20 +116,6 @@ def get_user_filters(self):
             res.append((row[0], json.loads(row[1])['filters']))
         return res
 
-    def get(self):
-        res = []
-        cur = self.get_connection().cursor()
-        cur.execute("SELECT * FROM processed ORDER BY 1")
-        while True:
-            row = cur.fetchone()
-            if row == None:
-                break
-            res.append(row[0])
-
-        self.__log__.info('already processed: ' + str(len(res)))
-        self.__log__.debug(str(res))
-        return res
-
     def get_last_run_time(self):
         cur = self.get_connection().cursor()
         cur.execute("SELECT * FROM executions ORDER BY timestamp DESC LIMIT 1")

flathunter/processor.py

@@ -5,6 +5,7 @@
 from flathunter.default_processors import AddressResolver
 from flathunter.default_processors import Filter
 from flathunter.default_processors import LambdaProcessor
+from flathunter.default_processors import CrawlExposeDetails
 from flathunter.sender_telegram import SenderTelegram
 from flathunter.gmaps_duration_processor import GMapsDurationProcessor
 from flathunter.idmaintainer import SaveAllExposesProcessor
@@ -30,6 +31,10 @@ def calculate_durations(self):
             self.processors.append(GMapsDurationProcessor(self.config))
         return self
 
+    def crawl_expose_details(self):
+        self.processors.append(CrawlExposeDetails(self.config))
+        return self
+
     def map(self, func):
         self.processors.append(LambdaProcessor(self.config, func))
         return self
@@ -55,4 +60,4 @@ def process(self, exposes):
 
     @staticmethod
     def builder(config):
-        return ProcessorChainBuilder(config)
+        return ProcessorChainBuilder(config)

flathunter/web/templates/exposes.html

@@ -1,7 +1,7 @@
 <div class="exposes">
   {% for expose in exposes %}
     <div class="expose">
-      <p>{{ expose['price'] }}, {{expose['rooms']}} rooms, {{expose['size']}}</p>
+      <p>{{ expose['price'] }}, {{expose['rooms']}} rooms, {{expose['size']}} from {{expose['from']}}</p>
       <a href="{{ expose['url'] }}" target="_blank">
         {% if expose['image'] %}
           <img src="{{ expose['image'] }}">
@@ -12,4 +12,4 @@
       <h3><a href="{{ expose['url'] }}" target="_blank">{{ expose['title'] }}</a></h3>
     </div>
   {% endfor %}
-</div>
+</div>

flathunter/web_hunter.py

@@ -14,8 +14,9 @@ def hunt_flats(self):
                        .build()
 
         processor_chain = ProcessorChain.builder(self.config) \
-                                        .save_all_exposes(self.id_watch) \
                                         .apply_filter(filter) \
+                                        .crawl_expose_details() \
+                                        .save_all_exposes(self.id_watch) \
                                         .resolve_addresses() \
                                         .calculate_durations() \
                                         .build()
@@ -44,4 +45,4 @@ def set_filters_for_user(self, user_id, filters):
         self.id_watch.set_filters_for_user(user_id, filters)
 
     def get_filters_for_user(self, user_id):
-        return self.id_watch.get_filters_for_user(user_id)
+        return self.id_watch.get_filters_for_user(user_id)

test/test_crawl_ebaykleinanzeigen.py

@@ -1,21 +1,31 @@
-import unittest
+import pytest
 from flathunter.crawl_ebaykleinanzeigen import CrawlEbayKleinanzeigen
 
-class EbayKleinanzeigenCrawlerTest(unittest.TestCase):
+TEST_URL = 'https://www.ebay-kleinanzeigen.de/s-wohnung-mieten/berlin/preis:1000:1500/c203l3331+wohnung_mieten.qm_d:70,+wohnung_mieten.zimmer_d:2'
 
-    TEST_URL = 'https://www.ebay-kleinanzeigen.de/s-wohnung-mieten/berlin/preis:1000:1500/c203l3331+wohnung_mieten.qm_d:70,+wohnung_mieten.zimmer_d:2'
+@pytest.fixture
+def crawler():
+    return CrawlEbayKleinanzeigen()
 
-    def setUp(self):
-        self.crawler = CrawlEbayKleinanzeigen()
-
-    def test(self):
-        soup = self.crawler.get_page(self.TEST_URL)
-        self.assertIsNotNone(soup, "Should get a soup from the URL")
-        entries = self.crawler.extract_data(soup)
-        self.assertIsNotNone(entries, "Should parse entries from search URL")
-        self.assertTrue(len(entries) > 0, "Should have at least one entry")
-        self.assertTrue(entries[0]['id'] > 0, "Id should be parsed")
-        self.assertTrue(entries[0]['url'].startswith("https://www.ebay-kleinanzeigen.de/s-anzeige"), u"URL should be an anzeige link")
-        for attr in [ 'title', 'price', 'size', 'rooms', 'address' ]:
-            self.assertIsNotNone(entries[0][attr], attr + " should be set")
+def test_crawler(crawler):
+    soup = crawler.get_page(TEST_URL)
+    assert soup is not None
+    entries = crawler.extract_data(soup)
+    assert entries is not None
+    assert len(entries) > 0
+    assert entries[0]['id'] > 0
+    assert entries[0]['url'].startswith("https://www.ebay-kleinanzeigen.de/s-anzeige")
+    for attr in [ 'title', 'price', 'size', 'rooms', 'address' ]:
+        assert entries[0][attr] is not None
 
+def test_process_expose_fetches_details(crawler):
+    soup = crawler.get_page(TEST_URL)
+    assert soup is not None
+    entries = crawler.extract_data(soup)
+    assert entries is not None
+    assert len(entries) > 0
+    updated_entries = [ crawler.get_expose_details(expose) for expose in entries ]
+    for expose in updated_entries:
+        print(expose)
+        for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'from' ]:
+            assert expose[attr] is not None

test/test_crawl_immobilienscout.py

@@ -1,21 +1,31 @@
-import unittest
-from flathunter.crawl_immobilienscout import CrawlImmobilienscout
+import pytest
 
-class ImmobilienscoutCrawlerTest(unittest.TestCase):
+from flathunter.crawl_immobilienscout import CrawlImmobilienscout
 
-    TEST_URL = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?numberofrooms=2.0-&price=-1500.0&livingspace=70.0-&sorting=2&pagenumber=1'
+TEST_URL = 'https://www.immobilienscout24.de/Suche/de/berlin/berlin/wohnung-mieten?numberofrooms=2.0-&price=-1500.0&livingspace=70.0-&sorting=2&pagenumber=1'
 
-    def setUp(self):
-        self.crawler = CrawlImmobilienscout()
+@pytest.fixture
+def crawler():
+    return CrawlImmobilienscout()
 
-    def test(self):
-        soup = self.crawler.get_page(self.TEST_URL, 1)
-        self.assertIsNotNone(soup, "Should get a soup from the URL")
-        entries = self.crawler.extract_data(soup)
-        self.assertIsNotNone(entries, "Should parse entries from search URL")
-        self.assertTrue(len(entries) > 0, "Should have at least one entry")
-        self.assertTrue(entries[0]['id'] > 0, "Id should be parsed")
-        self.assertTrue(entries[0]['url'].startswith("https://www.immobilienscout24.de/expose"), u"URL should be an exposé link")
-        for attr in [ 'title', 'price', 'size', 'rooms', 'address' ]:
-            self.assertIsNotNone(entries[0][attr], attr + " should be set")
+def test_crawl_works(crawler):
+    soup = crawler.get_page(TEST_URL, 1)
+    assert soup is not None
+    entries = crawler.extract_data(soup)
+    assert entries is not None
+    assert len(entries) > 0
+    assert entries[0]['id'] > 0
+    assert entries[0]['url'].startswith("https://www.immobilienscout24.de/expose")
+    for attr in [ 'title', 'price', 'size', 'rooms', 'address' ]:
+        assert entries[0][attr] is not None
 
+def test_process_expose_fetches_details(crawler):
+    soup = crawler.get_page(TEST_URL, 1)
+    assert soup is not None
+    entries = crawler.extract_data(soup)
+    assert entries is not None
+    assert len(entries) > 0
+    updated_entries = [ crawler.get_expose_details(expose) for expose in entries ]
+    for expose in updated_entries:
+        for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'from' ]:
+            assert expose[attr] is not None

test/test_crawl_immowelt.py

@@ -1,25 +1,38 @@
-import unittest
+import pytest
+
 from flathunter.crawl_immowelt import CrawlImmowelt
 from test_util import count
 
-class ImmoweltCrawlerTest(unittest.TestCase):
+TEST_URL = 'https://www.immowelt.de/liste/berlin/wohnungen/mieten?roomi=2&prima=1500&wflmi=70&sort=createdate%2Bdesc'
 
-    TEST_URL = 'https://www.immowelt.de/liste/berlin/wohnungen/mieten?roomi=2&prima=1500&wflmi=70&sort=createdate%2Bdesc'
+@pytest.fixture
+def crawler():
+    return CrawlImmowelt()
 
-    def setUp(self):
-        self.crawler = CrawlImmowelt()
 
-    def test(self):
-        soup = self.crawler.get_page(self.TEST_URL)
-        self.assertIsNotNone(soup, "Should get a soup from the URL")
-        entries = self.crawler.extract_data(soup)
-        self.assertIsNotNone(entries, "Should parse entries from search URL")
-        self.assertTrue(len(entries) > 0, "Should have at least one entry")
-        self.assertTrue(entries[0]['id'] > 0, "Id should be parsed")
-        self.assertTrue(entries[0]['url'].startswith("https://www.immowelt.de/expose"), u"URL should be an exposé link")
-        for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'image' ]:
-            self.assertIsNotNone(entries[0][attr], attr + " should be set")
+def test_crawler(crawler):
+    soup = crawler.get_page(TEST_URL)
+    assert soup is not None
+    entries = crawler.extract_data(soup)
+    assert entries is not None
+    assert len(entries) > 0
+    assert entries[0]['id'] > 0
+    assert entries[0]['url'].startswith("https://www.immowelt.de/expose")
+    for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'image' ]:
+        assert entries[0][attr] is not None
 
-def test_dont_crawl_other_urls():
-    exposes = CrawlImmowelt().crawl("https://www.example.com")
+def test_dont_crawl_other_urls(crawler):
+    exposes = crawler.crawl("https://www.example.com")
     assert count(exposes) == 0
+
+def test_process_expose_fetches_details(crawler):
+    soup = crawler.get_page(TEST_URL)
+    assert soup is not None
+    entries = crawler.extract_data(soup)
+    assert entries is not None
+    assert len(entries) > 0
+    updated_entries = [ crawler.get_expose_details(expose) for expose in entries ]
+    for expose in updated_entries:
+        print(expose)
+        for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'from' ]:
+            assert expose[attr] is not None

test/test_crawl_wggesucht.py

@@ -1,9 +1,10 @@
 import unittest
+from functools import reduce
 from flathunter.crawl_wggesucht import CrawlWgGesucht
 
 class WgGesuchtCrawlerTest(unittest.TestCase):
 
-    TEST_URL = 'https://www.wg-gesucht.de/wohnungen-in-Berlin.8.2.1.0.html?offer_filter=1&city_id=8&noDeact=1&categories%5B%5D=2&rent_types%5B%5D=2&sMin=70&rMax=1500&rmMin=2&fur=2&sin=2&exc=2&img_only=1'
+    TEST_URL = 'https://www.wg-gesucht.de/wohnungen-in-Berlin.8.2.1.0.html?offer_filter=1&city_id=8&noDeact=1&categories%5B%5D=2&rent_types%5B%5D=0&sMin=70&rMax=1500&rmMin=2&fur=2&sin=2&exc=2&img_only=1'
 
     def setUp(self):
         self.crawler = CrawlWgGesucht()
@@ -16,6 +17,9 @@ def test(self):
         self.assertTrue(len(entries) > 0, "Should have at least one entry")
         self.assertTrue(entries[0]['id'] > 0, "Id should be parsed")
         self.assertTrue(entries[0]['url'].startswith("https://www.wg-gesucht.de/wohnungen"), u"URL should be an apartment link")
-        for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'image' ]:
+        for attr in [ 'title', 'price', 'size', 'rooms', 'address', 'image', 'from' ]:
             self.assertIsNotNone(entries[0][attr], attr + " should be set")
+        for attr in [ 'to' ]:
+            found = reduce(lambda i, e: attr in e or i, entries, False)
+            self.assertTrue(found, "Expected " + attr + " to sometimes be set")
 

test/test_googlecloud_idmaintainer.py

@@ -27,12 +27,9 @@ def __init__(self):
 def id_watch():
     return MockGoogleCloudIdMaintainer()
 
-def test_read_from_empty_db(id_watch):
-    assert id_watch.get() == []
-
 def test_read_after_write(id_watch):
     id_watch.mark_processed(12345)
-    assert id_watch.get() == [12345]
+    assert id_watch.is_processed(12345)
 
 def test_get_last_run_time_none_by_default(id_watch):
     assert id_watch.get_last_run_time() == None
@@ -42,6 +39,15 @@ def test_get_list_run_time_is_updated(id_watch):
     assert time != None
     assert time == id_watch.get_last_run_time()
 
+def test_is_processed_works(id_watch):
+    config = Config(string=CONFIG_WITH_FILTERS)
+    config.set_searchers([DummyCrawler()])
+    hunter = Hunter(config, id_watch)
+    exposes = hunter.hunt_flats()
+    assert count(exposes) > 4
+    for expose in exposes:
+        assert id_watch.is_processed(expose['id'])
+
 def test_exposes_are_saved_to_maintainer(id_watch):
     config = Config(string=CONFIG_WITH_FILTERS)
     config.set_searchers([DummyCrawler()])

test/test_id_maintainer.py

@@ -29,12 +29,9 @@ class IdMaintainerTest(unittest.TestCase):
     def setUp(self):
         self.maintainer = IdMaintainer(":memory:")
 
-    def test_read_from_empty_db(self):
-        self.assertEqual(0, len(self.maintainer.get()), "Expected empty db to return empty array")
-
     def test_read_after_write(self):
         self.maintainer.mark_processed(12345)
-        self.assertEqual(12345, self.maintainer.get()[0], "Expected ID to be saved")
+        self.assertTrue(self.maintainer.is_processed(12345), "Expected ID to be saved")
 
     def test_get_last_run_time_none_by_default(self):
         self.assertIsNone(self.maintainer.get_last_run_time(), "Expected last run time to be none")
@@ -44,6 +41,16 @@ def test_get_list_run_time_is_updated(self):
         self.assertIsNotNone(time, "Expected time not to be none")
         self.assertEqual(time, self.maintainer.get_last_run_time(), "Expected last run time to be updated")
 
+def test_is_processed_works(mocker):
+    config = Config(string=IdMaintainerTest.DUMMY_CONFIG)
+    config.set_searchers([DummyCrawler()])
+    id_watch = IdMaintainer(":memory:")
+    hunter = Hunter(config, id_watch)
+    exposes = hunter.hunt_flats()
+    assert count(exposes) > 4
+    for expose in exposes:
+        assert id_watch.is_processed(expose['id'])
+
 def test_ids_are_added_to_maintainer(mocker):
     config = Config(string=IdMaintainerTest.DUMMY_CONFIG)
     config.set_searchers([DummyCrawler()])