Cria spider para Duque de Caxias RJ no padrão atual do projeto

data_collection/gazette/spiders/rj/rj_duque_caxias.py

@@ -0,0 +1,56 @@
+import re
+from datetime import date, datetime
+
+from scrapy import Request
+
+from gazette.items import Gazette
+from gazette.spiders.base import BaseGazetteSpider
+from gazette.utils import extract_date
+
+
+class RjDuqueCaxiasSpider(BaseGazetteSpider):
+    name = "rj_duque_caxias"
+    TERRITORY_ID = "3301702"
+    allowed_domains = ["duquedecaxias.rj.gov.br"]
+    start_date = date(2017, 1, 2)
+
+    def start_requests(self):
+        current_year = datetime.today().year
+        for year in range(self.start_date.year, self.end_date.year + 1):
+            if year == current_year:
+                yield Request(
+                    "https://duquedecaxias.rj.gov.br/portal/boletim-oficial.html"
+                )
+            else:
+                yield Request(f"https://duquedecaxias.rj.gov.br/portal/{year}.html")
+
+    def parse(self, response):
+        pdf_divs = response.xpath(
+            "//i[contains(@class, 'fa-file-pdf')]/ancestor::div[1]"
+        )
+        # use descending order to reduce iterations in the daily crawl
+        for pdf_div in pdf_divs[::-1]:
+            raw_gazette_date = pdf_div.xpath("./preceding-sibling::div[1]/text()").get()
+            gazette_date = extract_date(raw_gazette_date)
+            if not gazette_date or gazette_date > self.end_date:
+                continue
+            if gazette_date < self.start_date:
+                return
+
+            raw_gazette_edtion = pdf_div.xpath("./preceding-sibling::div[2]/text()")
+            is_extra_edition = bool(
+                re.search(r"extra|esp", raw_gazette_edtion.get(), re.IGNORECASE)
+            )
+            edition_number = (
+                None if is_extra_edition else raw_gazette_edtion.re_first(r"\d+")
+            )
+
+            gazette_url = response.urljoin(pdf_div.xpath(".//a/@href").get())
+
+            yield Gazette(
+                date=gazette_date,
+                edition_number=edition_number,
+                is_extra_edition=is_extra_edition,
+                file_urls=[gazette_url],
+                power="executive_legislative",
+            )

data_collection/gazette/utils.py

@@ -1,8 +1,27 @@
+import re
+
+import dateparser
+from fuzzywuzzy import process
 from sqlalchemy import create_engine, select
 from sqlalchemy.orm import sessionmaker
 
 from gazette.database.models import QueridoDiarioSpider
 
+MONTHS = [
+    "janeiro",
+    "fevereiro",
+    "março",
+    "abril",
+    "maio",
+    "junho",
+    "julho",
+    "agosto",
+    "setembro",
+    "outubro",
+    "novembro",
+    "dezembro",
+]
+
 
 def get_enabled_spiders(*, database_url, start_date=None, end_date=None):
     """Return list of all currently enabled spiders within date period.
@@ -22,3 +41,30 @@ def get_enabled_spiders(*, database_url, start_date=None, end_date=None):
     result = session.execute(stmt)
     for spider in result.scalars():
         yield spider.spider_name
+
+
+def extract_date(text):
+    """Extract a date from a text. This method attempts to correct typing errors in the month.
+
+    Args:
+        text: A text containing a date with the name of the month full version (%B)
+    Returns:
+        The date, if match. Otherwise, returns None.
+    """
+
+    text = re.sub(" +", " ", text).strip()
+    match_date = re.search(r"\d{1,2}º?(\sde)? +(\w+)(\sde)? +\d{4}", text)
+    if not match_date:
+        return None
+
+    raw_date = match_date.group(0)
+    raw_date = raw_date.replace("º", "").replace("°", "")
+    month = match_date.group(2)
+    if month.lower() not in MONTHS:
+        match_month, score = process.extractOne(month, MONTHS)
+        if score < 70:
+            return None
+        raw_date = raw_date.replace(month, match_month)
+
+    parsed_datetime = dateparser.parse(raw_date, languages=["pt"])
+    return parsed_datetime.date() if parsed_datetime else None