crawler.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os # To get the path of this file\n",
    "import scrapy # Library for the crawler\n",
    "import json # To output our data in JSON format\n",
    "import logging\n",
    "import PyPDF2 # To parse PDF files when a HTML version is not available\n",
    "import re # Substring searching\n",
    "\n",
    "from inline_requests import inline_requests # Addition to scrapy to make certain inline requests possible, used for PDFs\n",
    "\n",
    "# Scrapy specific imports\n",
    "from scrapy.crawler import CrawlerProcess\n",
    "from scrapy.spiders import CrawlSpider, Rule\n",
    "from scrapy.linkextractors import LinkExtractor\n",
    "from scrapy.http import Request\n",
    "\n",
    "# Settings for notebook & scrapy\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "DRY_RUN = False # If set to true, no files are actually written\n",
    "\n",
    "SEARCH_KEYS = [ # The search terms we are using to retrieve information\n",
    "    'cyber%20security',\n",
    "    'cybersecurity',\n",
    "    'cybercrime',\n",
    "]\n",
    "\n",
    "DEFAULT_URL = 'https://eur-lex.europa.eu/search.html?text={}&scope=EURLEX&type=quick&lang=en'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class JsonWriterPipeline(object):\n",
    "    \"\"\"\n",
    "    This pipeline will output all data in one JSON file, line separated.\n",
    "    \"\"\"\n",
    "\n",
    "    def open_spider(self, spider):\n",
    "        \"\"\"\n",
    "        When starting this pipeline, the file needs to be openend.\n",
    "        \"\"\"\n",
    "        self.file = open('euresult.jl', 'w')\n",
    "\n",
    "    def close_spider(self, spider):\n",
    "        \"\"\"\n",
    "        When the crawler is done, the file needs to be closed.\n",
    "        \"\"\"\n",
    "        self.file.close()\n",
    "\n",
    "    def process_item(self, item, spider):\n",
    "        \"\"\"\n",
    "        Every item that is processed is written to a new line in the file.\n",
    "        \"\"\"\n",
    "        line = json.dumps(dict(item)) + \"\\n\"\n",
    "        self.file.write(line)\n",
    "        return item\n",
    "    \n",
    "\n",
    "class JsonSeperateFileWriterPipeline(object):\n",
    "    \"\"\"\n",
    "    This pipeline will output all data in seperate JSON files, using the ID as file name, storing in the directory 'files'.\n",
    "    \"\"\"\n",
    "    \n",
    "    def open_spider(self, spider):\n",
    "        \"\"\"\n",
    "        We do not have to do anything when starting this pipeline\n",
    "        \"\"\"\n",
    "        logging.info(\"Starting JsonSeperateFileWriterPipeline\")\n",
    "        \n",
    "    def close_spider(self, spider):\n",
    "        \"\"\"\n",
    "        We print the statistics for this run when the spider quits\n",
    "        \"\"\"\n",
    "        logging.info(\"Closing JsonSeperateFileWriterPipeline\")\n",
    "\n",
    "    def process_item(self, item, spider):\n",
    "        \"\"\"\n",
    "        Every item that is processed is written to a seperate json file.\n",
    "        \"\"\"\n",
    "        # TODO: check if this approach works on Windows\n",
    "        \n",
    "        if not DRY_RUN:\n",
    "            \n",
    "            file_path = os.path.abspath(os.curdir) + '/files/{}.json'.format(item['local_id'].replace('/', ''))\n",
    "            \n",
    "            if not os.path.exists(file_path):\n",
    "                with open(file_path, 'w') as file:\n",
    "                    line = json.dumps(dict(item)) + \"\\n\"\n",
    "                    file.write(line)\n",
    "            else:\n",
    "                with open(file_path, 'r') as file:\n",
    "                    file_data = json.loads(file.read())\n",
    "                    item['search_terms'].update(file_data['search_terms'])\n",
    "                with open(file_path, 'w') as file:\n",
    "                    line = json.dumps(dict(item)) + \"\\n\"\n",
    "                    file.write(line)\n",
    "    \n",
    "        for key in item:\n",
    "            if not item.get(key) or item.get(key) == \"\":\n",
    "                logging.info(\"'\" + key + \"' is None/empty for \" + item['direct_url'])\n",
    "            \n",
    "        return item"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class EucybsecSpider(CrawlSpider):\n",
    "    \"\"\"\n",
    "    The spider designed for the European Union CELEX website.\n",
    "    \"\"\"\n",
    "    \n",
    "    name = 'eucybsec'\n",
    "    allowed_domains = ['eur-lex.europa.eu'] # URLs outside these domains are not followed\n",
    "    start_urls = [ DEFAULT_URL.format(key) for key in SEARCH_KEYS ]\n",
    "    \n",
    "    custom_settings = {\n",
    "        'LOG_LEVEL': logging.INFO,\n",
    "        'ITEM_PIPELINES': {\n",
    "#             '__main__.JsonWriterPipeline': 1,\n",
    "            '__main__.JsonSeperateFileWriterPipeline': 1,\n",
    "        }, \n",
    "        'RETRY_HTTP_CODES': [500, 503, 504, 400, 408, 404],\n",
    "        'RETRY_TIMES': 5,\n",
    "    }\n",
    "    \n",
    "    rules = (\n",
    "        Rule(\n",
    "            LinkExtractor(\n",
    "                allow=(),\n",
    "                restrict_xpaths=('(//a[@title=\"Next Page\"])[1]',), # This will extract the URL for the next page, if existent\n",
    "            ),\n",
    "            callback='parse_item',\n",
    "            follow=True,\n",
    "        ),\n",
    "    )\n",
    "    \n",
    "\n",
    "    @inline_requests\n",
    "    def parse_document_page(self, response):\n",
    "        \"\"\"\n",
    "        Yield the values for one specific document\n",
    "        \"\"\"\n",
    "        \n",
    "        # Get Document information page\n",
    "        di_rel_path = response.xpath('//a[contains(text(),\"Document information\")]/@href').extract_first()\n",
    "        doc_info = yield Request(response.urljoin(di_rel_path))\n",
    "        \n",
    "        # The neatest way to get the document date, but not always available\n",
    "        document_date = response.xpath('//meta[@property=\"eli:date_document\"]/@content').extract_first()\n",
    "        \n",
    "        # The other way to get the document date\n",
    "        if not document_date:\n",
    "            document_date = doc_info.xpath('//dt[contains(text(), \"Date of document\")]/following::dd[1]/text()').extract_first()\n",
    "            if document_date:\n",
    "                document_date = document_date.replace(';','')\n",
    "        \n",
    "        # The neatest way to get the publication date, but not always available\n",
    "        publication_date = response.xpath('//meta[@property=\"eli:date_publication\"]/@content').extract_first()\n",
    "        \n",
    "        # The other way to get the publication date.\n",
    "        if not publication_date:\n",
    "            publication_date = response.xpath('//p[@class=\"hd-date\"]/text()').extract_first()\n",
    "            if publication_date:\n",
    "                publication_date = publication_date.strip()\n",
    "                \n",
    "        # Turns out, if the publication date is still known, it is the same as the document date!\n",
    "        if not publication_date:\n",
    "            publication_date = document_date\n",
    "        \n",
    "        # Compress the content for efficiency\n",
    "        # TODO: See #5\n",
    "        content_elements = response.xpath('//div[@class=\"tabContent\"]//text()').extract()\n",
    "        content = ' '.join(content_elements)\n",
    "        \n",
    "        eurovoc_descriptors = doc_info.xpath('//dd[preceding-sibling::dt[contains(text(), \"EUROVOC\")]][1]/ul//span/text()').extract()\n",
    "        subject_matters = doc_info.xpath('//dt[contains(text(), \"Subject matter\")]/following::dd[1]//span/text()').extract()\n",
    "        author = doc_info.xpath('//dt[contains(text(), \"Author\")]/following::dd[1]/*/text()').extract_first()\n",
    "        \n",
    "        # Document type, two ways\n",
    "        document_type = response.xpath('//meta[@property=\"eli:type_document\"]/@resource').extract_first()\n",
    "        if not document_type:\n",
    "            document_type = doc_info.xpath('//dt[contains(text(), \"Form\")]/following::dd[1]/span/text()').extract_first()\n",
    "        \n",
    "        #### Content extraction\n",
    "        \n",
    "        # The HTML version is not available, so we are going to extract the PDF\n",
    "        if content == \"\":\n",
    "            \n",
    "            # Get the URL for the PDF file or stream page\n",
    "            pdf_href = response.xpath(\"//a[@id='format_language_table_PDF_EN']/@href\").extract_first()\n",
    "            \n",
    "            # Check whether this page even exist\n",
    "            if pdf_href:\n",
    "                                \n",
    "                # Get the PDF page\n",
    "                pdf_response = yield Request(response.urljoin(pdf_href))\n",
    "                \n",
    "                # Check if URL is actually a PDF file\n",
    "                content_type = pdf_response.headers.get('content-type').decode(\"utf-8\")\n",
    "                if not 'application/pdf' in content_type:\n",
    "                    \n",
    "                    # If not, there are probably multiple URLs here!\n",
    "                    streams = pdf_response.xpath(\"//ul[@class='multiStreams']//@href\").extract()\n",
    "                    \n",
    "                    # Extract every PDF seperately\n",
    "                    for stream in streams:\n",
    "                        stream_rsp = yield Request(pdf_response.urljoin(stream))\n",
    "                        content += self.extract_pdf(stream_rsp)\n",
    "                else:\n",
    "                    content = self.extract_pdf(pdf_response)\n",
    "            else:\n",
    "                logging.warning(\"No URL found! On url: \" + response.url)\n",
    "        \n",
    "        # Minimize the content for efficiency\n",
    "        content = ' '.join(content.split()).lower()\n",
    "        \n",
    "        #### End of content extraction\n",
    "                \n",
    "        yield {\n",
    "            'title': response.xpath(\"//p[@id='translatedTitle']/text()\").extract_first(),\n",
    "            'local_id': response.xpath('//*[contains(@class,\"DocumentTitle\")]/text()').extract_first().split()[1],\n",
    "            'document_type': document_type,\n",
    "            'publication_date': publication_date,\n",
    "            'document_date': document_date,\n",
    "            'direct_url': response.url,\n",
    "            'content': content,\n",
    "            'eurovoc_descriptors': eurovoc_descriptors,\n",
    "            'subject_matters': subject_matters,\n",
    "            'author': author,\n",
    "            'search_terms': { \n",
    "                response.meta['key_word']: {'page_number': response.meta['page_number'], 'order_id': response.meta['order_id']}\n",
    "            }\n",
    "        }\n",
    "\n",
    "    def parse_item(self, response):\n",
    "        \"\"\"\n",
    "        Parsing a page with search results, calling for the next page in the end.\n",
    "        \"\"\"\n",
    "        \n",
    "        key_word = re.search('text=(.*)&scope', response.request.url).group(1)\n",
    "        \n",
    "        page_number = int(response.xpath(\"//*[@title='Current page']/@value\").extract_first())\n",
    "        \n",
    "        item_links = response.xpath(\"//div[contains(@class,'SearchResult')]/h2/a/@href\").extract()\n",
    "            \n",
    "        for index, link in enumerate(item_links):\n",
    "            yield scrapy.Request(\n",
    "                response.urljoin(link), \n",
    "                callback=self.parse_document_page,\n",
    "                meta={\n",
    "                    'page_number': page_number,\n",
    "                    'order_id': index + 1,\n",
    "                    'key_word': key_word,\n",
    "                }\n",
    "            )       \n",
    "    \n",
    "    def parse_start_url(self, response):\n",
    "        \"\"\"\n",
    "        Parse the start_urls just as a regular page.\n",
    "        \"\"\"\n",
    "        return self.parse_item(response)\n",
    "    \n",
    "    \n",
    "    def extract_pdf(self, response):\n",
    "        \n",
    "        # Temporary store the PDF file locally\n",
    "        with open(\"tmp.pdf\", 'wb') as my_data:\n",
    "            my_data.write(response.body)\n",
    "\n",
    "        open_pdf_file = open(\"tmp.pdf\", 'rb')\n",
    "        \n",
    "        content = \"\"\n",
    "        \n",
    "        try:\n",
    "            pdfReader = PyPDF2.PdfFileReader(open_pdf_file)\n",
    "            \n",
    "            # append all the pages to the content\n",
    "            for page_number in range(pdfReader.numPages):\n",
    "                content += pdfReader.getPage(page_number).extractText()\n",
    "        \n",
    "        except Exception as e:\n",
    "            logging.warning(\"Could not read PDF on: \" + response.url +  \", Error:\")\n",
    "            logging.warning(e)\n",
    "            return \"\"\n",
    "          \n",
    "        return content\n",
    "            \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_crawler():\n",
    "    \"\"\"\n",
    "    Start the crawler\n",
    "    \"\"\"\n",
    "    process = CrawlerProcess({\n",
    "        'USER_AGENT': 'Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1)'\n",
    "    })\n",
    "    \n",
    "    process.crawl(EucybsecSpider)\n",
    "    process.start()\n",
    "    \n",
    "# TODO: Figure out how to 'restart' without having to fully restart the kernel.\n",
    "run_crawler()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.4.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}