Updates for Zappa hosting

brexit_legislation/s3_lambda_storage.py

@@ -0,0 +1,73 @@
+import os
+from distutils.dir_util import copy_tree
+
+from django.conf import settings
+from storages.backends.s3boto3 import S3Boto3Storage
+from django.contrib.staticfiles.storage import ManifestFilesMixin
+
+from django.contrib.staticfiles.finders import FileSystemFinder
+
+
+class ReadOnlySourceFileSystemFinder(FileSystemFinder):
+    """
+    A Django storage class for finding static files on a read only file system
+    or directory.
+
+    ## Why is this needed?
+
+    Finding files doesn't require write permissions, however 3rd party
+    projects like Django Pipeline can implement a `post_process` step.
+
+    This step takes the list of files returned by `list` and processes them,
+    in pipeline's case by converting them from scss to css etc.
+
+    The `post_process` step can change the file name, but if the file isn't
+    written to a path that a finder can find then it's ignored in later steps.
+
+    For example, ManifestFilesMixin won't know about that file.
+
+    To make everything work this Finder runs exactly like the normal
+    FileSystemFinder, but then copies the whole assets tree to a writable
+    location. It then updates the paths for all the files to that new location,
+    meaning all other processing of that file can happen as normal.
+
+    Manuel Both-Hanz pulled his hair out to bring you this information.
+    """
+
+    def list(self, ignore_patterns):
+        for origin_path, dest_path in settings.READ_ONLY_PATHS:
+            if dest_path.endswith('/'):
+                raise ValueError(
+                    "dest_path '{}' can't end with trailing /".format(
+                        dest_path)
+                )
+
+        super_list = list(super(
+            ReadOnlySourceFileSystemFinder, self).list(ignore_patterns))
+
+        for filename, file_storage in super_list:
+            location = file_storage.location
+            for origin_path, dest_path in settings.READ_ONLY_PATHS:
+                if location.startswith(origin_path):
+                    new_location = location.replace(origin_path, dest_path)
+                    copy_tree(location, new_location)
+                    file_storage.location = new_location
+
+        return super_list
+
+
+class MediaStorage(S3Boto3Storage):
+    """
+    Store media files at MEDIAFILES_LOCATION, post-process with pipeline
+    and then create manifest files for them.
+    """
+    location = settings.MEDIAFILES_LOCATION
+
+
+class StaticStorage(S3Boto3Storage):
+    """
+    Store static files at STATICFILES_LOCATION, post-process with pipeline
+    and then create manifest files for them.
+    """
+    location = settings.STATICFILES_LOCATION
+

brexit_legislation/settings/base.py

@@ -12,8 +12,10 @@
 
 # Build paths inside the project like this: os.path.join(BASE_DIR, ...)
 import os
+from os.path import join, abspath, dirname
 
 BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+root = lambda *x: join(abspath(BASE_DIR), *x)
 
 
 # Quick-start development settings - unsuitable for production
@@ -43,6 +45,7 @@
     'browse',
     'haystack',
     'search',
+    'django_filters'
 )
 
 MIDDLEWARE_CLASSES = (
@@ -125,15 +128,18 @@
 HAYSTACK_CONNECTIONS = {
     'default': {
         'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine',
-        'URL': 'http://localhost',
+        'URL': 'https://search-eu-regulation-lbekhmwifbj25rlbk2hgyqg5qy.eu-west-2.es.amazonaws.com',
         'INDEX_NAME': 'eu-regulations',
-        'TIMEOUT': 100,
+        'TIMEOUT': 2,
     },
 }
 
 # .local.py overrides all the common settings.
+import os
 try:
-    from .local import *  # noqa
+    if os.environ.get('FRAMEWORK', None) == 'Zappa':
+        from .zappa import *  # noqa
+    else:
+        from .local import *  # noqa
 except ImportError:
     pass
-

brexit_legislation/settings/zappa.py

@@ -0,0 +1,73 @@
+import os
+
+from .base import *  # noqa
+
+ZAPPA_STAGE = os.environ['STAGE']
+assert ZAPPA_STAGE in ('dev', 'prod')
+
+FORCE_SCRIPT_NAME = '/'
+
+ALLOWED_HOSTS = [
+    'pr9oybf6j5.execute-api.eu-west-1.amazonaws.com',  # Prod
+    'eu-regulation.labs.democracyclub.org.uk',  # Prod
+]
+USE_X_FORWARDED_HOST = True
+
+DATABASES = {
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql_psycopg2',
+        'HOST': os.environ.get('DATABASE_HOST'),
+        'USER': os.environ.get('DATABASE_USER'),
+        'PORT': '5432',
+        'NAME': os.environ.get('DATABASE_NAME'),
+        'PASSWORD': os.environ.get('DATABASE_PASS')
+    }
+}
+
+DEBUG = False
+
+HAYSTACK_CONNECTIONS = {
+    'default': {
+        'ENGINE': 'haystack.backends.elasticsearch2_backend.Elasticsearch2SearchEngine',
+        'URL': os.environ.get('ES_URL'),
+        'INDEX_NAME': 'eu-regulations',
+        'TIMEOUT': 2,
+    },
+}
+
+
+TMP_ASSETS_ROOT = "/tmp/assets_root"
+STATIC_ROOT = '/tmp/static_root/'
+
+# Make the tmp static dirs whenever django is started
+os.makedirs(TMP_ASSETS_ROOT, exist_ok=True)
+os.makedirs(STATIC_ROOT, exist_ok=True)
+
+STATICFILES_FINDERS = (
+    'brexit_legislation.s3_lambda_storage.ReadOnlySourceFileSystemFinder',
+    'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+)
+
+# Used by ReadOnlySourceFileSystemFinder
+READ_ONLY_PATHS = (
+    (root('assets'), TMP_ASSETS_ROOT),  # noqa
+)
+
+AWS_S3_SECURE_URLS = True
+AWS_S3_HOST = 's3-eu-west-1.amazonaws.com'
+AWS_S3_USE_SSL = False
+
+AWS_STORAGE_BUCKET_NAME = "static.labs.democracyclub.org.uk"
+AWS_S3_CUSTOM_DOMAIN = "static.labs.democracyclub.org.uk"
+
+STATICFILES_LOCATION = 'eu-regulation/static'
+STATIC_URL = "https://%s/%s/" % (AWS_S3_CUSTOM_DOMAIN, STATICFILES_LOCATION)
+STATICFILES_STORAGE = 'brexit_legislation.s3_lambda_storage.StaticStorage'
+
+MEDIAFILES_LOCATION = 'eu-regulation/media'
+MEDIA_URL = "https://{}/{}/".format(AWS_S3_CUSTOM_DOMAIN, MEDIAFILES_LOCATION)
+DEFAULT_FILE_STORAGE = 'brexit_legislation.s3_lambda_storage.MediaStorage'
+
+CSRF_TRUSTED_ORIGINS = [
+    'eu-regulation.labs.democracyclub.org.uk',
+]

browse/templates/search/indexes/regulations/regulation_text.txt

@@ -1 +1 @@
-{{ object.body_text }}
+{{ object.text_for_search_index }}

regulations/models.py

@@ -69,6 +69,14 @@ def html_text_only(self):
         soup = BeautifulSoup(self.body_html, "html5lib")
         return soup.find(id="TexteOnly")
 
+    def text_for_search_index(self):
+        text = ""
+        for summary in self.summaries.all():
+            soup = soup = BeautifulSoup(summary.summary, "html5lib")
+            text = text + " " + soup.get_text()
+        text = text + " " + self.body_text
+        return " ".join(text.split(" ")[:2000])
+
 
 class Summary(models.Model):
     uri = models.CharField(blank=True, max_length=1000)

requirements.txt

@@ -1,52 +1,17 @@
-agate==1.5.5
-agate-dbf==0.2.0
-agate-excel==0.2.0
-agate-sql==0.5.0
-appdirs==1.4.0
-appnope==0.1.0
-awesome-slugify==1.6.5
-Babel==2.3.4
 beautifulsoup4==4.5.3
-bs4==0.0.1
 csvkit==1.0.1
 csvtools==0.5.1
-dbfread==2.0.7
-decorator==4.0.11
-Django==1.10.5
+Django==1.11.8
 django-extensions==1.7.5
-django-filter==1.0.1
-django-haystack==2.6.0
-djangorestframework==3.5.3
+django-filter==1.1.0
+django-haystack==2.7.0
+djangorestframework==3.8.2
 drf-nested-routers==0.11.1
 elasticsearch==2.4.1
-et-xmlfile==1.0.1
 future==0.16.0
 html5lib==0.999999999
 ipdb==0.10.1
-ipython==5.1.0
-ipython-genutils==0.1.0
-isodate==0.5.4
-jdcal==1.3
-leather==0.3.3
 Markdown==2.6.7
-openpyxl==2.4.1
-packaging==16.8
-parsedatetime==2.2
-pexpect==4.2.1
-pickleshare==0.7.4
-prompt-toolkit==1.0.9
 psycopg2==2.6.2
-ptyprocess==0.5.1
-Pygments==2.2.0
-pyparsing==2.1.10
-pytimeparse==1.1.5
-pytz==2016.10
-regex==2017.1.17
-simplegeneric==0.8.1
-six==1.10.0
-SQLAlchemy==1.1.5
-traitlets==4.3.1
-Unidecode==0.4.20
-wcwidth==0.1.7
-webencodings==0.5
-xlrd==1.0.0
+zappa==0.46.1
+django-storages==1.6.5

search/views.py

@@ -1,9 +1,12 @@
+from django.views.decorators.csrf import csrf_exempt
+from django.utils.decorators import method_decorator
 from django.views.generic import TemplateView
 
 from .forms import SearchByDocumentForm
 from .more_like_text_helper import more_like_text
 
 
+@method_decorator(csrf_exempt, name='dispatch')
 class SearchView(TemplateView):
     template_name = "search/search.html"
 

zappa_settings.json

@@ -0,0 +1,33 @@
+{
+    "prod": {
+        "domain": "eu-regulation.labs.democracyclub.org.uk",
+        "aws_region": "eu-west-1",
+        "django_settings": "brexit_legislation.settings",
+        "profile_name": "dc",
+        "project_name": "eu_regulation",
+        "runtime": "python3.6",
+        "s3_bucket": "zappa-9pm0huf66",
+        "exclude": [
+            "*.csv",
+            "*.gz",
+            "*.log",
+            "*.pyc",
+            "*.rar",
+            "*.svg",
+            "*.sqlite3",
+            "babel",
+            "boto3*",
+            "botocore*",
+            "bower_components",
+            "faker",
+            "foundation-sites"
+            "IPython",
+            "local.py",
+            "pygments",
+            "sphinx",
+            "static_files",
+            "whoosh_index",
+            "__pycache__",
+        ]
+    }
+}