Set up CKAN with Vagrant and ansible provisioning

Largely copied from a Github gist (see comment in the playbook), this
provisions a Vagrant box containing a working installation of CKAN. CKAN
is installed from a release package, rather than from source; as best I
can tell we can extend it via extensions.

I tried to use Ubuntu 14.04 initially but the package install fails, and
it turns out that the current 2.2 release of CKAN doesn't support 14.04
(ckan/ckan#1651), so this uses Ubuntu 12.04.

.gitignore

@@ -0,0 +1,45 @@
+*.py[cod]
+
+# Packages
+*.egg
+*.egg-info
+dist
+build
+eggs
+parts
+bin
+var
+sdist
+develop-eggs
+.installed.cfg
+lib
+lib64
+__pycache__
+
+# Installer logs
+pip-log.txt
+
+# Unit test / coverage reports
+artifacts/
+.coverage
+.tox
+nosetests.xml
+.noseids
+
+# vagrant
+.vagrant
+
+# sass
+.sass-cache
+
+# Editor crap
+*.swp
+*.swo
+*~
+~*
+
+# OS X crap
+.DS_Store
+
+# ignore logs
+*.log

Vagrantfile

@@ -0,0 +1,74 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
+VAGRANTFILE_API_VERSION = "2"
+
+Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
+  # All Vagrant configuration is done here. The most common configuration
+  # options are documented and commented below. For a complete reference,
+  # please see the online documentation at vagrantup.com.
+
+  # Every Vagrant virtual environment requires a box to build off of.
+  config.vm.box = "ubuntu/precise64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  config.vm.network "forwarded_port", guest: 8080, host: 8025
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "192.168.8.25"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # If true, then any SSH connections made will enable agent forwarding.
+  # Default value: false
+  # config.ssh.forward_agent = true
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+#  Wire up the proxy
+  def local_ip
+    `hostname -I | cut -f1 -d' '`.strip
+  end
+  if Vagrant.has_plugin?("vagrant-proxyconf")
+    config.proxy.http     = "http://#{local_ip}:8123/"
+    config.proxy.https    = "http://#{local_ip}:8123/"
+    config.proxy.no_proxy = "localhost,127.0.0.1"
+  end
+  config.vm.provision :ansible do |ansible|
+    ansible.playbook = "deployment/ansible/site.yml"
+    # Note: If running vagrant < 1.3.0 you may need to change this to ansible.inventory_file
+    ansible.verbose = 'v'
+    ansible.inventory_path = "deployment/ansible/hosts/hosts.vagrant"
+    ansible.limit = 'all'
+  end
+# Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  # config.vm.provider "virtualbox" do |vb|
+  #   # Don't boot with headless mode
+  #   vb.gui = true
+  #
+  #   # Use VBoxManage to customize the VM. For example to change memory:
+  #   vb.customize ["modifyvm", :id, "--memory", "1024"]
+  # end
+  #
+  # View the documentation for the provider you're using for more
+  # information on available options.
+
+end

deployment/ansible/hosts/hosts.vagrant

@@ -0,0 +1,2 @@
+[vagrant]
+192.168.8.25    ansible_ssh_user=vagrant

deployment/ansible/roles/ckan/defaults/main.yml

@@ -0,0 +1,5 @@
+---
+db_name: ckan_default
+db_user: ckan_default
+db_password: ckan_default
+ckan_package_filename: 'python-ckan_2.2_amd64.deb'

deployment/ansible/roles/ckan/handlers/main.yml

@@ -0,0 +1,11 @@
+- name: Restart Apache
+  service: name=apache2 state=restarted
+
+- name: Restart Nginx
+  service: name=nginx state=restarted
+
+- name: Start Jetty
+  service: name=jetty state=started
+
+- name: Restart Jetty
+  service: name=jetty state=restarted

deployment/ansible/roles/ckan/tasks/main.yml

@@ -0,0 +1,151 @@
+# Adapted from https://gist.github.com/jeffbr13/08751e42c9355cc44f5d
+---
+  - name: Update apt cache
+    apt: update_cache=yes
+
+  - name: Upgrade all safe packages
+    apt: upgrade=safe
+
+  - name: Install unattended upgrades (Debian/Ubuntu only)
+    apt: pkg=unattended-upgrades state=installed
+    when: ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu'
+
+  - name: 'Make sure unattended-upgrades only installs from $ubuntu_release-security'
+    lineinfile: 'dest=/etc/apt/apt.conf.d/50unattended-upgrades regexp="$ubuntu_release-updates" state=absent'
+    when: ansible_distribution == 'Ubuntu'
+
+  - name: Install necessities and nice-to-haves
+    apt: pkg={{ item }} state=installed
+    with_items:
+      - acl
+      - apache2
+      - apt-transport-https
+      - apticron
+      - aptitude
+      - build-essential
+      - debian-goodies
+      - fail2ban
+      - git
+      - htop
+      - iftop
+      - iotop
+      - libapache2-mod-wsgi
+      - libpq5
+      - nginx
+      - postgresql
+      - python
+      - python-pip
+      - python-psycopg2
+      - python-pycurl
+      - python-software-properties
+      - screen
+      - solr-jetty
+      - sudo
+      - update-notifier-common
+      - wget
+    when: ansible_pkg_mgr == 'apt'
+
+#    - name: Enable passwordless sudo
+#      lineinfile: 'dest=/etc/sudoers regexp="sudo ALL=NOPASSWD: ALL" line="%sudo ALL=NOPASSWD: ALL" state=present'
+
+  - name: Download CKAN package
+    get_url: 'url="http://packaging.ckan.org/{{ ckan_package_filename }}" dest=/tmp/{{ ckan_package_filename }}'
+
+  - name:  Install CKAN package
+    command: 'dpkg --skip-same-version -i /tmp/{{ ckan_package_filename }}'
+    # http://stackoverflow.com/questions/19127493/in-ansible-how-do-you-prevent-a-dpkg-installation-task-to-notify-a-changed-stat
+    register: ckan_installed
+    changed_when: "'already installed' not in ckan_installed.stderr"
+    notify:
+      - Restart Apache
+      - Restart Nginx
+
+  # Jetty & Solr
+  - name: Set Jetty to start on boot
+    lineinfile: 'dest=/etc/default/jetty regexp=^NO_START line="NO_START=0"'
+
+  - name: Set Jetty host to localhost
+    lineinfile: 'dest=/etc/default/jetty regexp=^JETTY_HOST line="JETTY_HOST=127.0.0.1"'
+
+  - name: Set Jetty to port 8983
+    lineinfile: 'dest=/etc/default/jetty regexp=^JETTY_PORT line="JETTY_PORT=8983"'
+
+  - name: Set Jetty to use system java
+    lineinfile: 'dest=/etc/default/jetty regexp=JAVA_HOME line="JAVA_HOME=/usr/lib/jvm/java-6-openjdk-amd64/"'
+    notify: Start Jetty
+
+  - name: Remove CKAN schema file
+    file: path=/etc/solr/conf/schema.xml state=absent
+    when: ckan_installed.changed
+
+  - name: Ensure CKAN uses provided schema file
+    file: path=/etc/solr/conf/schema.xml src=/usr/lib/ckan/default/src/ckan/ckan/config/solr/schema.xml state=link
+    notify: Restart Jetty
+
+  - name: Set CKAN Solr server address
+    lineinfile: 'dest=/etc/ckan/default/production.ini regexp=solr_url line=solr_url=http://127.0.0.1:8983/solr'
+
+  # Postgres
+  - name: Ensure CKAN database is created
+    postgresql_db: 'name={{ db_name }}'
+    sudo_user: postgres
+
+  - name: Ensure CKAN database user can access CKAN database
+    postgresql_user: 'db={{ db_name }} name={{ db_user }} password={{ db_password }} priv=ALL'
+    sudo_user: postgres
+
+  - name: Minimise CKAN database user privileges
+    postgresql_user: 'name={{ db_user }} role_attr_flags=NOSUPERUSER,NOCREATEROLE,NOCREATEDB'
+    sudo_user: postgres
+
+  - name: Set CKAN database server address
+    lineinfile: 'dest=/etc/ckan/default/production.ini regexp=sqlalchemy.url line="sqlalchemy.url = postgresql://{{ db_user }}:{{ db_password }}@localhost/{{ db_name }}?sslmode=disable"'
+
+  - name: Ensure database is initialised
+    command: ckan db init
+    notify:
+      - Restart Apache
+      - Restart Nginx
+
+  - name: Remove Repoze.who configuration file for CKAN
+    file: path=/usr/lib/ckan/default/src/ckan/who.ini state=absent
+    when: ckan_installed.changed
+
+  - name: Link Repoze.who configuration file for CKAN
+    file: 'path=/usr/lib/ckan/default/src/ckan/who.ini src=/etc/ckan/default/who.ini state=link'
+
+  # DataStore
+  - name: Ensure DataStore database exists
+    postgresql_db: name=datastore_default owner=ckan_default
+    sudo_user: postgres
+
+  - name: Ensure CKAN database user owns DataStore database
+    postgresql_user: 'db=datastore_default name=ckan_default password={{ db_password }} priv=ALL'
+    sudo_user: postgres
+
+  - name: Ensure DataStore database user exists
+    postgresql_user: 'name=datastore_default password={{ db_password }}'
+    sudo_user: postgres
+
+  - name: Set DataStore database server write address
+    lineinfile: 'dest=/etc/ckan/default/production.ini regexp="ckan.datastore.write_url" line="ckan.datastore.write_url = postgresql://ckan_default:{{ db_password }}@localhost/datastore_default"'
+
+  - name: Set DataStore database server read address
+    lineinfile: 'dest=/etc/ckan/default/production.ini regexp="ckan.datastore.read_url" line="ckan.datastore.read_url = postgresql://datastore_default:{{ db_password }}@localhost/datastore_default"'
+
+  - name: Set DataStore database permissions
+    command: ckan datastore set-permissions postgres
+
+  # FileStore
+  - name: Ensure FileStore directory exists
+    file: path=/var/lib/ckan/default owner=www-data state=directory
+
+  - name: Set FileStore directory path
+    lineinfile: 'dest=/etc/ckan/default/production.ini regexp="ckan.storage_path" line="ckan.storage_path = /var/lib/ckan/default"'
+
+  - name: Ensure Apache can write to FileStore directory
+    # Introduced in Ansible 1.4, deprecated 3 months later in Ansible 1.5. *sigh*
+    acl: name=/var/lib/ckan/default entry='user:www-data:u+rwx'
+    # This is now the preferred syntax but unsupported in Ansible 1.4
+    #acl: name=/var/lib/ckan/default entity=www-data etype=user permissions=u+rwx
+    notify: Restart Apache

deployment/ansible/site.yml

@@ -0,0 +1,5 @@
+---
+- hosts: vagrant
+  sudo: True
+  roles:
+    - { role: ckan }