Merge branch 'release-1.4.0'

.travis.yml

@@ -17,13 +17,15 @@ sudo: false
 # Cache the dependencies installed by pip
 cache: pip
 
-# Install defaults to "pip install -r requirements.txt"
+install:
+  - pip install -r requirements.txt
+  - pip install -r test_requirements.txt
 
 # Commands that prepare things for the test
 before_script:
   # create an empty mysql database
   - mysql -u root -e "create database apel_rest"
-  - mysql -u root apel_rest < schemas/cloud.sql
+  - mysql -u root apel_rest < schemas/10-cloud.sql
   - export PYTHONPATH=$PYTHONPATH:`pwd -P`
 
 # Command to run tests

Dockerfile

@@ -0,0 +1,61 @@
+FROM centos:6
+
+MAINTAINER APEL Administrator <[email protected]>
+
+# Add EPEL repo so we can get pip
+RUN rpm -ivh http://dl.fedoraproject.org/pub/epel/6/x86_64/epel-release-6-8.noarch.rpm
+
+# Add IGTF trust repo, so it can be installed
+RUN touch /etc/yum.repos.d/EGI-trustanchors.repo
+RUN echo -e $'# EGI Software Repository - REPO META (releaseId,repositoryId,repofileId) - (10824,-,2000)\n\
+[EGI-trustanchors]\n\
+name=EGI-trustanchors\n\
+baseurl=http://repository.egi.eu/sw/production/cas/1/current/\n\
+enabled=1\n\
+gpgcheck=1\n\
+gpgkey=http://repository.egi.eu/sw/production/cas/1/GPG-KEY-EUGridPMA-RPM-3' >> /etc/yum.repos.d/EGI-trustanchors.repo
+
+# Install python tools
+# Install mysql
+# Install apache
+# Install cron
+# Install at (for scheduling the IGTF update after start up)
+# Install IGTF trust bundle
+# Install fetch-crl
+RUN yum -y install python-pip python-devel mysql mysql-devel gcc httpd httpd-devel mod_wsgi mod_ssl cronie at ca-policy-egi-core fetch-crl
+
+# Copy APEL REST files to apache root
+COPY . /var/www/html/
+
+# Set the work diirectory to /var/www/html
+WORKDIR /var/www/html
+
+# Install APEL REST requirements
+RUN pip install -r requirements.txt
+
+# Copy APEL REST conf files to apache conf
+RUN cp conf/apel_rest_api.conf /etc/httpd/conf.d/apel_rest_api.conf
+
+# Copy SSL conf files to apache conf
+RUN cp conf/ssl.conf /etc/httpd/conf.d/ssl.conf
+
+# Make a directory for the certificates
+RUN mkdir /etc/httpd/ssl/
+
+# Make apel config dir for db configs
+RUN mkdir /etc/apel
+
+# Make cloud spool dir
+RUN mkdir -p /var/spool/apel/cloud/
+
+# Make cloud spool dir owned by apache
+RUN chown apache -R /var/spool/apel/cloud/
+
+# Generate static files
+RUN echo "yes" | python manage.py collectstatic 
+
+# Expose apache and SSL ports
+EXPOSE 80
+EXPOSE 443
+
+ENTRYPOINT docker/run_on_entry.sh

README.md

@@ -11,25 +11,55 @@ The collectors produce "Usage Records" in the APEL-Cloud v0.2 or v0.4 message fo
 
 These records need to be sent as POST requests to the REST endpoint `.../api/v1/cloud/record`, where `...` is the machine hosting the docker image. A POST request requires an X.509 certificate to authenticate the request. The hostname, which should be the same as the common name (CN) contained in the X.509 certificate, must be listed as a provider [here](http://indigo.cloud.plgrid.pl/cmdb/service/list) for the request to be authorized.
 
-Accepted records are summarised twice daily. These summaries can be accessed with a GET request to `.../api/v1/cloud/record/summary`. Summaries can be filtered using `key=value` pairs. See [Supported key=value pairs](doc/user.md#supported-keyvalue-pairs) for a list of valid supported `key=value` pairs. A GET request requires an IAM access token be included in the request. This token is then sent to the IAM to authenticate the ID of the service requesting access to the summary. This ID needs to be in `ALLOWED_FOR_GET` in `apel_rest/settings.py` for access to be authorized. See [Authorize new WP5 components to view Summaries](doc/admin.md#authorize-new-wp5-components-to-view-summaries) for instructions on adding service to `ALLOWED_FOR_GET`
+Accepted records are summarised twice daily. These summaries can be accessed with a GET request to `.../api/v1/cloud/record/summary`. Summaries can be filtered using `key=value` pairs. See [Supported key=value pairs](doc/user.md#supported-keyvalue-pairs) for a list of valid supported `key=value` pairs. A GET request requires an IAM access token be included in the request. This token is then sent to the IAM to authenticate the ID of the service requesting access to the summary. This ID needs to be in `ALLOWED_FOR_GET` in `yaml/apel_rest_interface.env` for access to be authorized. See [Authorize new WP5 components to view Summaries](doc/admin.md#authorize-new-wp5-components-to-view-summaries) for instructions on adding service to `ALLOWED_FOR_GET`
 
 It is currently expected that only the QoS/SLA tool will interact with these summaries.
 
-### Features of Version 1.3.0-1
+### Features of Version 1.4.0-1
 
 - Accept APEL-Cloud v0.2 and v0.4 usage records via POST requests to the REST endpoint `.../api/v1/cloud/record`
 - Provide access to summaries via GET requests to REST endpoint `.../api/v1/cloud/record/summary`
 
 ## Running the docker image on Centos 7 and Ubuntu 16.04
-We recommend using the docker image to run the Accounting server and REST interface. As such, having Docker installed is a prerequisite.
+We recommend using the docker image to run the Accounting server and REST interface. As such, having Docker and docker-compose installed is a prerequisite.
 
 See [Ubuntu 16.04 Instructions](https://docs.docker.com/engine/installation/linux/ubuntulinux/) or [Centos 7 Instructions](https://docs.docker.com/engine/installation/linux/centos/) for details of how to install Docker.
 
-1. Download and unzip the zip file of the latest version from [here](https://github.com/indigo-dc/Accounting/releases/latest). 
+See [Install Docker Compose](https://docs.docker.com/compose/install/) for details of how to install docker-compose
+
+1. Download the source code for the version you wish to deploy, see [here](https://github.com/indigo-dc/Accounting/releases) for a list of releases and corresponding docker image tag. The source code contains schemas and yaml files needed for deploying via docker.
 
 2. Register the service as a protected resource with the Indigo Identity Access Management (IAM) service. See [here](doc/admin.md#register-the-service-as-a-protected-resource-with-the-indigo-identity-access-management-iam) for instructions.
 
-3. Populate the following variables in `docker/run_container.sh`
+3. Populate the following variables in `yaml/apel_rest_interface.env`
+   ```
+   DJANGO_SECRET_KEY: The Django server requires its own "secret".
+
+   PROVIDERS_URL: Points to the JSON list of Resource Providers
+
+   IAM_URL: The introspect URL for the IAM repsonsible for token based authN/authZ
+
+   SERVER_IAM_ID: An IAM ID corresponding to this instance, used to validate tokens.
+
+   SERVER_IAM_SECRET: An IAM secret corresponding to this instance, used to validate tokens.
+
+   ALLOWED_FOR_GET: A (Python) list of IAM service IDs allowed to submit GET requests.
+                    (e.g. ['ac2f23e0-8103-4581-8014-e0e82c486e36'])
+
+   ALLOWED_TO_POST: A (Python) list of X.509 HostDNs allowed to submit POST requests,
+                    in addition to those listed by the PROVIDERS_URL.
+                    (e.g. ['/C=XX/O=XX/OU=XX/L=XX/CN=special_host.test'])
+
+   BANNED_FROM_POST: A (Python) list of X.509 HostDNs banned from submitting POST requests,
+                    even if they are listed by the PROVIDERS_URL.
+                    (e.g. ['/C=XX/O=XX/OU=XX/L=XX/CN=banned_host.test'])
+
+   SERVER_IAM_ID: An IAM ID corresponding to this instance, used to validate tokens.
+
+   SERVER_IAM_SECRET: An IAM secret corresponding to this instance, used to validate tokens.
+   ```
+
+4. Populate the following variables in `yaml/mysql.env`
    ```
    MYSQL_ROOT_PASSWORD: The APEL server will use this to communicate with the database. If run_container.sh
                         is deploying the database (which by default it will be), the database root password
@@ -38,20 +68,22 @@ See [Ubuntu 16.04 Instructions](https://docs.docker.com/engine/installation/linu
    MYSQL_PASSWORD: The APEL server will use this to communicate with the database. If run_container.sh
                    is deploying the database (which by default it will be) the APEL user password
                    is set to this.
+   ```
 
-   ALLOWED_FOR_GET: A (Python) list of IAM service IDs allowed to submit GET requests. This bash variable
-                    needs to be interpreted by python as a list of strings
-                    (e.g. [\'ac2f23e0-8103-4581-8014-e0e82c486e36\'])
+5. `MYSQL_PASSWORD` will also need to be added to the password field `docker/etc/apel/clouddb.cfg` and `docker/etc/mysql/my.cnf`
 
-   SERVER_IAM_ID: An IAM ID corresponding to this instance, used to validate tokens.
+6. Before the REST interface will start, a certificate needs to be added to the container. This can be done by placing a certificate (`apache.crt`) and key (`apache.key`) under `/etc/httpd/ssl/`. This directory will be mounted into the container by docker-compose.
 
-   SERVER_IAM_SECRET: An IAM secret corresponding to this instance, used to validate tokens.
-
-   DJANGO_SECRET_KEY: The Django server requires its own "secret".
+7. Make `docker/etc/cron.d` owned by `root`. This is required because this directory gets mounted into the container and it needs to be owned by root for cron jobs in the container to run.
+   ```
+   chown -R root docker/etc/cron.d
    ```
 
-4. Run `./docker/run_container.sh indigodatacloud/accounting:X.X.X-X`, where X.X.X-X is the version number of the container to launch.
-
-5. Before the server will start, a certificate needs to be added to the container. This can be done by either modifying `./docker/run_container.sh` to load the docker image with a certificate mounted into it, or by interacting with the image after start up with `docker exec -it <docker_id> bash`. If choosing the latter, run `service httpd start` before exiting the container.
+8. Launch the containers. It is recommeded to wait 1 minute in order for each container to configure fully before launching the next
+   ```
+   docker-compose -f yaml/docker-compose.yaml up -d --force-recreate apel_mysql
+   docker-compose -f yaml/docker-compose.yaml up -d --force-recreate apel_server
+   docker-compose -f yaml/docker-compose.yaml up -d --force-recreate apel_rest_interface
+   ```
 
-6. Navigate a web browser to `https://\<hostname\>/api/v1/cloud/record/summary/`
+9. Navigate a web browser to `https://<hostname>/api/v1/cloud/record/summary`

apel_rest/__init__.py

@@ -15,4 +15,4 @@
 
 @author Greg Corbett
 """
-__version__ = '1, 3, 0'
+__version__ = '1, 4, 0'

apel_rest/settings.py

@@ -18,9 +18,6 @@
 # Quick-start development settings - unsuitable for production
 # See https://docs.djangoproject.com/en/1.6/howto/deployment/checklist/
 
-# SECURITY WARNING: keep the secret key used in production secret!
-SECRET_KEY = 'Put a secret here'
-
 # SECURITY WARNING: don't run with debug turned on in production!
 DEBUG = False
 
@@ -161,13 +158,15 @@
 # this doesnt do anything, probably because of using older Django
 URL_FORMAT_OVERRIDE = None
 
-# Points to the JSON list of Resource Providers
-PROVIDERS_URL = "http://indigo.cloud.plgrid.pl/cmdb/service/list"
+# These variables replaced on entry into the docker container
+SECRET_KEY = 'not_a_secure_secret'
+
+PROVIDERS_URL = 'provider_url'
 
-# The introspect URL for the IAM repsonsible for token based authN/authZ
-IAM_URL = "https://iam-test.indigo-datacloud.eu/introspect"
+IAM_URL = 'iam_url'
+SERVER_IAM_ID = 'server_iam_id'
+SERVER_IAM_SECRET = 'server_iam_secret'
 
-# Use these variables to revoke/grant POST rights.
-# Remember these variables require a web server restart to take effect.
-ALLOWED_TO_POST = []
-BANNED_FROM_POST = []
+ALLOWED_TO_POST = ['allowed_to_post']
+BANNED_FROM_POST = ['banned_from_post']
+ALLOWED_FOR_GET = ['allowed_for_get']

api/__init__.py

@@ -15,4 +15,4 @@
 
 @author Greg Corbett
 """
-__version__ = '1, 3, 0'
+__version__ = '1, 4, 0'

api/tests/__init__.py

@@ -1,18 +1 @@
-"""
-Copyright (C) 2016 STFC.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-@author Greg Corbett
-"""
-__version__ = '1, 3, 0'
+"""This file allows test to be imported as a python package."""