Merge pull request #55 from scossu/development

Alpha 13.

.github/release_template.md

@@ -0,0 +1,25 @@
+# What Changed In This Release
+
+[High-level, colloquial summary of changes—mandatory]
+
+## New Features
+
+-
+
+## Enhancements
+
+-
+
+## Bug Fixes
+
+-
+
+## Other Changes
+
+-
+
+## Notes & Caveats
+
+-
+
+## Acknowledgments

.gitignore

@@ -104,5 +104,4 @@ venv.bak/
 .mypy_cache/
 
 # Default LAKEsuperior data directories
-data/ldpnr_store
-data/ldprs_store
+/data

.travis.yml

@@ -15,5 +15,6 @@ deploy:
     on:
         tags: true
         branch: master
+        python: "3.5"
     distributions: "bdist_wheel"
 

MANIFEST.in

@@ -1,4 +1,5 @@
 include README.rst
 include LICENSE
+graft lakesuperior/data/bootstrap
 graft lakesuperior/endpoints/templates
 graft lakesuperior/etc.defaults

VERSION

@@ -0,0 +1 @@
+1.0.0a13

conftest.py

@@ -1,22 +1,31 @@
-import sys
-
 import pytest
 
-sys.path.append('.')
-from lakesuperior.config_parser import test_config
+from os import makedirs, path
+from shutil import rmtree
+from tempfile import gettempdir
+
+from lakesuperior import env
+from lakesuperior.config_parser import parse_config
 from lakesuperior.globals import AppGlobals
-from lakesuperior.env import env
+from lakesuperior.util.generators import random_image
 
-env.config = test_config
-env.app_globals = AppGlobals(test_config)
+
+# Override data directory locations.
+config = parse_config()
+data_dir = path.join(gettempdir(), 'lsup_test', 'data')
+config['application']['data_dir'] = data_dir
+config['application']['store']['ldp_nr']['location'] = (
+        path.join(data_dir, 'ldpnr_store'))
+config['application']['store']['ldp_rs']['location'] = (
+        path.join(data_dir, 'ldprs_store'))
+
+env.app_globals = AppGlobals(config)
 from lakesuperior.app import create_app
-from lakesuperior.util.generators import random_image
 
-env.config = test_config
 
 @pytest.fixture(scope='module')
 def app():
-    app = create_app(env.config['application'])
+    app = create_app(env.app_globals.config['application'])
 
     yield app
 
@@ -26,14 +35,17 @@ def db(app):
     '''
     Set up and tear down test triplestore.
     '''
-    rdfly = env.app_globals.rdfly
-    rdfly.bootstrap()
+    makedirs(data_dir, exist_ok=True)
+    env.app_globals.rdfly.bootstrap()
     env.app_globals.nonrdfly.bootstrap()
+    print('Initialized data store.')
 
-    yield rdfly
+    yield env.app_globals.rdfly
 
-    print('Tearing down fixture graph store.')
-    rdfly.store.destroy(rdfly.store.path)
+    # TODO improve this by using tempfile.TemporaryDirectory as a context
+    # manager.
+    print('Removing fixture data directory.')
+    rmtree(data_dir)
 
 
 @pytest.fixture

docs/apidoc/lakesuperior.rst

@@ -34,7 +34,7 @@ lakesuperior\.config\_parser module
 lakesuperior\.env module
 ------------------------
 
-.. automodule:: lakesuperior.env
+.. automodule:: lakesuperior
     :members:
     :undoc-members:
     :show-inheritance:

docs/cli.rst

@@ -1,13 +1,39 @@
-LAKEsuperior Command Line Reference
-===================================
+Command Line Reference
+======================
 
-The LAKEsuperior command line tool is used for maintenance and
-administration purposes.
+LAKEsuperior comes with some command-line tools aimed at several purposes.
 
-The script should be in your executable path if you install LAKEsuperior with
-``pip``. The tool is self-documented, so this is just a redundant overview::
+If LAKEsuperior is installed via ``pip``, all tools can be invoked as normal
+commands (i.e. they are in the virtualenv ``PATH``). 
 
-    $ lsup_admin
+The tools are currently not directly available on Docker instances (*TODO add
+instructions and/or code changes to access them*).
+
+``fcrepo``
+----------
+
+This is the main server command. It has no parameters. The command spawns
+Gunicorn workers (as many as set up in the configuration) and can be sent in
+the background, or started via init script.
+
+The tool must be run in the same virtual environment LAKEsuperior
+was installed in (if it was)—i.e.::
+
+    source <virtualenv root>/bin/activate
+
+must be run before running the server.
+
+In the case an init script is used, ``coilmq`` (belonging to a 3rd party
+package) needs to be launched as well; unless a message broker is already set
+up, or if messaging is disabled in the configuration.
+
+``lsup-admin``
+--------------
+
+``lsup-admin`` is the principal repository management tool. It is
+self-documented, so this is just a redundant overview::
+
+    $ lsup-admin
     Usage: lsup-admin [OPTIONS] COMMAND [ARGS]...
 
     Options:
@@ -21,8 +47,6 @@ The script should be in your executable path if you install LAKEsuperior with
       migrate       Migrate an LDP repository to LAKEsuperior.
       stats         Print repository statistics.
 
-*TODO: Add instructions to access from Docker.*
-
 All entries marked ``[STUB]`` are not yet implemented, however the
 ``lsup_admin <command> --help`` command will issue a description of what
 the command is meant to do. Check the
@@ -31,3 +55,37 @@ the radar.
 
 All of the above commands are also available via, and based upon, the
 native Python API.
+
+``lsup-benchmark``
+------------------
+
+``lsup-benchmark`` is used to run performance tests in a predictable way.
+
+The command has no options but prompts the user for a few settings
+interactively (N.B. this may change in favor of parameters).
+
+The benchmark tool is able to create RDF sources, or non-RDF, or an equal mix
+of them, via POST or PUT, in the currently running LAKEsuperior server. It
+runs single-threaded.
+
+The RDF sources are randomly generated graphs of consistent size and
+complexity. They include a mix of in-repository references, literals, and
+external URIs. Each graph has 200 triples.
+
+The non-RDF sources are randomly generated 1024x1024 pixel PNG images.
+
+You are warmly encouraged to run the script and share the performance results (
+*TODO add template for posting results*).
+
+``profiler``
+------------
+
+This command launches a single-threaded HTTP server (Flask) on port 5000 that
+logs profiling information. This is useful for analyzing application
+performance.
+
+For more information, consult the `Python profilers guide
+<https://docs.python.org/3/library/profile.html>`__.
+
+Do not launch this while a WSGI server (``fcrepo``) is already running, because
+that also launches a Flask server on port 5000.

docs/conf.py

@@ -17,13 +17,11 @@
 # add these directories to sys.path here. If the directory is relative to the
 # documentation root, use os.path.abspath to make it absolute, like shown here.
 #
-import os
 import sys
 
+from os import path
 from unittest.mock import MagicMock
 
-#sys.path.append(os.path.abspath('../'))
-
 class MockModule(MagicMock):
     @classmethod
     def __getattr__(cls, name):
@@ -37,6 +35,8 @@ def __getattr__(cls, name):
 import lakesuperior.env_setup
 
 
+here = path.abspath(path.dirname(__file__))
+
 # -- General configuration ------------------------------------------------
 
 # If your documentation needs a minimal Sphinx version, state it here.
@@ -74,10 +74,9 @@ def __getattr__(cls, name):
 # |version| and |release|, also used in various other places throughout the
 # built documents.
 #
-# The short X.Y version.
-version = '1.0-alpha'
-# The full version, including alpha/beta/rc tags.
-release = '1.0.0-alpha.8'
+# Version and release are the same.
+with open(path.realpath(path.join(here, '..', 'VERSION'))) as fh:
+    version = release = fh.readlines()[0]
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/setup.rst

@@ -79,9 +79,14 @@ Configuration
 
 The app should run for testing and evaluation purposes without any
 further configuration. All the application data are stored by default in
-the ``data`` directory.
+the ``data`` directory of the Python package.
 
-To change the default configuration you should:
+This setup is not recommended for anything more than a quick look at the
+application. If more complex interaction is needed, or upgrades to the package
+are foreseen, it is strongly advised to set up proper locations for
+configuration and data.
+
+To change the default configuration you need to:
 
 #. Copy the ``etc.default`` folder to a separate location
 #. Set the configuration folder location in the environment:
@@ -94,10 +99,13 @@ To change the default configuration you should:
 
 The configuration options are documented in the files.
 
-**Note:** ``test.yml`` must specify a different location for the graph
-and for the binary stores than the default one, otherwise running a test
-suite will destroy your main data store. The application will issue an
-error message and refuse to start if these locations overlap.
+One thing worth noting is that some locations can be specified as relative
+paths. These paths will be relative to the ``data_dir`` location specified in
+the ``application.yml`` file.
+
+If ``data_dir`` is empty, as it is in the default configuration, it defaults
+to the ``data`` directory inside the Python package. This is the option that
+one may want to change before anything else.
 
 Production deployment
 ---------------------

docs/usage.rst

@@ -123,11 +123,12 @@ Before using the API, either do::
 
 Or, to specify an alternative configuration::
 
+    >>> from lakesuperior import env
     >>> from lakesuperior.config_parser import parse_config
     >>> from lakesuperior.globals import AppGlobals
-    >>> env.config, test_config = parse_config('/my/custom/config_dir')
+    >>> config = parse_config('/my/custom/config_dir')
     Reading configuration at /my/custom/config_dir
-    >>> env.app_globals = AppGlobals(env.config)
+    >>> env.app_globals = AppGlobals(config)
 
 Create and replace resources
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~

lakesuperior/__init__.py

@@ -0,0 +1,49 @@
+import threading
+
+from os import path
+
+basedir = path.dirname(path.realpath(__file__))
+"""
+Base directory for the module.
+
+This can be used by modules looking for configuration and data files to be
+referenced or copied with a known path relative to the package root.
+
+:rtype: str
+"""
+
+class Env:
+    pass
+
+env = Env()
+"""
+A pox on "globals are evil".
+
+All-purpose bucket for storing global variables. Different environments
+(e.g. webapp, test suite) put the appropriate value in it.
+The most important values to be stored are app_conf (either from
+lakesuperior.config_parser.config or lakesuperior.config_parser.test_config)
+and app_globals (obtained by an instance of lakesuperior.globals.AppGlobals).
+
+e.g.::
+
+    >>> from lakesuperior.config_parser import config
+    >>> from lakesuperior.globals import AppGlobals
+    >>> from lakesuperior import env
+    >>> env.app_globals = AppGlobals(config)
+
+This is automated in non-test environments by importing
+`lakesuperior.env_setup`.
+
+:rtype: Object
+"""
+
+thread_env = threading.local()
+"""
+Thread-local environment.
+
+This is used to store thread-specific variables such as start/end request
+timestamps.
+
+:rtype: threading.local
+"""

lakesuperior/api/admin.py

@@ -1,8 +1,7 @@
 import logging
 
+from lakesuperior import env
 from lakesuperior.config_parser import parse_config
-from lakesuperior.env import env
-from lakesuperior.globals import AppGlobals
 from lakesuperior.migrator import Migrator
 from lakesuperior.store.ldp_nr.default_layout import DefaultLayout as FileLayout
 from lakesuperior.store.ldp_rs.lmdb_store import TxnManager
@@ -47,17 +46,12 @@ def migrate(src, dest, start_pts=None, list_file=None, **kwargs):
     return Migrator(src, dest, **kwargs).migrate(start_pts, list_file)
 
 
-def integrity_check(config_dir=None):
+def integrity_check():
     """
     Check integrity of the data set.
 
     At the moment this is limited to referential integrity. Other checks can
     be added and triggered by different argument flags.
     """
-    if config_dir:
-        env.config = parse_config(config_dir)[0]
-        env.app_globals = AppGlobals(env.config)
-    else:
-        import lakesuperior.env_setup
     with TxnManager(env.app_globals.rdfly.store):
-        return { t for t in env.app_globals.rdfly.find_refint_violations()}
+        return set(env.app_globals.rdfly.find_refint_violations())

lakesuperior/api/query.py

@@ -2,9 +2,9 @@
 
 from io import BytesIO
 
+from lakesuperior import env
 from lakesuperior.dictionaries.namespaces import ns_collection as nsc
 from lakesuperior.dictionaries.namespaces import ns_mgr as nsm
-from lakesuperior.env import env
 from lakesuperior.store.ldp_rs.lmdb_store import LmdbStore, TxnManager