Merge branch 'release/2.3'

Author: kamidev

manage/fabfile/__init__.py

@@ -19,6 +19,8 @@
 
 env.toolsdir = sep.join((env.projectroot, 'tools'))
 
+env.resources = sep.join((env.projectroot, 'resources'))
+
 env.builddir = sep.join((env.projectroot, '_build'))
 
 env.docbuild = sep.join((env.builddir, 'documentation'))
@@ -54,6 +56,7 @@
     'test': [],
     'regression': [],
     'skrapat': [],
+    'collectreg': [],
 }
 
 ##
@@ -69,4 +72,5 @@
 import app.checker
 import app.docs
 import app.tools
-import app.demodata
+import app.demodata
+import app.tasks

manage/fabfile/app/admin.py

@@ -3,7 +3,7 @@
 from fabric.api import *
 from fabric.contrib.files import exists
 from fabric.contrib.project import rsync_project
-from fabfile.util import slashed, cygpath, exit_on_error
+from fabfile.util import slashed, cygpath, exit_on_error, role_is_active
 from fabfile.target import _needs_targetenv
 from fabfile.util import venv, fullpath
 from fabfile.server import restart_apache
@@ -63,6 +63,15 @@ def all(source=None):
     deploy()
 
 
+@task
+@roles('admin')
+def serve_local_folder(source=None, port="8280"):
+    """Package and serve the admin feed at selected port on localhost."""
+    package(source=source)
+    with lcd("%(toolsdir)s/rinfomain" % env):
+        local("groovy serve_folder.groovy %s/_build/%s/rinfo-admin %s" % (env.projectroot, env.target, port) )
+
+
 @task
 @roles('admin')
 def test():
@@ -99,6 +108,8 @@ def ping_main():
 @task
 @roles('admin')
 def test_all():
+    if not role_is_active('admin'):
+        return
     all()
     restart_apache()
     restart_tomcat()

manage/fabfile/app/checker.py

@@ -1,7 +1,7 @@
 import sys
 from fabric.api import *
 from fabric.contrib.files import exists
-from fabfile.util import venv, exit_on_error
+from fabfile.util import venv, exit_on_error, role_is_active
 from fabfile.app import local_lib_rinfo_pkg, _deploy_war_norestart
 from fabfile.target import _needs_targetenv
 from fabfile.server import restart_apache
@@ -76,7 +76,9 @@ def clean():
 @task
 @roles('admin')
 def test_all():
-    all(deps="0", test="0")
+    if not role_is_active('service'):
+        return
+    all(deps="1", test="0")
     restart_tomcat()
     restart_apache()
     msg_sleep(20, "restart and tomcat apache")

manage/fabfile/app/main.py

@@ -1,7 +1,7 @@
 import sys
 from fabric.api import *
 from fabric.contrib.files import exists
-from fabfile.util import venv, exit_on_error
+from fabfile.util import venv, exit_on_error, role_is_active
 from fabfile.app import local_lib_rinfo_pkg
 from fabfile.app import _deploy_war_norestart
 from fabfile.target import _needs_targetenv
@@ -49,6 +49,7 @@ def setup():
         sudo("mkdir %(target_config_dir)s" % env)
     put("%(java_packages)s/rinfo-main/src/environments/%(target)s/rinfo-main.properties"  % env,"%(target_config_dir)srinfo-main.properties"  % env, use_sudo=True)
 
+
 @task
 @roles('main')
 @exit_on_error
@@ -94,28 +95,102 @@ def test():
         print "Created report '%s'" % file_name
 
 
+@task
+@roles('main')
+def count_rdfs_in_publ_in_depot(type=""):
+    with cd("%s/depot/publ/%s" % (env.rinfo_main_store, type)):
+        return run("find | grep .rdf | wc -l")
+
+
+@task
+@roles('main')
+def get_rdf_list_in_depot(type=""):
+    _needs_targetenv()
+    file_name = 'rdf_list_%s.txt' % env.target
+    remote_file_and_path = '/tmp/%s' % file_name
+    with cd("%s/depot/publ/%s" % (env.rinfo_main_store, type)):
+        run("find | grep .rdf > %s" % remote_file_and_path)
+    get(remote_file_and_path, '/tmp/%s' % file_name)
+    local('sort /tmp/%s > /tmp/sorted_%s' % (file_name, file_name))
+
+
+@task
+@roles('main')
+def verify_depot_minimum_size():
+    EXPECTED_SFS_DOCUMENT_COUNT = 58268
+    EXPECTED_VA_DOCUMENT_COUNT = 24841
+    sfs_document_count = int(count_rdfs_in_publ_in_depot('sfs'))
+    if sfs_document_count < EXPECTED_SFS_DOCUMENT_COUNT:
+        raise Exception("SFS Document count verification failed! Expected %s documents, but found %s! Aborting..." %
+                        (EXPECTED_SFS_DOCUMENT_COUNT,sfs_document_count))
+    va_document_count = int(count_rdfs_in_publ_in_depot('dom')) +  int(count_rdfs_in_publ_in_depot('rf'))
+    if va_document_count < EXPECTED_VA_DOCUMENT_COUNT:
+        raise Exception("VA Document count verification failed! Expected %s documents, but found %s! Aborting..." %
+                        (EXPECTED_VA_DOCUMENT_COUNT,va_document_count))
+    print "Found %s SFS documents" % sfs_document_count
+    print "Found %s VA documents" % va_document_count
+
+
 @task
 @roles('main')
 def ping_start_collect_admin():
     _needs_targetenv()
     feed_url = ''
     if env.target=='regression':
         feed_url = "http://%s/feed/current.atom" % env.roledefs['demosource'][0]
+    elif env.target=='dev_unix':
+        feed_url = "http://%s:8280/feed/current.atom" % env.roledefs['admin'][0]
     else:
         feed_url = "http://%s/feed/current" % env.roledefs['admin'][0]
-    collector_url = "http://%s/collector" % env.roledefs['main'][0]
+    main_host_and_port = env.roledefs['main'][0] if env.target!='dev_unix' else "%s:8180" % env.roledefs['main'][0]
+    collector_url = "http://%s/collector" % main_host_and_port
     if not verify_url_content(" --data 'feed=%(feed_url)s' %(collector_url)s" % vars(), "Scheduled collect of"):
         raise Exception("Test failed")
 
 
 @task
 @roles('main')
-def ping_start_collect_feed():
+def ping_start_collect_feed(default_feed=None):
     _needs_targetenv()
-    feed_url = "http://%s/feed/current.atom" % env.roledefs['demosource'][0]
-    collector_url = "http://%s/collector" % env.roledefs['main'][0]
-    if not verify_url_content(" --data 'feed=%(feed_url)s' %(collector_url)s" % vars(), "Scheduled collect of"):
-        raise Exception("Test failed")
+    if default_feed:
+        if not verify_url_content(" --data 'feed=%(default_feed)s' %(collector_url)s" % vars(),
+                                  "Scheduled collect of",
+                                  alternate_string_exists_in_content="is already scheduled for collect"):
+            print "Failed to start collect of '%s'" % default_feed
+        return
+    main_host_and_port = env.roledefs['main'][0] if env.target!='dev_unix' else "%s:8180" % env.roledefs['main'][0]
+    collector_url = "http://%s/collector" % main_host_and_port
+    if env.target=='regression':
+        feed_url = "http://%s/feed/current.atom" % env.roledefs['demosource'][0]
+        if not verify_url_content(" --data 'feed=%(feed_url)s' %(collector_url)s" % vars(), "Scheduled collect of"):
+            raise Exception("Scheduled collect failed")
+    else:
+        filename = "%(resources)s/%(target)s/datasources.n3" % venv()
+        read_file = open(filename, 'r')
+        for line in read_file:
+            line = line.strip()
+            if line.startswith('iana:current'):
+                start_index = line.index('<') + 1
+                end_index = line.index('>')
+                feed_url = line[start_index:end_index].strip()
+                if feed_url=='http://rinfo.lagrummet.se/feed/current':
+                    continue
+                if not verify_url_content(" --data 'feed=%(feed_url)s' %(collector_url)s" % vars(),
+                                          "Scheduled collect of",
+                                          alternate_string_exists_in_content="is already scheduled for collect"):
+                    print "Failed to start collect of '%s'" % feed_url
+
+
+@task
+@roles('main')
+def destroy_main_data(start_top_tomcat=True):
+    if not role_is_active('main'):
+        return
+    if start_top_tomcat:
+        tomcat_stop()
+    sudo("rm -rf %(rinfo_main_store)s/*" % venv())
+    if start_top_tomcat:
+        tomcat_start()
 
 
 @task
@@ -148,7 +223,9 @@ def install_regression_data(restart_tomcat=True):
 @task
 @roles('main')
 def test_all():
-    all(deps="0", test="0")
+    if not role_is_active('service'):
+        return
+    all(deps="1", test="0")
     restart_apache()
     #if env.target=='regression':
     #    install_regression_data()

manage/fabfile/app/service.py

@@ -2,7 +2,7 @@
 import sys
 from fabric.api import *
 from fabric.contrib.files import exists
-from fabfile.util import venv, mkdirpath, exit_on_error
+from fabfile.util import venv, mkdirpath, exit_on_error, role_is_active
 from fabfile import app, sysconf
 from fabfile.target import _needs_targetenv
 from fabfile.app import _deploy_war_norestart
@@ -92,6 +92,13 @@ def deploy_sesame():
         app._deploy_war("%(local_sesame_dir)s/%(warname)s.war" % venv(), warname)
 
 
+@task
+@roles('service')
+def destroy_service_repository(default_repository='rinfo'):
+    _needs_targetenv()
+    run("curl -X POST http://localhost:8080/sesame-workbench/repositories/%s/clear" % default_repository)
+
+
 def _patch_catalina_properties():
     # This will patch catalina.properties so that it contains the system 
     # property that controls where sesame stores its data
@@ -197,6 +204,13 @@ def start_elasticsearch():
     sudo("/etc/init.d/elasticsearch start")
 
 
+@task
+@roles('service')
+def delete_elasticsearch_index(index_name='rinfo'):
+    _needs_targetenv()
+    run("curl -XDELETE 'http://localhost:9200/%s/'" % index_name)
+
+
 ##
 # Varnish install and setup
 
@@ -265,6 +279,16 @@ def ping_start_collect():
         raise Exception("Test failed")
 
 
+@task
+@roles('service')
+def destroy_service_data():
+    if not role_is_active('service'):
+        return
+    destroy_service_repository()
+    delete_elasticsearch_index()
+    ban_varnish()
+
+
 @task
 @roles('service')
 def clean():
@@ -281,7 +305,9 @@ def clean():
 @task
 @roles('service')
 def test_all():
-    all(deps="0", test="0")
+    if not role_is_active('service'):
+        return
+    all(deps="1", test="0")
     restart_apache()
     restart_tomcat()
     msg_sleep(20, "restart apache, tomcat and wait for service to start")

manage/fabfile/app/tasks.py

@@ -0,0 +1,19 @@
+"""
+Specific tasks spanning over multiple apps
+"""
+from fabric.api import *
+from fabfile.app.main import destroy_main_data
+from fabfile.app.service import destroy_service_data
+from fabfile.server import tomcat_stop, tomcat_start
+
+
+@task
+@roles('main','service')
+def destroy_all_data(start_top_tomcat=True):
+    destroy_main_data(start_top_tomcat=False)
+    if start_top_tomcat:
+        tomcat_stop()
+    destroy_main_data(start_top_tomcat=False)
+    destroy_service_data(start_top_tomcat=False)
+    if start_top_tomcat:
+        tomcat_start()

manage/fabfile/app/tools.py

@@ -21,4 +21,6 @@ def ping_service_collector():
     #require('roledefs', provided_by=target)
     collector_url = "http://%s/collector" % env.roledefs['service'][0]
     feed_url = "http://%s/feed/current" % env.roledefs['main'][0]
-    ping_collector(collector_url, feed_url)
+    ping_collector(collector_url, feed_url)
+
+

manage/fabfile/server.py

@@ -200,16 +200,33 @@ def ftp_fetch(filename, ftp_address, target_path, username, password, test=False
 
 
 
-def tar_and_ftp_push(snapshot_name, name, password, source_tar_path, target_path, username, test=False):
+def tar_and_ftp_push(snapshot_name, name, password, source_tar_path, target_path, username, test=False, md5sum=True):
     file_to_upload = '%s/%s.tar.gz' % (target_path, name)
     tar(file_to_upload, source_tar_path, test=test)
     ftp_push(file_to_upload, '%s/%s/%s.tar.gz' % (env.ftp_server_url, snapshot_name, name), username, password,
              test=test)
+    if md5sum:
+        md5_file = '%s/%s.md5' % (target_path, name)
+        run("md5sum %s > %s" % (file_to_upload, md5_file) )
+        ftp_push(md5_file, '%s/%s/%s.md5' % (env.ftp_server_url, snapshot_name, name), username, password,
+                 test=test)
 
 
-def ftp_fetch_and_untar(snapshot_name, name, tmp_path, target_tar_unpack_path, username, password, test=False, is_local=False):
+
+def ftp_fetch_and_untar(snapshot_name, name, tmp_path, target_tar_unpack_path, username, password, test=False, is_local=False, md5sum=True):
     file_to_download = '%s.tar.gz' % name
     ftp_fetch(file_to_download, "%s/%s" % (env.ftp_server_url, snapshot_name), tmp_path, username, password, test=test, is_local=is_local)
+    if md5sum:
+        md5_file = '%s.md5' % name
+        ignore = False
+        try:
+            ftp_fetch(md5_file, "%s/%s" % (env.ftp_server_url, snapshot_name), tmp_path, username, password, test=test, is_local=is_local)
+        except: # no md5 file -> ignore
+            ignore = True
+        if not ignore:
+            with cd(tmp_path):
+                run('md5sum -c %s' % md5_file)
+
     clean_path(target_tar_unpack_path, use_sudo=True, test=test, is_local=is_local)
     untar('%s/%s.tar.gz' % (tmp_path, name), target_tar_unpack_path, use_sudo=True, test=test, is_local=is_local)
 
@@ -427,9 +444,11 @@ def prepare_sudo_for_debian_and_add_rinfo_user():
 
 
 @task
-@roles('main', 'service', 'checker', 'admin', 'lagrummet', 'emfs', 'test', 'regression', 'skrapat', 'demosource')
-def bootstrap():
+@roles('main', 'service', 'checker', 'admin', 'lagrummet', 'emfs', 'test', 'regression', 'skrapat', 'demosource', 'collectreg')
+def bootstrap(role=None):
     _needs_targetenv()
+    if role and not role_is_active(role):
+        return
     #if not os_version() == 'Debian7':
     #    print 'Unsupported os version %%' % os_version()
     #    return