improve test handling in existing brokers

docs/source/test_writing/quickstart.rst

@@ -13,10 +13,10 @@ Example 1 - Bundle of glance and openstack::
         glance-registry:
           status: 'running'
       process:
-        - 'glance-api'
+        glance-api: True
       network:
         port:
-          - 9292
+          '9292': True
     mysql-db:
       service:
         mysql:

jujuna/brokers/network.py

@@ -27,9 +27,10 @@ async def run(self, test_case, unit, idx):
                     results = {'sockets': []}
                 # print(results)
 
-                local_ports = [s['local_port'] for s in results['sockets']]
+                local_ports = [str(s['local_port']) for s in results['sockets']]
 
-                for port in test_case['port']:
-                    rows.append((idx, '{}.{} == {}'.format('port', port, 'open'), port in local_ports), )
+                for port, open in test_case['port'].items():
+                    status = 'open' if open else 'closed'
+                    rows.append((idx, '{}.{} == {}'.format('port', port, status), (str(port) in local_ports) == open), )
 
         return rows

jujuna/brokers/process.py

@@ -25,7 +25,8 @@ async def run(self, test_case, unit, idx):
                 log.debug(exc)
                 results = {}
             # print(results)
-            for condition in test_case:
-                rows.append((idx, '{} == present'.format(condition), condition in results), )
+            for condition, present in test_case.items():
+                status = 'present' if present else 'absent'
+                rows.append((idx, '{} == {}'.format(condition, status), (condition in results) == present), )
 
         return rows

jujuna/brokers/service.py

@@ -27,11 +27,15 @@ async def run(self, test_case, unit, idx):
             # print(results['services'])
             # print(test_case)
             for service, condition in test_case.items():
-                rows.append((idx, '{} == {}'.format(service, 'exists'), service in results['services']), )
                 for c, v in condition.items():
-                    rows.append((
-                        idx, '{}.{} == {}'.format(service, c, v),
-                        service in results['services'] and results['services'][service][c] == v
-                    ), )
+                    if c == 'exists':
+                        rows.append((
+                            idx, '{}.{} == {}'.format(service, c, v), (service in results['services']) == v
+                        ), )
+                    else:
+                        rows.append((
+                            idx, '{}.{} == {}'.format(service, c, v),
+                            service in results['services'] and results['services'][service][c] == v
+                        ), )
 
         return rows

jujuna/tests.py

@@ -46,12 +46,17 @@ async def test(
     :param password: string
     :param cacert: string
     """
+    log.info('Load tests')
     if test_suite:
         with open(test_suite.name, 'r') as stream:
             try:
                 suite = yaml.full_load(stream)
+                suite_apps = suite.keys()
             except yaml.YAMLError as exc:
                 log.error(exc)
+                suite_apps = []
+
+    log.info('Applications: {}'.format(', '.join(suite_apps)))
 
     controller, model = await connect_juju(
         ctrl_name,
@@ -69,7 +74,15 @@ async def test(
         for app_name, app in model.applications.items():
             if suite and app_name in suite:
                 app_passed, app_failed = 0, 0
-                log.info('[{}]: {} {} [{}]'.format(app_name, app.status, app.alive, len(app.units)))
+                try:
+                    test_cases = len([
+                        key for x in suite[app_name].values() for y in x.values() if isinstance(x, dict) for key in y
+                    ])
+                except Exception:
+                    test_cases = 0
+                log.info('{} - {} - {} units - {} tests - {} {}'.format(
+                    "----", app_name, len(app.units), test_cases, app.status, app.alive
+                ))
                 for idx, unit in enumerate(app.units):
                     async with async_timeout.timeout(60):
                         passed, failed = await execute_brokers(suite[app_name], unit, idx)
@@ -79,7 +92,8 @@ async def test(
                             failed_units.add(unit.name)
                 model_passed += app_passed
                 model_failed += app_failed
-                log.info('[{}]: Passed: {} Failed: {}'.format(app_name, app_passed, app_failed))
+                log.info('{} - {}: Passed tests: {}'.format("====", app_name, app_passed))
+                log.info('{} - {}: Failed tests: {}'.format("====", app_name, app_failed))
 
         alive = defaultdict(int)
         status = defaultdict(int)
@@ -93,9 +107,15 @@ async def test(
         ):
             log.info('All juju apps state to be alive and active.')
         else:
-            log.warning('Finished with errors: Alive: {} Status: {}'.format(dict(alive), dict(status)))
+            log.warning('Finished with errors')
+            log.warning('Alive: {}'.format(dict(alive)))
+            log.warning('Status: {}'.format(dict(status)))
 
-        log.info('[FINISHED] Passed tests: {} Failed tests: {}'.format(model_passed, model_failed))
+        log.info('{}: {}'.format("Passed tests", model_passed))
+        if model_failed:
+            log.warning('{}: {}'.format("Failed tests", model_failed))
+        else:
+            log.info('{}: {}'.format("Failed tests", model_failed))
 
     except JujuError as e:
         log.error('JujuError during tests')
@@ -147,14 +167,14 @@ async def execute_brokers(app_test_suite, unit, idx):
 
     for test_case in app_test_suite.keys():
         if test_case in broker_map.keys():
-            log.info('[{}]: {}'.format(unit.entity_id, test_case))
+            # log.info('{} - {}: {}'.format("unit", unit.entity_id, test_case))
             rows = await broker_map[test_case]().run(app_test_suite[test_case], unit, idx)
             for row in rows:
                 if row[2]:
-                    log.info('[{}]: {} {} [{}]'.format(unit.entity_id, test_case, row[1], "Pass"))
+                    log.info('{} - {}: {} {}'.format("PASS", unit.entity_id, test_case, row[1]))
                     passed += 1
                 else:
-                    log.warning('[{}]: {} {} [{}]'.format(unit.entity_id, test_case, row[1], "Fail"))
+                    log.info('{} - {}: {} {}'.format("FAIL", unit.entity_id, test_case, row[1]))
                     failed += 1
         else:
             log.warning("TEST: Skipped (Broker '{}' not registered)".format(test_case))