From d8ffe1ac8c0112a3f5f65308f07a11a1f83a0ee6 Mon Sep 17 00:00:00 2001 From: Matus Kosut Date: Mon, 3 Feb 2020 09:14:30 +0100 Subject: [PATCH] improve test handling in existing brokers --- docs/source/test_writing/quickstart.rst | 4 +-- jujuna/brokers/network.py | 7 ++--- jujuna/brokers/process.py | 5 ++-- jujuna/brokers/service.py | 14 ++++++---- jujuna/tests.py | 34 ++++++++++++++++++++----- 5 files changed, 45 insertions(+), 19 deletions(-) diff --git a/docs/source/test_writing/quickstart.rst b/docs/source/test_writing/quickstart.rst index 5eeaa8b..1b219e3 100644 --- a/docs/source/test_writing/quickstart.rst +++ b/docs/source/test_writing/quickstart.rst @@ -13,10 +13,10 @@ Example 1 - Bundle of glance and openstack:: glance-registry: status: 'running' process: - - 'glance-api' + glance-api: True network: port: - - 9292 + '9292': True mysql-db: service: mysql: diff --git a/jujuna/brokers/network.py b/jujuna/brokers/network.py index 33a7716..689a719 100644 --- a/jujuna/brokers/network.py +++ b/jujuna/brokers/network.py @@ -27,9 +27,10 @@ async def run(self, test_case, unit, idx): results = {'sockets': []} # print(results) - local_ports = [s['local_port'] for s in results['sockets']] + local_ports = [str(s['local_port']) for s in results['sockets']] - for port in test_case['port']: - rows.append((idx, '{}.{} == {}'.format('port', port, 'open'), port in local_ports), ) + for port, open in test_case['port'].items(): + status = 'open' if open else 'closed' + rows.append((idx, '{}.{} == {}'.format('port', port, status), (str(port) in local_ports) == open), ) return rows diff --git a/jujuna/brokers/process.py b/jujuna/brokers/process.py index 50e4ce1..dc33424 100644 --- a/jujuna/brokers/process.py +++ b/jujuna/brokers/process.py @@ -25,7 +25,8 @@ async def run(self, test_case, unit, idx): log.debug(exc) results = {} # print(results) - for condition in test_case: - rows.append((idx, '{} == present'.format(condition), condition in results), ) + for condition, present in test_case.items(): + status = 'present' if present else 'absent' + rows.append((idx, '{} == {}'.format(condition, status), (condition in results) == present), ) return rows diff --git a/jujuna/brokers/service.py b/jujuna/brokers/service.py index 43fc8fe..aae2074 100644 --- a/jujuna/brokers/service.py +++ b/jujuna/brokers/service.py @@ -27,11 +27,15 @@ async def run(self, test_case, unit, idx): # print(results['services']) # print(test_case) for service, condition in test_case.items(): - rows.append((idx, '{} == {}'.format(service, 'exists'), service in results['services']), ) for c, v in condition.items(): - rows.append(( - idx, '{}.{} == {}'.format(service, c, v), - service in results['services'] and results['services'][service][c] == v - ), ) + if c == 'exists': + rows.append(( + idx, '{}.{} == {}'.format(service, c, v), (service in results['services']) == v + ), ) + else: + rows.append(( + idx, '{}.{} == {}'.format(service, c, v), + service in results['services'] and results['services'][service][c] == v + ), ) return rows diff --git a/jujuna/tests.py b/jujuna/tests.py index 7a89b12..f48a883 100644 --- a/jujuna/tests.py +++ b/jujuna/tests.py @@ -46,12 +46,17 @@ async def test( :param password: string :param cacert: string """ + log.info('Load tests') if test_suite: with open(test_suite.name, 'r') as stream: try: suite = yaml.full_load(stream) + suite_apps = suite.keys() except yaml.YAMLError as exc: log.error(exc) + suite_apps = [] + + log.info('Applications: {}'.format(', '.join(suite_apps))) controller, model = await connect_juju( ctrl_name, @@ -69,7 +74,15 @@ async def test( for app_name, app in model.applications.items(): if suite and app_name in suite: app_passed, app_failed = 0, 0 - log.info('[{}]: {} {} [{}]'.format(app_name, app.status, app.alive, len(app.units))) + try: + test_cases = len([ + key for x in suite[app_name].values() for y in x.values() if isinstance(x, dict) for key in y + ]) + except Exception: + test_cases = 0 + log.info('{} - {} - {} units - {} tests - {} {}'.format( + "----", app_name, len(app.units), test_cases, app.status, app.alive + )) for idx, unit in enumerate(app.units): async with async_timeout.timeout(60): passed, failed = await execute_brokers(suite[app_name], unit, idx) @@ -79,7 +92,8 @@ async def test( failed_units.add(unit.name) model_passed += app_passed model_failed += app_failed - log.info('[{}]: Passed: {} Failed: {}'.format(app_name, app_passed, app_failed)) + log.info('{} - {}: Passed tests: {}'.format("====", app_name, app_passed)) + log.info('{} - {}: Failed tests: {}'.format("====", app_name, app_failed)) alive = defaultdict(int) status = defaultdict(int) @@ -93,9 +107,15 @@ async def test( ): log.info('All juju apps state to be alive and active.') else: - log.warning('Finished with errors: Alive: {} Status: {}'.format(dict(alive), dict(status))) + log.warning('Finished with errors') + log.warning('Alive: {}'.format(dict(alive))) + log.warning('Status: {}'.format(dict(status))) - log.info('[FINISHED] Passed tests: {} Failed tests: {}'.format(model_passed, model_failed)) + log.info('{}: {}'.format("Passed tests", model_passed)) + if model_failed: + log.warning('{}: {}'.format("Failed tests", model_failed)) + else: + log.info('{}: {}'.format("Failed tests", model_failed)) except JujuError as e: log.error('JujuError during tests') @@ -147,14 +167,14 @@ async def execute_brokers(app_test_suite, unit, idx): for test_case in app_test_suite.keys(): if test_case in broker_map.keys(): - log.info('[{}]: {}'.format(unit.entity_id, test_case)) + # log.info('{} - {}: {}'.format("unit", unit.entity_id, test_case)) rows = await broker_map[test_case]().run(app_test_suite[test_case], unit, idx) for row in rows: if row[2]: - log.info('[{}]: {} {} [{}]'.format(unit.entity_id, test_case, row[1], "Pass")) + log.info('{} - {}: {} {}'.format("PASS", unit.entity_id, test_case, row[1])) passed += 1 else: - log.warning('[{}]: {} {} [{}]'.format(unit.entity_id, test_case, row[1], "Fail")) + log.info('{} - {}: {} {}'.format("FAIL", unit.entity_id, test_case, row[1])) failed += 1 else: log.warning("TEST: Skipped (Broker '{}' not registered)".format(test_case))