From ca14122e8749914a08c4c5b8b70d1ea9e7586e43 Mon Sep 17 00:00:00 2001 From: "David M. Rogers" Date: Thu, 30 Mar 2023 15:18:02 -0400 Subject: [PATCH 1/2] Added async runner in terminal mode. --- src/reposcanner/manager.py | 43 +++++++++++++++++++++++++------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/src/reposcanner/manager.py b/src/reposcanner/manager.py index ea108b4..95feda4 100644 --- a/src/reposcanner/manager.py +++ b/src/reposcanner/manager.py @@ -12,6 +12,7 @@ from tqdm import tqdm # For progress checking in non-GUI mode. from abc import ABC, abstractmethod +import asyncio class TaskFactory: def createManagerRepositoryRoutineTask(self, projectID, projectName, url, request): @@ -45,7 +46,7 @@ def hasResponse(self): def getResponse(self): return self._response - def process(self, agents, store, notebook): + async def process(self, agents, store, notebook): """ Scan through a set of available routines or analyses and see if any can execute the request held by this task. If no routines or analyses can handle @@ -381,20 +382,34 @@ def executeWithNoGUI(self): """ Plain-text execution mode. """ - for task in tqdm(self._tasks): - task.process( - self._repositoryRoutines + - self._externalCommandLineToolRoutines + - self._analyses, - self._store, - self._notebook) - response = task.getResponse() - print(task.getResponseDescription()) - if not task.getResponse().wasSuccessful(): + + sem = asyncio.Semaphore(6) + + async def do_task(task): # -> List[str] + async with sem: # semaphore limits num of simultaneous tasks + await task.process( + self._repositoryRoutines + + self._externalCommandLineToolRoutines + + self._analyses, + self._store, + self._notebook) + return task + + async def exec_loop(): + tasks = [] + for task in self._tasks: + tasks.append(asyncio.create_task( do_task(task) )) + for c in tqdm(asyncio.as_completed(tasks)): + task = await c + response = [task.getResponse()] + print( task.getResponseDescription() ) + if not task.getResponse().wasSuccessful(): + for attachment in response.getAttachments(): + print(attachment) for attachment in response.getAttachments(): - print(attachment) - for attachment in response.getAttachments(): - self._store.insert(attachment) + self._store.insert(attachment) + + asyncio.run(exec_loop()) def executeWithGUI(self): """ From 11913658fe3623e441a831f4b4cbd1d2e38f986a Mon Sep 17 00:00:00 2001 From: "David M. Rogers" Date: Thu, 30 Mar 2023 15:28:10 -0400 Subject: [PATCH 2/2] Fixing response listification error. --- src/reposcanner/manager.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/src/reposcanner/manager.py b/src/reposcanner/manager.py index 95feda4..30501df 100644 --- a/src/reposcanner/manager.py +++ b/src/reposcanner/manager.py @@ -401,7 +401,7 @@ async def exec_loop(): tasks.append(asyncio.create_task( do_task(task) )) for c in tqdm(asyncio.as_completed(tasks)): task = await c - response = [task.getResponse()] + response = task.getResponse() print( task.getResponseDescription() ) if not task.getResponse().wasSuccessful(): for attachment in response.getAttachments(): @@ -525,11 +525,13 @@ def centerTextPosition(text, windowWidth): footer.addstr(1, 4, taskDescription, curses.A_BOLD) footer.border(2) footer.refresh() - currentTask.process( - self._repositoryRoutines + - self._analyses, - self._store, - self._notebook) + asyncio.run( + currentTask.process( + self._repositoryRoutines + + self._analyses, + self._store, + self._notebook) + ) for attachment in currentTask.getResponse().getAttachments(): self._store.insert(attachment)