Speedup GitHub

that_is_me_on_github/config.py

@@ -1 +1,13 @@
-VERSION = "v0.0.5"
+from enum import unique
+
+from lib.utils import StrEnum
+
+VERSION = "v0.0.4"
+
+
+@unique
+class TaskNameEnum(StrEnum):
+    PR = "pr"
+    ISSUE = "issue"
+    USER = "single_user"
+    OWNED_REPOS = "owned_repos"

that_is_me_on_github/lib/utils.py

@@ -6,6 +6,9 @@
 from github.Repository import Repository
 from github import Github
 
+from enum import Enum
+from concurrent.futures import ThreadPoolExecutor
+
 
 # build query by params
 def build_query(params: str) -> str:
@@ -20,10 +23,10 @@ def single_user(g: Github, username: str) -> NamedUser:
 # get repos owned by username
 def owned_repos(g: Github, username, is_public=True) -> List[Repository]:
     params = ["user:{}".format(username)]
-
+    
     if is_public:
         params.append("is:public")
-
+    
     return [
         repo for repo in g.search_repositories(build_query(params), "stars", "desc")
     ]
@@ -34,14 +37,14 @@ def commits(g: Github, username, is_public=True, orgs=[], repos=[]) -> List[Comm
     params = ["author:{}".format(username)]
     if is_public:
         params.append("is:public")
-
+    
     if orgs or repos:
         for org in orgs:
             params.append("org:{}".format(org))
-
+        
         for repo in repos:
             params.append("repo:{}".format(repo))
-
+    
     return [
         commit
         for commit in g.search_commits(build_query(params), "author-date", "desc")
@@ -50,28 +53,50 @@ def commits(g: Github, username, is_public=True, orgs=[], repos=[]) -> List[Comm
 
 # get issues or prs authored by username and filtered by certain repos and organizations
 def issues_and_prs(
-    g: Github, username: str, is_public=True, type="", orgs=[], repos=[]
+        g: Github, username: str, is_public=True, type="", orgs=[], repos=[]
 ) -> Dict[str, List[Issue]]:
     params = ["author:{}".format(username)]
-
+    
     if type:
         params.append("type:{}".format(type))
-
+    
     if is_public:
         params.append("is:public")
-
+    
     if orgs or repos:
         for org in orgs:
             params.append("org:{}".format(org))
-
+        
         for repo in repos:
             params.append("repo:{}".format(repo))
-
+    
     issues_and_prs = {}
     for issue_or_pr in g.search_issues(build_query(params), "updated", "desc"):
         if issue_or_pr.repository.name not in issues_and_prs:
             issues_and_prs[issue_or_pr.repository.name] = [issue_or_pr]
         else:
             issues_and_prs[issue_or_pr.repository.name].append(issue_or_pr)
-
+    
     return issues_and_prs
+
+
+class StrEnum(str, Enum):
+    def __new__(cls, *args):
+        for arg in args:
+            if not isinstance(arg, str):
+                raise TypeError('Not str: {}'.format(arg))
+        return super(StrEnum, cls).__new__(cls, *args)
+
+
+def handle_tasks(tasks):
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        futures = []
+        for task in tasks:
+            fn, args = task["func"], task["args"]
+            kwargs = task.get("kwargs", {})  # type: dict
+            name = task.get("name", fn.__name__)
+            future = executor.submit(fn, *args, **kwargs)
+            setattr(future, "name", name)
+            futures.append(future)
+        results = {future.name: future.result() for future in futures}
+        return results

that_is_me_on_github/main.py

@@ -5,6 +5,7 @@
 from lib.utils import *
 from urllib3.util.retry import Retry
 import os
+from config import TaskNameEnum
 
 
 @click.group()
@@ -67,15 +68,15 @@ def version():
     "--retry", default=1, type=int, help="Retry times for per request, default 1."
 )
 def generate(
-    username: str,
-    do_auth: bool,
-    auth_username: str,
-    auth_password: str,
-    org_filter: str,
-    repo_filter: str,
-    output: str,
-    timeout: int,
-    retry: int,
+        username: str,
+        do_auth: bool,
+        auth_username: str,
+        auth_password: str,
+        org_filter: str,
+        repo_filter: str,
+        output: str,
+        timeout: int,
+        retry: int,
 ):
     path = os.path.expanduser(output)
     try:
@@ -84,7 +85,7 @@ def generate(
     except IOError:
         click.echo("Error: output path not exist and not creatable.")
         raise click.Abort()
-
+    
     try:
         if do_auth:
             if not auth_username:
@@ -93,7 +94,7 @@ def generate(
                 auth_password = click.prompt(
                     "Your github password", type=str, hide_input=True
                 )
-
+            
             g = Github(
                 login_or_token=auth_username,
                 password=auth_password,
@@ -102,30 +103,60 @@ def generate(
             )
         else:
             g = Github(timeout=int(timeout), retry=Retry(retry))
-
-        user = single_user(g, username)
-        if not user:
-            click.echo("User {} Not Found.".format(username))
-            raise click.Abort()
-
+        
+        # user = single_user(g, username)
+        # if not user:
+        #     click.echo("User {} Not Found.".format(username))
+        #     raise click.Abort()
+        
         click.echo("Please wait for a few seconds.")
-
+        
         org_filter = (
             [item.strip() for item in org_filter.split(",")] if org_filter else []
         )
         repo_filter = (
             [item.strip() for item in repo_filter.split(",")] if repo_filter else []
         )
-
+
+        container = [
+            {"name": TaskNameEnum.OWNED_REPOS,
+             "func": owned_repos,
+             "args": [g, username]},
+
+            {"name": TaskNameEnum.ISSUE,
+             "func": issues_and_prs,
+             "args": [g, username],
+             "kwargs": {'type': "issue",
+                        'orgs': org_filter,
+                        'repos': repo_filter}},
+
+            {"name": TaskNameEnum.USER,
+             "func": single_user,
+             "args": [g, username]},
+
+            {"name": TaskNameEnum.PR,
+             "func": issues_and_prs,
+             "args": [g, username],
+             "kwargs": {'type': "pr",
+                        'orgs': org_filter,
+                        'repos': repo_filter}},
+        ]
+
+        results = handle_tasks(container)
+        user_info, repo_info, pr_info, issue_info = results[TaskNameEnum.USER], results[TaskNameEnum.OWNED_REPOS], \
+                                                    results[TaskNameEnum.PR], results[TaskNameEnum.ISSUE]
+        if not user_info:
+            click.echo("User {} Not Found.".format(username))
+            raise click.Abort()
+
         Render().render(
-            user,
-            owned_repos(g, username),
-            issues_and_prs(g, username, type="pr", orgs=org_filter, repos=repo_filter),
-            issues_and_prs(
-                g, username, type="issue", orgs=org_filter, repos=repo_filter
-            ),
+            user_info,
+            repo_info,
+            pr_info,
+            issue_info,
             path,
         )
+
     except RateLimitExceededException:
         click.echo(
             "Github rate limit reached, Please provide username, password or api_token (not support yet), and try again"