Refactor codebase to asyncio

Use asyncio to perform concurrent http requests or file operations
wherever possible to reduce latency of batched operations.

Replaces all threaded operations.

Author: deathaxe

package_control/automatic_upgrader.py

@@ -11,6 +11,7 @@
 from .activity_indicator import ActivityIndicator
 from .console_write import console_write
 from .package_tasks import PackageTaskRunner
+from .vendor import anyio
 
 
 class AutomaticUpgrader:
@@ -28,8 +29,8 @@ def __init__(self, manager):
         self.next_run = 0
         self.current_version = int(sublime.version())
 
-    def run(self):
-        self.load_last_run()
+    async def run(self):
+        await self.load_last_run()
 
         if self.last_version != self.current_version and self.last_version != 0:
             console_write(
@@ -50,16 +51,16 @@ def run(self):
             )
             return
 
-        self.upgrade_packages()
+        await self.upgrade_packages()
 
-    def load_last_run(self):
+    async def load_last_run(self):
         """
         Loads the last run time from disk into memory
         """
 
         try:
-            with open(os.path.join(sys_path.pc_cache_dir(), 'last_run.json')) as fobj:
-                last_run_data = json.load(fobj)
+            async with await anyio.open_file(os.path.join(sys_path.pc_cache_dir(), 'last_run.json')) as fobj:
+                last_run_data = json.loads(await fobj.read())
             self.last_run = int(last_run_data['timestamp'])
             self.last_version = int(last_run_data['st_version'])
         except (FileNotFoundError, ValueError, TypeError):
@@ -69,18 +70,18 @@ def load_last_run(self):
         if frequency and self.last_run:
             self.next_run = int(self.last_run) + (frequency * 60 * 60)
 
-    def save_last_run(self):
+    async def save_last_run(self):
         """
         Saves a record of when the last run was
         """
 
-        with open(os.path.join(sys_path.pc_cache_dir(), 'last_run.json'), 'w') as fobj:
-            json.dump({
+        async with await anyio.open_file(os.path.join(sys_path.pc_cache_dir(), 'last_run.json'), 'w') as fobj:
+            await fobj.write(json.dumps({
                 'timestamp': int(time.time()),
                 'st_version': self.current_version
-            }, fp=fobj)
+            }))
 
-    def upgrade_packages(self):
+    async def upgrade_packages(self):
         """
         Upgrades all packages that are not currently upgraded to the latest
         version. Also renames any installed packages to their new names.
@@ -92,17 +93,17 @@ def upgrade_packages(self):
             # upgrade existing libraries
             required_libraries = upgrader.manager.find_required_libraries()
             missing_libraries = upgrader.manager.find_missing_libraries(required_libraries=required_libraries)
-            upgrader.manager.install_libraries(
+            await upgrader.manager.install_libraries(
                 libraries=required_libraries - missing_libraries,
                 fail_early=False
             )
 
             # run updater synchronously to delay any "You must restart ST" dialogues
             # Note: we are in PackageCleanup thread here
-            completed = upgrader.upgrade_packages(
+            completed = await upgrader.upgrade_packages(
                 ignore_packages=upgrader.manager.settings.get('auto_upgrade_ignore'),
                 unattended=True,
                 progress=progress
             )
             if completed:
-                self.save_last_run()
+                await self.save_last_run()

package_control/bootstrap.py

@@ -1,10 +1,11 @@
+import asyncio
 import json
 import os
 import zipfile
 from textwrap import dedent
-from threading import Thread
 
 import sublime
+import sublime_aio
 
 from . import library, sys_path
 from .clear_directory import delete_directory
@@ -37,7 +38,7 @@ def disable_package_control():
     )
 
 
-def bootstrap():
+async def bootstrap():
     """
     Bootstrap Package Control
 
@@ -51,19 +52,17 @@ def bootstrap():
     if not os.path.exists(LOADER_PACKAGE_PATH):
         # Start shortly after Sublime starts so package renames don't cause errors
         # with key bindings, settings, etc. disappearing in the middle of parsing
-        sublime.set_timeout(PackageCleanup().start, 2000)
+        await asyncio.sleep(2)
+        await PackageCleanup().run()
         return
 
-    sublime.set_timeout(_bootstrap, 10)
-
-
-def _bootstrap():
     PackageDisabler.disable_packages({PackageDisabler.LOADER: LOADER_PACKAGE_NAME})
     # Give ST a second to disable 0_package_control_loader
-    sublime.set_timeout(Thread(target=_migrate_dependencies).start, 1000)
+    await asyncio.sleep(1)
+    await _migrate_dependencies()
 
 
-def _migrate_dependencies():
+async def _migrate_dependencies():
     """
     Moves old Package Control 3-style dependencies to the new 4-style
     libraries, which use the Lib folder
@@ -110,17 +109,16 @@ def _migrate_dependencies():
 
         os.remove(LOADER_PACKAGE_PATH)
 
-        def _reenable_loader():
-            PackageDisabler.reenable_packages({PackageDisabler.LOADER: LOADER_PACKAGE_NAME})
-            show_message(
-                '''
-                Dependencies have just been migrated to python libraries.
+        await asyncio.sleep(500)
 
-                You may need to restart Sublime Text.
-                '''
-            )
+        PackageDisabler.reenable_packages({PackageDisabler.LOADER: LOADER_PACKAGE_NAME})
+        show_message(
+            '''
+            Dependencies have just been migrated to python libraries.
 
-        sublime.set_timeout(_reenable_loader, 500)
+            You may need to restart Sublime Text.
+            '''
+        )
 
     except (OSError) as e:
         console_write('Error trying to migrate dependencies - %s' % e)

package_control/clients/bitbucket_client.py

@@ -69,7 +69,7 @@ def repo_url(user_name, repo_name):
 
         return 'https://bitbucket.com/%s/%s' % (quote(user_name), quote(repo_name))
 
-    def download_info(self, url, tag_prefix=None):
+    async def download_info(self, url, tag_prefix=None):
         """
         Retrieve information about downloading a package
 
@@ -97,12 +97,12 @@ def download_info(self, url, tag_prefix=None):
               `date` - the ISO-8601 timestamp string when the version was published
         """
 
-        output = self.download_info_from_branch(url)
+        output = await self.download_info_from_branch(url)
         if output is None:
-            output = self.download_info_from_tags(url, tag_prefix)
+            output = await self.download_info_from_tags(url, tag_prefix)
         return output
 
-    def download_info_from_branch(self, url, default_branch=None):
+    async def download_info_from_branch(self, url, default_branch=None):
         """
         Retrieve information about downloading a package
 
@@ -135,18 +135,18 @@ def download_info_from_branch(self, url, default_branch=None):
         if branch is None:
             branch = default_branch
             if branch is None:
-                repo_info = self.fetch_json(self._api_url(user_repo))
+                repo_info = await self.fetch_json(self._api_url(user_repo))
                 branch = repo_info['mainbranch'].get('name', 'master')
 
         branch_url = self._api_url(user_repo, '/refs/branches/%s' % branch)
-        branch_info = self.fetch_json(branch_url)
+        branch_info = await self.fetch_json(branch_url)
 
         timestamp = branch_info['target']['date'][0:19].replace('T', ' ')
         version = re.sub(r'[\-: ]', '.', timestamp)
 
         return [self._make_download_info(user_repo, branch, version, timestamp)]
 
-    def download_info_from_releases(self, url, asset_templates, tag_prefix=None):
+    async def download_info_from_releases(self, url, asset_templates, tag_prefix=None):
         """
         BitBucket doesn't support releases in ways GitHub/Gitlab do.
 
@@ -158,7 +158,7 @@ def download_info_from_releases(self, url, asset_templates, tag_prefix=None):
 
         return None
 
-    def download_info_from_tags(self, url, tag_prefix=None):
+    async def download_info_from_tags(self, url, tag_prefix=None):
         """
         Retrieve information about downloading a package
 
@@ -188,12 +188,12 @@ def download_info_from_tags(self, url, tag_prefix=None):
         if not tags_match:
             return None
 
-        def _get_releases(user_repo, tag_prefix, page_size=100):
+        async def _get_releases(user_repo, tag_prefix, page_size=100):
             used_versions = set()
             query_string = urlencode({'pagelen': page_size})
             tags_url = self._api_url(user_repo, '/refs/tags?%s' % query_string)
             while tags_url:
-                tags_json = self.fetch_json(tags_url)
+                tags_json = await self.fetch_json(tags_url)
                 for tag in tags_json['values']:
                     version = version_match_prefix(tag['name'], tag_prefix)
                     if version and version not in used_versions:
@@ -212,7 +212,7 @@ def _get_releases(user_repo, tag_prefix, page_size=100):
         num_releases = 0
 
         output = []
-        for release in sorted(_get_releases(user_repo, tag_prefix), reverse=True):
+        async for release in _get_releases(user_repo, tag_prefix):
             version, tag, timestamp = release
 
             output.append(self._make_download_info(user_repo, tag, str(version), timestamp))
@@ -221,9 +221,10 @@ def _get_releases(user_repo, tag_prefix, page_size=100):
             if max_releases > 0 and num_releases >= max_releases:
                 break
 
+        output.sort(reverse=True)
         return output
 
-    def repo_info(self, url):
+    async def repo_info(self, url):
         """
         Retrieve general information about a repository
 
@@ -254,7 +255,7 @@ def repo_info(self, url):
 
         user_repo = "%s/%s" % (user_name, repo_name)
         api_url = self._api_url(user_repo)
-        repo_info = self.fetch_json(api_url)
+        repo_info = await self.fetch_json(api_url)
 
         if branch is None:
             branch = repo_info['mainbranch'].get('name', 'master')
@@ -266,7 +267,7 @@ def repo_info(self, url):
             author = repo_info['owner'].get('username')
 
         is_client = self.settings.get('min_api_calls', False)
-        readme_url = None if is_client else self._readme_url(user_repo, branch)
+        readme_url = None if is_client else await self._readme_url(user_repo, branch)
 
         return {
             'name': repo_info['name'],
@@ -279,7 +280,7 @@ def repo_info(self, url):
             'default_branch': branch
         }
 
-    def user_info(self, url):
+    async def user_info(self, url):
         """
         For API compatibility with other clients.
 
@@ -341,7 +342,7 @@ def _api_url(self, user_repo, suffix=''):
 
         return 'https://api.bitbucket.org/2.0/repositories/%s%s' % (user_repo, suffix)
 
-    def _readme_url(self, user_repo, branch, prefer_cached=False):
+    async def _readme_url(self, user_repo, branch, prefer_cached=False):
         """
         Parse the root directory listing for the repo and return the URL
         to any file that looks like a readme
@@ -367,7 +368,7 @@ def _readme_url(self, user_repo, branch, prefer_cached=False):
 
         try:
             while listing_url:
-                root_dir_info = self.fetch_json(listing_url, prefer_cached)
+                root_dir_info = await self.fetch_json(listing_url, prefer_cached)
 
                 for entry in root_dir_info['values']:
                     if entry['path'].lower() in _readme_filenames: