diff --git a/.github/workflows/mypy.yml b/.github/workflows/mypy.yml new file mode 100644 index 0000000000..b2178c2426 --- /dev/null +++ b/.github/workflows/mypy.yml @@ -0,0 +1,39 @@ +name: mypy + +on: + push: + pull_request: + + # Allows you to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + test-linux: + runs-on: ubuntu-20.04 + + steps: + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: "3.10" + cache: "pip" + + # This is needed for pygobject-stubs + - name: Install system packages + run: | + sudo apt install libgirepository1.0-dev + + - name: Install dependencies + run: | + pip install --upgrade pip wheel + pip install -r requirements.txt -r requirements-tests.txt + pip install -e . + + - name: Run mypy + run: | + mypy deluge/ diff --git a/deluge/core/core.py b/deluge/core/core.py index 1090b0f2a3..7823a44585 100644 --- a/deluge/core/core.py +++ b/deluge/core/core.py @@ -14,7 +14,9 @@ import tempfile import threading from base64 import b64decode, b64encode -from urllib.request import URLError, urlopen +from typing import Any, Dict, List, Optional, Tuple, Union +from urllib.error import URLError +from urllib.request import urlopen from twisted.internet import defer, reactor, task from twisted.web.client import Agent, readBody @@ -38,7 +40,7 @@ from deluge.core.preferencesmanager import PreferencesManager from deluge.core.rpcserver import export from deluge.core.torrentmanager import TorrentManager -from deluge.decorators import deprecated +from deluge.decorators import deprecated, maybe_coroutine from deluge.error import ( AddTorrentError, DelugeError, @@ -131,13 +133,13 @@ def __init__( self.session.add_extension('smart_ban') # Create the components - self.eventmanager = EventManager() - self.preferencesmanager = PreferencesManager() - self.alertmanager = AlertManager() - self.pluginmanager = PluginManager(self) - self.torrentmanager = TorrentManager() - self.filtermanager = FilterManager(self) - self.authmanager = AuthManager() + self.eventmanager: EventManager = EventManager() + self.preferencesmanager: PreferencesManager = PreferencesManager() + self.alertmanager: AlertManager = AlertManager() + self.pluginmanager: PluginManager = PluginManager(self) + self.torrentmanager: TorrentManager = TorrentManager() + self.filtermanager: FilterManager = FilterManager(self) + self.authmanager: AuthManager = AuthManager() # New release check information self.new_release = None @@ -240,13 +242,12 @@ def apply_session_settings(self, settings): """Apply libtorrent session settings. Args: - settings (dict): A dict of lt session settings to apply. - + settings: A dict of lt session settings to apply. """ self.session.apply_settings(settings) @staticmethod - def _create_peer_id(version): + def _create_peer_id(version: str) -> str: """Create a peer_id fingerprint. This creates the peer_id and modifies the release char to identify @@ -261,11 +262,10 @@ def _create_peer_id(version): ``--DE201b--`` (beta pre-release of v2.0.1) Args: - version (str): The version string in PEP440 dotted notation. + version: The version string in PEP440 dotted notation. Returns: - str: The formatted peer_id with Deluge prefix e.g. '--DE200s--' - + The formatted peer_id with Deluge prefix e.g. '--DE200s--' """ split = deluge.common.VersionSplit(version) # Fill list with zeros to length of 4 and use lt to create fingerprint. @@ -315,12 +315,7 @@ def _save_session_state(self): shutil.move(filepath_bak, filepath) def _load_session_state(self): - """Loads the libtorrent session state - - Returns: - dict: A libtorrent sesion state, empty dict if unable to load it. - - """ + """Loads the libtorrent session state""" filename = 'session.state' filepath = get_config_dir(filename) filepath_bak = filepath + '.bak' @@ -401,18 +396,23 @@ def check_new_release(self): # Exported Methods @export - def add_torrent_file_async(self, filename, filedump, options, save_state=True): + def add_torrent_file_async( + self, + filename: str, + filedump: Union[str, bytes], + options: dict, + save_state: bool = True, + ) -> defer.Deferred[Optional[str]]: """Adds a torrent file to the session asynchronously. Args: - filename (str): The filename of the torrent. - filedump (str): A base64 encoded string of torrent file contents. - options (dict): The options to apply to the torrent upon adding. - save_state (bool): If the state should be saved after adding the file. + filename: The filename of the torrent. + filedump: A base64 encoded string of torrent file contents. + options: The options to apply to the torrent upon adding. + save_state: If the state should be saved after adding the file. Returns: - Deferred: The torrent ID or None. - + The torrent ID or None. """ try: filedump = b64decode(filedump) @@ -433,7 +433,9 @@ def add_torrent_file_async(self, filename, filedump, options, save_state=True): return d @export - def prefetch_magnet_metadata(self, magnet, timeout=30): + def prefetch_magnet_metadata( + self, magnet: str, timeout: int = 30 + ) -> defer.Deferred[Tuple[str, bytes]]: """Download magnet metadata without adding to Deluge session. Used by UIs to get magnet files for selection before adding to session. @@ -441,12 +443,11 @@ def prefetch_magnet_metadata(self, magnet, timeout=30): The metadata is bencoded and for transfer base64 encoded. Args: - magnet (str): The magnet URI. - timeout (int): Number of seconds to wait before canceling request. + magnet: The magnet URI. + timeout: Number of seconds to wait before canceling request. Returns: - Deferred: A tuple of (torrent_id (str), metadata (str)) for the magnet. - + A tuple of (torrent_id (str), metadata (str)) for the magnet. """ def on_metadata(result, result_d): @@ -456,21 +457,23 @@ def on_metadata(result, result_d): d = self.torrentmanager.prefetch_metadata(magnet, timeout) # Use a separate callback chain to handle existing prefetching magnet. - result_d = defer.Deferred() + result_d: defer.Deferred = defer.Deferred() d.addBoth(on_metadata, result_d) return result_d @export - def add_torrent_file(self, filename, filedump, options): + def add_torrent_file( + self, filename: str, filedump: Union[str, bytes], options: dict + ) -> Optional[str]: """Adds a torrent file to the session. Args: - filename (str): The filename of the torrent. - filedump (str): A base64 encoded string of the torrent file contents. - options (dict): The options to apply to the torrent upon adding. + filename: The filename of the torrent. + filedump: A base64 encoded string of the torrent file contents. + options: The options to apply to the torrent upon adding. Returns: - str: The torrent_id or None. + The torrent_id or None. """ try: filedump = b64decode(filedump) @@ -486,48 +489,45 @@ def add_torrent_file(self, filename, filedump, options): raise @export - def add_torrent_files(self, torrent_files): + @maybe_coroutine + async def add_torrent_files( + self, torrent_files: List[Tuple[str, Union[str, bytes], dict]] + ) -> List[AddTorrentError]: """Adds multiple torrent files to the session asynchronously. Args: - torrent_files (list of tuples): Torrent files as tuple of - ``(filename, filedump, options)``. + torrent_files: Torrent files as tuple of + ``(filename, filedump, options)``. Returns: - Deferred - + A list of errors (if there were any) """ - - @defer.inlineCallbacks - def add_torrents(): - errors = [] - last_index = len(torrent_files) - 1 - for idx, torrent in enumerate(torrent_files): - try: - yield self.add_torrent_file_async( - torrent[0], torrent[1], torrent[2], save_state=idx == last_index - ) - except AddTorrentError as ex: - log.warning('Error when adding torrent: %s', ex) - errors.append(ex) - defer.returnValue(errors) - - return task.deferLater(reactor, 0, add_torrents) + errors = [] + last_index = len(torrent_files) - 1 + for idx, torrent in enumerate(torrent_files): + try: + await self.add_torrent_file_async( + torrent[0], torrent[1], torrent[2], save_state=idx == last_index + ) + except AddTorrentError as ex: + log.warning('Error when adding torrent: %s', ex) + errors.append(ex) + return errors @export - def add_torrent_url(self, url, options, headers=None): - """ - Adds a torrent from a URL. Deluge will attempt to fetch the torrent + def add_torrent_url( + self, url: str, options: dict, headers: dict = None + ) -> defer.Deferred[Optional[str]]: + """Adds a torrent from a URL. Deluge will attempt to fetch the torrent from the URL prior to adding it to the session. - :param url: the URL pointing to the torrent file - :type url: string - :param options: the options to apply to the torrent on add - :type options: dict - :param headers: any optional headers to send - :type headers: dict + Args: + url: the URL pointing to the torrent file + options: the options to apply to the torrent on add + headers: any optional headers to send - :returns: a Deferred which returns the torrent_id as a str or None + Returns: + a Deferred which returns the torrent_id as a str or None """ log.info('Attempting to add URL %s', url) @@ -553,55 +553,52 @@ def on_download_fail(failure): return d @export - def add_torrent_magnet(self, uri, options): - """ - Adds a torrent from a magnet link. - - :param uri: the magnet link - :type uri: string - :param options: the options to apply to the torrent on add - :type options: dict + def add_torrent_magnet(self, uri: str, options: dict) -> str: + """Adds a torrent from a magnet link. - :returns: the torrent_id - :rtype: string + Args: + uri: the magnet link + options: the options to apply to the torrent on add + Returns: + the torrent_id """ log.debug('Attempting to add by magnet URI: %s', uri) return self.torrentmanager.add(magnet=uri, options=options) @export - def remove_torrent(self, torrent_id, remove_data): + def remove_torrent(self, torrent_id: str, remove_data: bool) -> bool: """Removes a single torrent from the session. Args: - torrent_id (str): The torrent ID to remove. - remove_data (bool): If True, also remove the downloaded data. + torrent_id: The torrent ID to remove. + remove_data: If True, also remove the downloaded data. Returns: - bool: True if removed successfully. + True if removed successfully. Raises: InvalidTorrentError: If the torrent ID does not exist in the session. - """ log.debug('Removing torrent %s from the core.', torrent_id) return self.torrentmanager.remove(torrent_id, remove_data) @export - def remove_torrents(self, torrent_ids, remove_data): + def remove_torrents( + self, torrent_ids: List[str], remove_data: bool + ) -> defer.Deferred[List[Tuple[str, str]]]: """Remove multiple torrents from the session. Args: - torrent_ids (list): The torrent IDs to remove. - remove_data (bool): If True, also remove the downloaded data. + torrent_ids: The torrent IDs to remove. + remove_data: If True, also remove the downloaded data. Returns: - list: An empty list if no errors occurred otherwise the list contains - tuples of strings, a torrent ID and an error message. For example: - - [('', 'Error removing torrent')] + An empty list if no errors occurred otherwise the list contains + tuples of strings, a torrent ID and an error message. For example: + [('', 'Error removing torrent')] """ log.info('Removing %d torrents from core.', len(torrent_ids)) @@ -625,17 +622,17 @@ def do_remove_torrents(): return task.deferLater(reactor, 0, do_remove_torrents) @export - def get_session_status(self, keys): + def get_session_status(self, keys: List[str]) -> Dict[str, Union[int, float]]: """Gets the session status values for 'keys', these keys are taking from libtorrent's session status. See: http://www.rasterbar.com/products/libtorrent/manual.html#status - :param keys: the keys for which we want values - :type keys: list - :returns: a dictionary of {key: value, ...} - :rtype: dict + Args: + keys: the keys for which we want values + Returns: + a dictionary of {key: value, ...} """ if not keys: return self.session_status @@ -656,13 +653,13 @@ def get_session_status(self, keys): return status @export - def force_reannounce(self, torrent_ids): + def force_reannounce(self, torrent_ids: List[str]) -> None: log.debug('Forcing reannouncment to: %s', torrent_ids) for torrent_id in torrent_ids: self.torrentmanager[torrent_id].force_reannounce() @export - def pause_torrent(self, torrent_id): + def pause_torrent(self, torrent_id: str) -> None: """Pauses a torrent""" log.debug('Pausing: %s', torrent_id) if not isinstance(torrent_id, str): @@ -671,7 +668,7 @@ def pause_torrent(self, torrent_id): self.torrentmanager[torrent_id].pause() @export - def pause_torrents(self, torrent_ids=None): + def pause_torrents(self, torrent_ids: List[str] = None) -> None: """Pauses a list of torrents""" if not torrent_ids: torrent_ids = self.torrentmanager.get_torrent_list() @@ -679,27 +676,27 @@ def pause_torrents(self, torrent_ids=None): self.pause_torrent(torrent_id) @export - def connect_peer(self, torrent_id, ip, port): + def connect_peer(self, torrent_id: str, ip: str, port: int): log.debug('adding peer %s to %s', ip, torrent_id) if not self.torrentmanager[torrent_id].connect_peer(ip, port): log.warning('Error adding peer %s:%s to %s', ip, port, torrent_id) @export - def move_storage(self, torrent_ids, dest): + def move_storage(self, torrent_ids: List[str], dest: str): log.debug('Moving storage %s to %s', torrent_ids, dest) for torrent_id in torrent_ids: if not self.torrentmanager[torrent_id].move_storage(dest): log.warning('Error moving torrent %s to %s', torrent_id, dest) @export - def pause_session(self): + def pause_session(self) -> None: """Pause the entire session""" if not self.session.is_paused(): self.session.pause() component.get('EventManager').emit(SessionPausedEvent()) @export - def resume_session(self): + def resume_session(self) -> None: """Resume the entire session""" if self.session.is_paused(): self.session.resume() @@ -708,12 +705,12 @@ def resume_session(self): component.get('EventManager').emit(SessionResumedEvent()) @export - def is_session_paused(self): + def is_session_paused(self) -> bool: """Returns the activity of the session""" return self.session.is_paused() @export - def resume_torrent(self, torrent_id): + def resume_torrent(self, torrent_id: str) -> None: """Resumes a torrent""" log.debug('Resuming: %s', torrent_id) if not isinstance(torrent_id, str): @@ -722,7 +719,7 @@ def resume_torrent(self, torrent_id): self.torrentmanager[torrent_id].resume() @export - def resume_torrents(self, torrent_ids=None): + def resume_torrents(self, torrent_ids: List[str] = None) -> None: """Resumes a list of torrents""" if not torrent_ids: torrent_ids = self.torrentmanager.get_torrent_list() @@ -755,7 +752,9 @@ def create_torrent_status( return status @export - def get_torrent_status(self, torrent_id, keys, diff=False): + def get_torrent_status( + self, torrent_id: str, keys: List[str], diff: bool = False + ) -> dict: torrent_keys, plugin_keys = self.torrentmanager.separate_keys( keys, [torrent_id] ) @@ -769,14 +768,14 @@ def get_torrent_status(self, torrent_id, keys, diff=False): ) @export - @defer.inlineCallbacks - def get_torrents_status(self, filter_dict, keys, diff=False): - """ - returns all torrents , optionally filtered by filter_dict. - """ + @maybe_coroutine + async def get_torrents_status( + self, filter_dict: dict, keys: List[str], diff: bool = False + ) -> dict: + """returns all torrents , optionally filtered by filter_dict.""" all_keys = not keys torrent_ids = self.filtermanager.filter_torrent_ids(filter_dict) - status_dict, plugin_keys = yield self.torrentmanager.torrents_status_update( + status_dict, plugin_keys = await self.torrentmanager.torrents_status_update( torrent_ids, keys, diff=diff ) # Ask the plugin manager to fill in the plugin keys @@ -786,36 +785,37 @@ def get_torrents_status(self, filter_dict, keys, diff=False): return status_dict @export - def get_filter_tree(self, show_zero_hits=True, hide_cat=None): - """ - returns {field: [(value,count)] } + def get_filter_tree( + self, show_zero_hits: bool = True, hide_cat: List[str] = None + ) -> Dict: + """returns {field: [(value,count)] } for use in sidebar(s) """ return self.filtermanager.get_filter_tree(show_zero_hits, hide_cat) @export - def get_session_state(self): + def get_session_state(self) -> List[str]: """Returns a list of torrent_ids in the session.""" # Get the torrent list from the TorrentManager return self.torrentmanager.get_torrent_list() @export - def get_config(self): + def get_config(self) -> dict: """Get all the preferences as a dictionary""" return self.config.config @export - def get_config_value(self, key): + def get_config_value(self, key: str) -> Any: """Get the config value for key""" return self.config.get(key) @export - def get_config_values(self, keys): + def get_config_values(self, keys: List[str]) -> Dict[str, Any]: """Get the config values for the entered keys""" return {key: self.config.get(key) for key in keys} @export - def set_config(self, config): + def set_config(self, config: Dict[str, Any]): """Set the config with values from dictionary""" # Load all the values into the configuration for key in config: @@ -824,21 +824,20 @@ def set_config(self, config): self.config[key] = config[key] @export - def get_listen_port(self): + def get_listen_port(self) -> int: """Returns the active listen port""" return self.session.listen_port() @export - def get_proxy(self): + def get_proxy(self) -> Dict[str, Any]: """Returns the proxy settings Returns: - dict: Contains proxy settings. + Proxy settings. Notes: Proxy type names: 0: None, 1: Socks4, 2: Socks5, 3: Socks5 w Auth, 4: HTTP, 5: HTTP w Auth, 6: I2P - """ settings = self.session.get_settings() @@ -861,36 +860,38 @@ def get_proxy(self): return proxy_dict @export - def get_available_plugins(self): + def get_available_plugins(self) -> List[str]: """Returns a list of plugins available in the core""" return self.pluginmanager.get_available_plugins() @export - def get_enabled_plugins(self): + def get_enabled_plugins(self) -> List[str]: """Returns a list of enabled plugins in the core""" return self.pluginmanager.get_enabled_plugins() @export - def enable_plugin(self, plugin): + def enable_plugin(self, plugin: str) -> defer.Deferred[bool]: return self.pluginmanager.enable_plugin(plugin) @export - def disable_plugin(self, plugin): + def disable_plugin(self, plugin: str) -> defer.Deferred[bool]: return self.pluginmanager.disable_plugin(plugin) @export - def force_recheck(self, torrent_ids): + def force_recheck(self, torrent_ids: List[str]) -> None: """Forces a data recheck on torrent_ids""" for torrent_id in torrent_ids: self.torrentmanager[torrent_id].force_recheck() @export - def set_torrent_options(self, torrent_ids, options): + def set_torrent_options( + self, torrent_ids: List[str], options: Dict[str, Any] + ) -> None: """Sets the torrent options for torrent_ids Args: - torrent_ids (list): A list of torrent_ids to set the options for. - options (dict): A dict of torrent options to set. See + torrent_ids: A list of torrent_ids to set the options for. + options: A dict of torrent options to set. See ``torrent.TorrentOptions`` class for valid keys. """ if 'owner' in options and not self.authmanager.has_account(options['owner']): @@ -903,12 +904,14 @@ def set_torrent_options(self, torrent_ids, options): self.torrentmanager[torrent_id].set_options(options) @export - def set_torrent_trackers(self, torrent_id, trackers): + def set_torrent_trackers( + self, torrent_id: str, trackers: List[Dict[str, Any]] + ) -> None: """Sets a torrents tracker list. trackers will be ``[{"url", "tier"}]``""" return self.torrentmanager[torrent_id].set_trackers(trackers) @export - def get_magnet_uri(self, torrent_id): + def get_magnet_uri(self, torrent_id: str) -> str: return self.torrentmanager[torrent_id].get_magnet_uri() @deprecated @@ -1056,7 +1059,7 @@ def _create_torrent_thread( self.add_torrent_file(os.path.split(target)[1], filedump, options) @export - def upload_plugin(self, filename, filedump): + def upload_plugin(self, filename: str, filedump: Union[str, bytes]) -> None: """This method is used to upload new plugins to the daemon. It is used when connecting to the daemon remotely and installing a new plugin on the client side. ``plugin_data`` is a ``xmlrpc.Binary`` object of the file data, @@ -1074,26 +1077,24 @@ def upload_plugin(self, filename, filedump): component.get('CorePluginManager').scan_for_plugins() @export - def rescan_plugins(self): - """ - Re-scans the plugin folders for new plugins - """ + def rescan_plugins(self) -> None: + """Re-scans the plugin folders for new plugins""" component.get('CorePluginManager').scan_for_plugins() @export - def rename_files(self, torrent_id, filenames): - """ - Rename files in ``torrent_id``. Since this is an asynchronous operation by + def rename_files( + self, torrent_id: str, filenames: List[Tuple[int, str]] + ) -> defer.Deferred: + """Rename files in ``torrent_id``. Since this is an asynchronous operation by libtorrent, watch for the TorrentFileRenamedEvent to know when the files have been renamed. - :param torrent_id: the torrent_id to rename files - :type torrent_id: string - :param filenames: a list of index, filename pairs - :type filenames: ((index, filename), ...) - - :raises InvalidTorrentError: if torrent_id is invalid + Args: + torrent_id: the torrent_id to rename files + filenames: a list of index, filename pairs + Raises: + InvalidTorrentError: if torrent_id is invalid """ if torrent_id not in self.torrentmanager.torrents: raise InvalidTorrentError('torrent_id is not in session') @@ -1104,21 +1105,20 @@ def rename(): return task.deferLater(reactor, 0, rename) @export - def rename_folder(self, torrent_id, folder, new_folder): - """ - Renames the 'folder' to 'new_folder' in 'torrent_id'. Watch for the + def rename_folder( + self, torrent_id: str, folder: str, new_folder: str + ) -> defer.Deferred: + """Renames the 'folder' to 'new_folder' in 'torrent_id'. Watch for the TorrentFolderRenamedEvent which is emitted when the folder has been renamed successfully. - :param torrent_id: the torrent to rename folder in - :type torrent_id: string - :param folder: the folder to rename - :type folder: string - :param new_folder: the new folder name - :type new_folder: string - - :raises InvalidTorrentError: if the torrent_id is invalid + Args: + torrent_id: the torrent to rename folder in + folder: the folder to rename + new_folder: the new folder name + Raises: + InvalidTorrentError: if the torrent_id is invalid """ if torrent_id not in self.torrentmanager.torrents: raise InvalidTorrentError('torrent_id is not in session') @@ -1126,7 +1126,7 @@ def rename_folder(self, torrent_id, folder, new_folder): return self.torrentmanager[torrent_id].rename_folder(folder, new_folder) @export - def queue_top(self, torrent_ids): + def queue_top(self, torrent_ids: List[str]) -> None: log.debug('Attempting to queue %s to top', torrent_ids) # torrent_ids must be sorted in reverse before moving to preserve order for torrent_id in sorted( @@ -1140,7 +1140,7 @@ def queue_top(self, torrent_ids): log.warning('torrent_id: %s does not exist in the queue', torrent_id) @export - def queue_up(self, torrent_ids): + def queue_up(self, torrent_ids: List[str]) -> None: log.debug('Attempting to queue %s to up', torrent_ids) torrents = ( (self.torrentmanager.get_queue_position(torrent_id), torrent_id) @@ -1165,7 +1165,7 @@ def queue_up(self, torrent_ids): prev_queue_position = queue_position @export - def queue_down(self, torrent_ids): + def queue_down(self, torrent_ids: List[str]) -> None: log.debug('Attempting to queue %s to down', torrent_ids) torrents = ( (self.torrentmanager.get_queue_position(torrent_id), torrent_id) @@ -1190,7 +1190,7 @@ def queue_down(self, torrent_ids): prev_queue_position = queue_position @export - def queue_bottom(self, torrent_ids): + def queue_bottom(self, torrent_ids: List[str]) -> None: log.debug('Attempting to queue %s to bottom', torrent_ids) # torrent_ids must be sorted before moving to preserve order for torrent_id in sorted( @@ -1204,17 +1204,15 @@ def queue_bottom(self, torrent_ids): log.warning('torrent_id: %s does not exist in the queue', torrent_id) @export - def glob(self, path): + def glob(self, path: str) -> List[str]: return glob.glob(path) @export - def test_listen_port(self): - """ - Checks if the active port is open - - :returns: True if the port is open, False if not - :rtype: bool + def test_listen_port(self) -> defer.Deferred[Optional[bool]]: + """Checks if the active port is open + Returns: + True if the port is open, False if not """ port = self.get_listen_port() url = 'https://deluge-torrent.org/test_port.php?port=%s' % port @@ -1233,18 +1231,17 @@ def on_error(failure): return d @export - def get_free_space(self, path=None): - """ - Returns the number of free bytes at path - - :param path: the path to check free space at, if None, use the default download location - :type path: string + def get_free_space(self, path: str = None) -> int: + """Returns the number of free bytes at path - :returns: the number of free bytes at path - :rtype: int + Args: + path: the path to check free space at, if None, use the default download location - :raises InvalidPathError: if the path is invalid + Returns: + the number of free bytes at path + Raises: + InvalidPathError: if the path is invalid """ if not path: path = self.config['download_location'] @@ -1257,46 +1254,40 @@ def _on_external_ip_event(self, external_ip): self.external_ip = external_ip @export - def get_external_ip(self): - """ - Returns the external IP address received from libtorrent. - """ + def get_external_ip(self) -> str: + """Returns the external IP address received from libtorrent.""" return self.external_ip @export - def get_libtorrent_version(self): - """ - Returns the libtorrent version. - - :returns: the version - :rtype: string + def get_libtorrent_version(self) -> str: + """Returns the libtorrent version. + Returns: + the version """ return LT_VERSION @export - def get_completion_paths(self, args): - """ - Returns the available path completions for the input value. - """ + def get_completion_paths(self, args: Dict[str, Any]) -> Dict[str, Any]: + """Returns the available path completions for the input value.""" return path_chooser_common.get_completion_paths(args) @export(AUTH_LEVEL_ADMIN) - def get_known_accounts(self): + def get_known_accounts(self) -> List[Dict[str, Any]]: return self.authmanager.get_known_accounts() @export(AUTH_LEVEL_NONE) - def get_auth_levels_mappings(self): + def get_auth_levels_mappings(self) -> Tuple[Dict[str, int], Dict[int, str]]: return (AUTH_LEVELS_MAPPING, AUTH_LEVELS_MAPPING_REVERSE) @export(AUTH_LEVEL_ADMIN) - def create_account(self, username, password, authlevel): + def create_account(self, username: str, password: str, authlevel: str) -> bool: return self.authmanager.create_account(username, password, authlevel) @export(AUTH_LEVEL_ADMIN) - def update_account(self, username, password, authlevel): + def update_account(self, username: str, password: str, authlevel: str) -> bool: return self.authmanager.update_account(username, password, authlevel) @export(AUTH_LEVEL_ADMIN) - def remove_account(self, username): + def remove_account(self, username: str) -> bool: return self.authmanager.remove_account(username) diff --git a/deluge/core/preferencesmanager.py b/deluge/core/preferencesmanager.py index 79e960252e..d921da0aab 100644 --- a/deluge/core/preferencesmanager.py +++ b/deluge/core/preferencesmanager.py @@ -23,14 +23,13 @@ from deluge._libtorrent import lt from deluge.event import ConfigValueChangedEvent -GeoIP = None try: from GeoIP import GeoIP except ImportError: try: from pygeoip import GeoIP except ImportError: - pass + GeoIP = None log = logging.getLogger(__name__) diff --git a/deluge/core/torrentmanager.py b/deluge/core/torrentmanager.py index 1bc05365c4..69c33714cf 100644 --- a/deluge/core/torrentmanager.py +++ b/deluge/core/torrentmanager.py @@ -16,6 +16,7 @@ from base64 import b64encode from collections import namedtuple from tempfile import gettempdir +from typing import Any, Dict, List, Optional from twisted.internet import defer, error, reactor, threads from twisted.internet.defer import Deferred, DeferredList @@ -33,6 +34,7 @@ from deluge.configmanager import ConfigManager, get_config_dir from deluge.core.authmanager import AUTH_LEVEL_ADMIN from deluge.core.torrent import Torrent, TorrentOptions, sanitize_filepath +from deluge.decorators import maybe_coroutine from deluge.error import AddTorrentError, InvalidTorrentError from deluge.event import ( ExternalIPEvent, @@ -247,8 +249,8 @@ def start(self): self.save_resume_data_timer.start(190, False) self.prev_status_cleanup_loop.start(10) - @defer.inlineCallbacks - def stop(self): + @maybe_coroutine + async def stop(self): # Stop timers if self.save_state_timer.running: self.save_state_timer.stop() @@ -260,11 +262,11 @@ def stop(self): self.prev_status_cleanup_loop.stop() # Save state on shutdown - yield self.save_state() + await self.save_state() self.session.pause() - result = yield self.save_resume_data(flush_disk_cache=True) + result = await self.save_resume_data(flush_disk_cache=True) # Remove the temp_file to signify successfully saved state if result and os.path.isfile(self.temp_file): os.remove(self.temp_file) @@ -293,7 +295,7 @@ def update(self): if not torrent.handle.status().paused: torrent.pause() - def __getitem__(self, torrent_id): + def __getitem__(self, torrent_id: str) -> Torrent: """Return the Torrent with torrent_id. Args: @@ -305,11 +307,11 @@ def __getitem__(self, torrent_id): """ return self.torrents[torrent_id] - def get_torrent_list(self): + def get_torrent_list(self) -> List[str]: """Creates a list of torrent_ids, owned by current user and any marked shared. Returns: - list: A list of torrent_ids. + A list of torrent_ids. """ torrent_ids = list(self.torrents) @@ -323,11 +325,11 @@ def get_torrent_list(self): torrent_ids.pop(torrent_ids.index(torrent_id)) return torrent_ids - def get_torrent_info_from_file(self, filepath): + def get_torrent_info_from_file(self, filepath: str) -> Optional[Dict[str, Any]]: """Retrieves torrent_info from the file specified. Args: - filepath (str): The filepath to extract torrent info from. + filepath: The filepath to extract torrent info from. Returns: lt.torrent_info: A libtorrent torrent_info dict or None if invalid file or data. @@ -340,15 +342,16 @@ def get_torrent_info_from_file(self, filepath): torrent_info = lt.torrent_info(filepath) except RuntimeError as ex: log.warning('Unable to open torrent file %s: %s', filepath, ex) + return None else: return torrent_info - def prefetch_metadata(self, magnet, timeout): + def prefetch_metadata(self, magnet: str, timeout: int) -> Deferred: """Download the metadata for a magnet URI. Args: - magnet (str): A magnet URI to download the metadata for. - timeout (int): Number of seconds to wait before canceling. + magnet: A magnet URI to download the metadata for. + timeout: Number of seconds to wait before canceling. Returns: Deferred: A tuple of (torrent_id (str), metadata (dict)) @@ -359,7 +362,7 @@ def prefetch_metadata(self, magnet, timeout): if torrent_id in self.prefetching_metadata: return self.prefetching_metadata[torrent_id].defer - add_torrent_params = {} + add_torrent_params: Dict[str, Any] = {} add_torrent_params['save_path'] = gettempdir() add_torrent_params['url'] = magnet.strip().encode('utf8') add_torrent_params['flags'] = ( @@ -374,9 +377,9 @@ def prefetch_metadata(self, magnet, timeout): torrent_handle = self.session.add_torrent(add_torrent_params) - d = Deferred() + d: Deferred = Deferred() # Cancel the defer if timeout reached. - defer_timeout = self.callLater(timeout, d.cancel) + defer_timeout = TorrentManager.callLater(timeout, d.cancel) d.addBoth(self.on_prefetch_metadata, torrent_id, defer_timeout) Prefetch = namedtuple('Prefetch', 'defer handle') self.prefetching_metadata[torrent_id] = Prefetch(defer=d, handle=torrent_handle) diff --git a/deluge/decorators.py b/deluge/decorators.py index 8ca8b805d0..b5a249de31 100644 --- a/deluge/decorators.py +++ b/deluge/decorators.py @@ -10,6 +10,9 @@ import re import warnings from functools import wraps +from typing import Any, Callable, Coroutine, TypeVar + +from twisted.internet import defer def proxy(proxy_func): @@ -159,3 +162,54 @@ def depr_func(*args, **kwargs): return func(*args, **kwargs) return depr_func + + +class CoroutineDeferred(defer.Deferred): + """Wraps a coroutine in a Deferred. + It will dynamically pass through the underlying coroutine without wrapping where apporpriate.""" + + def __init__(self, coro: Coroutine): + # Delay this import to make sure a reactor was installed first + from twisted.internet import reactor + + super().__init__() + self.coro = coro + self.awaited = None + self.activate_deferred = reactor.callLater(0, self.activate) + + def __await__(self): + if self.awaited in [None, True]: + self.awaited = True + return self.coro.__await__() + # Already in deferred mode + return super().__await__() + + def activate(self): + """If the result wasn't awaited before the next context switch, we turn it into a deferred.""" + if self.awaited is None: + self.awaited = False + d = defer.Deferred.fromCoroutine(self.coro) + d.chainDeferred(self) + + def addCallbacks(self, *args, **kwargs): # noqa: N802 + assert not self.awaited, 'Cannot add callbacks to an already awaited coroutine.' + self.activate() + return super().addCallbacks(*args, **kwargs) + + +_RetT = TypeVar('_RetT') + + +def maybe_coroutine( + f: Callable[..., Coroutine[Any, Any, _RetT]] +) -> Callable[..., defer.Deferred[_RetT]]: + """Wraps a coroutine function to make it usable as a normal function that returns a Deferred.""" + + @wraps(f) + def wrapper(*args, **kwargs): + # Uncomment for quick testing to make sure CoroutineDeferred magic isn't at fault + # from twisted.internet.defer import ensureDeferred + # return ensureDeferred(f(*args, **kwargs)) + return CoroutineDeferred(f(*args, **kwargs)) + + return wrapper diff --git a/deluge/log.py b/deluge/log.py index 6ce6c2df69..70151e6342 100644 --- a/deluge/log.py +++ b/deluge/log.py @@ -20,6 +20,7 @@ from twisted.python.log import PythonLoggingObserver from deluge import common +from deluge.decorators import maybe_coroutine __all__ = ('setup_logger', 'set_logger_level', 'get_plugin_logger', 'LOG') @@ -51,39 +52,39 @@ def __init__(self, logger_name): ) ) - @defer.inlineCallbacks - def garbage(self, msg, *args, **kwargs): - yield LoggingLoggerClass.log(self, 1, msg, *args, **kwargs) + @maybe_coroutine + async def garbage(self, msg, *args, **kwargs): + LoggingLoggerClass.log(self, 1, msg, *args, **kwargs) - @defer.inlineCallbacks - def trace(self, msg, *args, **kwargs): - yield LoggingLoggerClass.log(self, 5, msg, *args, **kwargs) + @maybe_coroutine + async def trace(self, msg, *args, **kwargs): + LoggingLoggerClass.log(self, 5, msg, *args, **kwargs) - @defer.inlineCallbacks - def debug(self, msg, *args, **kwargs): - yield LoggingLoggerClass.debug(self, msg, *args, **kwargs) + @maybe_coroutine + async def debug(self, msg, *args, **kwargs): + LoggingLoggerClass.debug(self, msg, *args, **kwargs) - @defer.inlineCallbacks - def info(self, msg, *args, **kwargs): - yield LoggingLoggerClass.info(self, msg, *args, **kwargs) + @maybe_coroutine + async def info(self, msg, *args, **kwargs): + LoggingLoggerClass.info(self, msg, *args, **kwargs) - @defer.inlineCallbacks - def warning(self, msg, *args, **kwargs): - yield LoggingLoggerClass.warning(self, msg, *args, **kwargs) + @maybe_coroutine + async def warning(self, msg, *args, **kwargs): + LoggingLoggerClass.warning(self, msg, *args, **kwargs) warn = warning - @defer.inlineCallbacks - def error(self, msg, *args, **kwargs): - yield LoggingLoggerClass.error(self, msg, *args, **kwargs) + @maybe_coroutine + async def error(self, msg, *args, **kwargs): + LoggingLoggerClass.error(self, msg, *args, **kwargs) - @defer.inlineCallbacks - def critical(self, msg, *args, **kwargs): - yield LoggingLoggerClass.critical(self, msg, *args, **kwargs) + @maybe_coroutine + async def critical(self, msg, *args, **kwargs): + LoggingLoggerClass.critical(self, msg, *args, **kwargs) - @defer.inlineCallbacks - def exception(self, msg, *args, **kwargs): - yield LoggingLoggerClass.exception(self, msg, *args, **kwargs) + @maybe_coroutine + async def exception(self, msg, *args, **kwargs): + LoggingLoggerClass.exception(self, msg, *args, **kwargs) def findCaller(self, *args, **kwargs): # NOQA: N802 f = logging.currentframe().f_back diff --git a/deluge/path_chooser_common.py b/deluge/path_chooser_common.py index 858f7c2da0..0ea92341cf 100644 --- a/deluge/path_chooser_common.py +++ b/deluge/path_chooser_common.py @@ -40,7 +40,7 @@ def get_completion_paths(args): :param args: options :type args: dict :returns: the args argument containing the available completions for the completion_text - :rtype: list + :rtype: dict """ args['paths'] = [] diff --git a/deluge/tests/test_maybe_coroutine.py b/deluge/tests/test_maybe_coroutine.py new file mode 100644 index 0000000000..2717e78bb6 --- /dev/null +++ b/deluge/tests/test_maybe_coroutine.py @@ -0,0 +1,213 @@ +# +# This file is part of Deluge and is licensed under GNU General Public License 3.0, or later, with +# the additional special exception to link portions of this program with the OpenSSL library. +# See LICENSE for more details. +# +import pytest +import pytest_twisted +import twisted.python.failure +from twisted.internet import defer, reactor, task +from twisted.internet.defer import maybeDeferred + +from deluge.decorators import maybe_coroutine + + +@defer.inlineCallbacks +def inline_func(): + result = yield task.deferLater(reactor, 0, lambda: 'function_result') + return result + + +@defer.inlineCallbacks +def inline_error(): + raise Exception('function_error') + yield + + +@maybe_coroutine +async def coro_func(): + result = await task.deferLater(reactor, 0, lambda: 'function_result') + return result + + +@maybe_coroutine +async def coro_error(): + raise Exception('function_error') + + +@defer.inlineCallbacks +def coro_func_from_inline(): + result = yield coro_func() + return result + + +@defer.inlineCallbacks +def coro_error_from_inline(): + result = yield coro_error() + return result + + +@maybe_coroutine +async def coro_func_from_coro(): + return await coro_func() + + +@maybe_coroutine +async def coro_error_from_coro(): + return await coro_error() + + +@maybe_coroutine +async def inline_func_from_coro(): + return await inline_func() + + +@maybe_coroutine +async def inline_error_from_coro(): + return await inline_error() + + +@pytest_twisted.inlineCallbacks +def test_standard_twisted(): + """Sanity check that twisted tests work how we expect. + + Not really testing deluge code at all. + """ + result = yield inline_func() + assert result == 'function_result' + + with pytest.raises(Exception, match='function_error'): + yield inline_error() + + +@pytest.mark.parametrize( + 'function', + [ + inline_func, + coro_func, + coro_func_from_coro, + coro_func_from_inline, + inline_func_from_coro, + ], +) +@pytest_twisted.inlineCallbacks +def test_from_inline(function): + """Test our coroutines wrapped with maybe_coroutine as if they returned plain twisted deferreds.""" + result = yield function() + assert result == 'function_result' + + def cb(result): + assert result == 'function_result' + + d = function() + d.addCallback(cb) + yield d + + +@pytest.mark.parametrize( + 'function', + [ + inline_error, + coro_error, + coro_error_from_coro, + coro_error_from_inline, + inline_error_from_coro, + ], +) +@pytest_twisted.inlineCallbacks +def test_error_from_inline(function): + """Test our coroutines wrapped with maybe_coroutine as if they returned plain twisted deferreds that raise.""" + with pytest.raises(Exception, match='function_error'): + yield function() + + def eb(result): + assert isinstance(result, twisted.python.failure.Failure) + assert result.getErrorMessage() == 'function_error' + + d = function() + d.addErrback(eb) + yield d + + +@pytest.mark.parametrize( + 'function', + [ + inline_func, + coro_func, + coro_func_from_coro, + coro_func_from_inline, + inline_func_from_coro, + ], +) +@pytest_twisted.ensureDeferred +async def test_from_coro(function): + """Test our coroutines wrapped with maybe_coroutine work from another coroutine.""" + result = await function() + assert result == 'function_result' + + +@pytest.mark.parametrize( + 'function', + [ + inline_error, + coro_error, + coro_error_from_coro, + coro_error_from_inline, + inline_error_from_coro, + ], +) +@pytest_twisted.ensureDeferred +async def test_error_from_coro(function): + """Test our coroutines wrapped with maybe_coroutine work from another coroutine with errors.""" + with pytest.raises(Exception, match='function_error'): + await function() + + +@pytest_twisted.ensureDeferred +async def test_tracebacks_preserved(): + with pytest.raises(Exception) as exc: + await coro_error_from_coro() + traceback_lines = [ + 'await coro_error_from_coro()', + 'return await coro_error()', + "raise Exception('function_error')", + ] + # If each coroutine got wrapped with ensureDeferred, the traceback will be mangled + # verify the coroutines passed through by checking the traceback. + for expected, actual in zip(traceback_lines, exc.traceback): + assert expected in str(actual) + + +@pytest_twisted.ensureDeferred +async def test_maybe_deferred_coroutine(): + result = await maybeDeferred(coro_func) + assert result == 'function_result' + + +@pytest_twisted.ensureDeferred +async def test_callback_before_await(): + def cb(res): + assert res == 'function_result' + return res + + d = coro_func() + d.addCallback(cb) + result = await d + assert result == 'function_result' + + +@pytest_twisted.ensureDeferred +async def test_callback_after_await(): + """If it has already been used as a coroutine, can't be retroactively turned into a Deferred. + This limitation could be fixed, but the extra complication doesn't feel worth it. + """ + + def cb(res): + pass + + d = coro_func() + await d + with pytest.raises( + Exception, match='Cannot add callbacks to an already awaited coroutine' + ): + d.addCallback(cb) diff --git a/docs/requirements.txt b/docs/requirements.txt index 18ef4fe47f..70739640a4 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,3 +2,4 @@ sphinx==4.* myst-parser sphinx_rtd_theme sphinxcontrib-spelling==7.3.0 +sphinx-autodoc-typehints diff --git a/docs/source/conf.py b/docs/source/conf.py index f04653e031..9c5853c739 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -48,6 +48,7 @@ 'sphinx.ext.coverage', 'sphinxcontrib.spelling', 'myst_parser', + 'sphinx_autodoc_typehints', ] napoleon_include_init_with_doc = True diff --git a/pyproject.toml b/pyproject.toml index 67ebe0a0c9..e26d1075bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,3 +9,20 @@ skip-string-normalization = true [tool.isort] profile = "black" + +[tool.mypy] +python_version = 3.6 +namespace_packages = true +plugins = ["mypy_zope:plugin"] +allow_redefinition = true +mypy_path = ["$MYPY_CONFIG_FILE_DIR/stubs"] + +[[tool.mypy.overrides]] +module = ["win32con", "win32api", "win32file", "win32process", "win32event", "winerror"] +ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ["GeoIP", "pygeoip"] +ignore_missing_imports = true +[[tool.mypy.overrides]] +module = ["pytest", "pytest_twisted"] +ignore_missing_imports = true diff --git a/requirements-tests.txt b/requirements-tests.txt index 705d967745..1805f30d33 100644 --- a/requirements-tests.txt +++ b/requirements-tests.txt @@ -9,3 +9,11 @@ flake8-isort pep8-naming mccabe pylint +# Type checking stuff +mypy +mypy-zope +pygobject-stubs +types-certifi +types-chardet +types-pyOpenSSL +types-setuptools diff --git a/stubs/twisted/__init__.pyi b/stubs/twisted/__init__.pyi new file mode 100644 index 0000000000..d9817e5de2 --- /dev/null +++ b/stubs/twisted/__init__.pyi @@ -0,0 +1,3 @@ +from . import __all__ + +__version__: str diff --git a/stubs/twisted/internet/__init__.pyi b/stubs/twisted/internet/__init__.pyi new file mode 100644 index 0000000000..88066105b8 --- /dev/null +++ b/stubs/twisted/internet/__init__.pyi @@ -0,0 +1,4 @@ +from .base import ReactorBase + + +reactor: ReactorBase