From cd727d9cba53a7ff64abac64511ca49d8eb0ead0 Mon Sep 17 00:00:00 2001 From: jalew188 Date: Sun, 7 Jan 2024 14:49:08 +0100 Subject: [PATCH 1/3] FIX #129: use multiprocessing.Manager().dict() to pass global_settings to all subprocesses --- peptdeep/cli_argparse.py | 83 +++++++++++++++----------- peptdeep/model/featurize.py | 32 ++-------- peptdeep/pretrained_models.py | 15 +++-- peptdeep/settings.py | 108 ++++++++++++++++++++++------------ 4 files changed, 136 insertions(+), 102 deletions(-) diff --git a/peptdeep/cli_argparse.py b/peptdeep/cli_argparse.py index 4eaf0601..d87ba696 100644 --- a/peptdeep/cli_argparse.py +++ b/peptdeep/cli_argparse.py @@ -1,11 +1,13 @@ #!python import os - -from peptdeep.settings import global_settings, load_global_settings - import argparse +from peptdeep.settings import ( + global_settings, load_global_settings, + _refine_global_settings +) + __argparse_dict_level_sep="--" # do not change def convert_dict_to_argparse( @@ -18,40 +20,48 @@ def convert_dict_to_argparse( return [(prefix_key, settings)] ret = [] for key, val in settings.items(): - ret += convert_dict_to_argparse( - val, prefix_key=(prefix_key+dict_level_sep+key) if prefix_key else key - ) + if key in [ + "labeling_channels", + "psm_modification_mapping", + "user_defined_modifications", + "instrument_group", + ]: + ret += [(prefix_key+dict_level_sep+key, val)] + else: + ret += convert_dict_to_argparse( + val, prefix_key=(prefix_key+dict_level_sep+key) if prefix_key else key + ) return ret else: return [(prefix_key, settings)] def _set_dict_val(_dict, keys, val): if len(keys) < 1: return + elif keys[0] == "labeling_channels": + def _get(x:str): + i = x.find(":") + k,v = x[:i], x[i+1:] + k = int(k) if k.isdigit() else k + v = v.split(";") + return k,v + _dict[keys[0]].update(dict([_get(s) for s in val])) + elif keys[0] == "psm_modification_mapping": + def _get(x): + i = x.find(":", x.find("@")) + k,v = x[:i], x[i+1:] + return k, v.split(";") + _dict[keys[0]].update(dict([_get(s) for s in val])) + elif keys[0] == "user_defined_modifications": + def _get(x): + i = x.find(":", x.find("@")) + k,v = x[:i], x[i+1:] + items = v.split(";") + if len(items) == 1: + return k, {"composition":items[0]} + else: + return k, {"composition": items[0], "modloss_composition": items[1]} + _dict[keys[0]].update(dict([_get(s) for s in val])) elif len(keys) == 1: - if keys[0] == "labeling_channels": - def _get(x:str): - i = x.find(":") - k,v = x[:i], x[i+1:] - k = int(k) if k.isdigit() else k - v = v.split(";") - return k,v - val = dict([_get(s) for s in val]) - elif keys[0] == "psm_modification_mapping": - def _get(x): - i = x.find(":", x.find("@")) - k,v = x[:i], x[i+1:] - return k, v.split(";") - val = dict([_get(s) for s in val]) - elif keys[0] == "user_defined_modifications": - def _get(x): - i = x.find(":", x.find("@")) - k,v = x[:i], x[i+1:] - items = v.split(";") - if len(items) == 1: - return k, {"composition":items[0]} - else: - return k, {"composition": items[0], "modloss_composition": items[1]} - val = dict([_get(s) for s in val]) _dict[keys[0]] = val else: _set_dict_val(_dict[keys[0]], keys[1:], val) @@ -83,8 +93,7 @@ def get_parser(): return parser def parse_args_to_global_settings(parser, args): - args, extras = parser.parse_known_args(args) - args_dict = vars(args) + args_dict = vars(parser.parse_known_args(args)[0]) if "settings_yaml" in args_dict: if os.path.isfile( args_dict["settings_yaml"] @@ -95,7 +104,15 @@ def parse_args_to_global_settings(parser, args): else: print(f"Settings.yaml `{args_dict['settings_yaml']}` does not exist.") args_dict.pop("settings_yaml") - for key, val in vars(args).items(): + used_args = {} + for arg in args: + if arg.startswith("--"): + arg = arg[2:].replace("--","__") + if arg in args_dict: + used_args[arg] = args_dict[arg] + + for key, val in used_args.items(): keys = key.split("__") _set_dict_val(global_settings, keys, val) + _refine_global_settings() return global_settings \ No newline at end of file diff --git a/peptdeep/model/featurize.py b/peptdeep/model/featurize.py index fd6b643d..af2699ef 100644 --- a/peptdeep/model/featurize.py +++ b/peptdeep/model/featurize.py @@ -1,35 +1,11 @@ import numpy as np import pandas as pd from typing import List, Union -from alphabase.constants.modification import MOD_DF -from peptdeep.settings import model_const - -mod_elements = model_const['mod_elements'] -mod_feature_size = len(mod_elements) - -mod_elem_to_idx = dict(zip(mod_elements, range(mod_feature_size))) - -def _parse_mod_formula(formula): - ''' - Parse a modification formula to a feature vector - ''' - feature = np.zeros(mod_feature_size) - elems = formula.strip(')').split(')') - for elem in elems: - chem, num = elem.split('(') - num = int(num) - if chem in mod_elem_to_idx: - feature[mod_elem_to_idx[chem]] = num - else: - feature[-1] += num - return feature - -MOD_TO_FEATURE = {} -def update_all_mod_features(): - for modname, formula in MOD_DF[['mod_name','composition']].values: - MOD_TO_FEATURE[modname] = _parse_mod_formula(formula) -update_all_mod_features() +from peptdeep.settings import ( + model_const, mod_feature_size, + MOD_TO_FEATURE +) def parse_mod_feature( nAA:int, diff --git a/peptdeep/pretrained_models.py b/peptdeep/pretrained_models.py index 93e5c26f..54803ea3 100644 --- a/peptdeep/pretrained_models.py +++ b/peptdeep/pretrained_models.py @@ -29,7 +29,7 @@ ccs_to_mobility_for_df ) -from peptdeep.settings import global_settings +from peptdeep.settings import global_settings, add_user_defined_modifications from peptdeep.utils import logging, process_bar from peptdeep.settings import global_settings @@ -43,7 +43,7 @@ uniform_sampling, evaluate_linear_regression ) -from peptdeep.settings import global_settings +from peptdeep.settings import global_settings, update_global_settings pretrain_dir = os.path.join( os.path.join( @@ -909,8 +909,9 @@ def predict_mobility(self, precursor_df:pd.DataFrame, precursor_df ) - def _predict_func_for_mp(self, arg_dict): + def _predict_func_for_mp(self, arg_dict:dict): """Internal function, for multiprocessing""" + update_global_settings(arg_dict.pop("mp_global_settings")) return self.predict_all( multiprocessing=False, **arg_dict ) @@ -930,6 +931,10 @@ def predict_all_mp(self, precursor_df:pd.DataFrame, df_groupby = precursor_df.groupby('nAA') + mgr = mp.Manager() + mp_global_settings = mgr.dict() + mp_global_settings.update(global_settings) + def get_batch_num_mp(df_groupby): batch_num = 0 for group_len in df_groupby.size().values: @@ -944,6 +949,7 @@ def mp_param_generator(df_groupby): 'precursor_df': df.iloc[i:i+mp_batch_size,:], 'predict_items': predict_items, 'frag_types': frag_types, + 'mp_global_settings': mp_global_settings } precursor_df_list = [] @@ -1076,7 +1082,8 @@ def refine_df(df): update_precursor_mz(precursor_df) if ( - self.ms2_model.device_type!='cpu' or not multiprocessing + self.ms2_model.device_type!='cpu' + or not multiprocessing or process_num <= 1 or len(precursor_df) < min_required_precursor_num_for_mp ): refine_df(precursor_df) diff --git a/peptdeep/settings.py b/peptdeep/settings.py index 62cf6f18..b6c4efb5 100644 --- a/peptdeep/settings.py +++ b/peptdeep/settings.py @@ -1,10 +1,11 @@ import os import collections +import numpy as np from alphabase.yaml_utils import load_yaml from alphabase.constants.modification import ( load_mod_df, keep_modloss_by_importance, - add_new_modifications + add_new_modifications, MOD_DF ) from peptdeep.constants._const import CONST_FOLDER @@ -17,6 +18,66 @@ it controls all functionalities of PeptDeep. """ +model_const = load_yaml( + os.path.join(CONST_FOLDER, 'model_const.yaml') +) + +### MOD_TO_FEATURE +mod_elements = model_const['mod_elements'] +mod_feature_size = len(mod_elements) + +mod_elem_to_idx = dict(zip(mod_elements, range(mod_feature_size))) + +def _parse_mod_formula(formula): + ''' + Parse a modification formula to a feature vector + ''' + feature = np.zeros(mod_feature_size) + elems = formula.strip(')').split(')') + for elem in elems: + chem, num = elem.split('(') + num = int(num) + if chem in mod_elem_to_idx: + feature[mod_elem_to_idx[chem]] = num + else: + feature[-1] += num + return feature + +MOD_TO_FEATURE = {} +def update_all_mod_features(): + for modname, formula in MOD_DF[['mod_name','composition']].values: + MOD_TO_FEATURE[modname] = _parse_mod_formula(formula) +update_all_mod_features() + + +def add_user_defined_modifications( + user_mods:dict=None +): + """ + Add user-defined modifications into the system, + this is userful for isotope labeling. + + Parameters + ---------- + user_mods : dict, optional + Example: + ``` + { + "Dimethyl2@Any N-term": { + "composition": "H(2)2H(2)C(2)", + "modloss_composition": "" + }, ... + } + ``` + Set as `global_settings["user_defined_modifications"]` if it is None. + By default None. + """ + if user_mods is None: + user_mods = global_settings["common"]["user_defined_modifications"] + add_new_modifications(user_mods) + + update_all_mod_features() + def _refine_global_settings(): global_settings["thread_num"] = min( global_settings["thread_num"], @@ -46,11 +107,10 @@ def _refine_global_settings(): global_settings['model_mgr'][ 'instrument_group' ][key.upper()] = val -_refine_global_settings() -model_const = load_yaml( - os.path.join(CONST_FOLDER, 'model_const.yaml') -) + add_user_defined_modifications() + +_refine_global_settings() def update_settings(dict_, new_dict): for k, v in new_dict.items(): @@ -60,10 +120,13 @@ def update_settings(dict_, new_dict): dict_[k] = v return dict_ +def update_global_settings(new_settings): + update_settings(global_settings, new_settings) + _refine_global_settings() + def load_global_settings(yaml:str): d = load_yaml(yaml) - update_settings(global_settings, d) - _refine_global_settings() + update_global_settings(d) def update_modifications(tsv:str="", modloss_importance_level:float=1.0 @@ -88,33 +151,4 @@ def update_modifications(tsv:str="", add_user_defined_modifications() -def add_user_defined_modifications( - user_mods:dict=None -): - """ - Add user-defined modifications into the system, - this is userful for isotope labeling. - - Parameters - ---------- - user_mods : dict, optional - Example: - ``` - { - "Dimethyl2@Any N-term": { - "composition": "H(2)2H(2)C(2)", - "modloss_composition": "" - }, ... - } - ``` - Set as `global_settings["user_defined_modifications"]` if it is None. - By default None. - """ - if user_mods is None: - user_mods = global_settings["common"]["user_defined_modifications"] - add_new_modifications(user_mods) - - from peptdeep.model.featurize import update_all_mod_features - update_all_mod_features() - -update_modifications() +update_modifications() \ No newline at end of file From 308af0cc33ac10ba8b51047e34a38c98199c22f9 Mon Sep 17 00:00:00 2001 From: jalew188 Date: Sun, 7 Jan 2024 15:00:28 +0100 Subject: [PATCH 2/3] move MOD features from featurize.py to settings.py --- nbdev_nbs/settings.ipynb | 7 +++++-- peptdeep/model/featurize.py | 5 +++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/nbdev_nbs/settings.ipynb b/nbdev_nbs/settings.ipynb index 2a9cc34e..9a5da80b 100644 --- a/nbdev_nbs/settings.ipynb +++ b/nbdev_nbs/settings.ipynb @@ -40,7 +40,9 @@ { "data": { "text/plain": [ - "{'Lumos': 'Lumos',\n", + "{'ThermoTOF': 'ThermoTOF',\n", + " 'Astral': 'ThermoTOF',\n", + " 'Lumos': 'Lumos',\n", " 'QE': 'QE',\n", " 'timsTOF': 'timsTOF',\n", " 'SciexTOF': 'SciexTOF',\n", @@ -55,6 +57,8 @@ " 'QEHFX': 'QE',\n", " 'Exploris': 'QE',\n", " 'Exploris480': 'QE',\n", + " 'THERMOTOF': 'ThermoTOF',\n", + " 'ASTRAL': 'ThermoTOF',\n", " 'LUMOS': 'Lumos',\n", " 'TIMSTOF': 'timsTOF',\n", " 'SCIEXTOF': 'SciexTOF',\n", @@ -86,7 +90,6 @@ "metadata": {}, "outputs": [], "source": [ - "from peptdeep.model.featurize import MOD_TO_FEATURE, MOD_DF\n", "add_user_defined_modifications({\n", " \"Hi@S\":{'composition':\"H(2)\"},\n", " \"AlphaX@S\":{'composition':\"O(10)\",'modloss_composition':\"O(3)\"}\n", diff --git a/peptdeep/model/featurize.py b/peptdeep/model/featurize.py index af2699ef..c2525181 100644 --- a/peptdeep/model/featurize.py +++ b/peptdeep/model/featurize.py @@ -3,8 +3,9 @@ from typing import List, Union from peptdeep.settings import ( - model_const, mod_feature_size, - MOD_TO_FEATURE + model_const, mod_feature_size, MOD_TO_FEATURE, + mod_elements, mod_elem_to_idx, + _parse_mod_formula, update_all_mod_features, ) def parse_mod_feature( From 22711d79d94c3e84a4bf5351ce376f2f574b766e Mon Sep 17 00:00:00 2001 From: jalew188 Date: Sun, 7 Jan 2024 15:02:12 +0100 Subject: [PATCH 3/3] =?UTF-8?q?Bump=20version:=201.1.3=20=E2=86=92=201.1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .bumpversion.cfg | 2 +- docs/conf.py | 2 +- peptdeep/__init__.py | 2 +- release/one_click_linux_gui/control | 2 +- release/one_click_linux_gui/create_installer_linux.sh | 2 +- release/one_click_macos_gui/Info.plist | 4 ++-- release/one_click_macos_gui/create_installer_macos.sh | 4 ++-- release/one_click_macos_gui/distribution.xml | 2 +- release/one_click_windows_gui/create_installer_windows.sh | 2 +- release/one_click_windows_gui/peptdeep_innoinstaller.iss | 2 +- settings.ini | 2 +- 11 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index dee760e1..2e653458 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.1.3 +current_version = 1.1.4 commit = True tag = False parse = (?P\d+)\.(?P\d+)\.(?P\d+)(\-(?P[a-z]+)(?P\d+))? diff --git a/docs/conf.py b/docs/conf.py index c515ab27..92a8f8a6 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -23,7 +23,7 @@ copyright = '2022, Mann Labs, MPIB' author = 'Mann Labs, MPIB' -release = "1.1.3" +release = "1.1.4" # -- General configuration --------------------------------------------------- diff --git a/peptdeep/__init__.py b/peptdeep/__init__.py index 091cec1f..be3613bd 100644 --- a/peptdeep/__init__.py +++ b/peptdeep/__init__.py @@ -11,7 +11,7 @@ # pass __project__ = "peptdeep" -__version__ = "1.1.3" +__version__ = "1.1.4" __license__ = "Apache 2.0" __description__ = "The AlphaX deep learning framework for Proteomics" __author__ = "Mann Labs" diff --git a/release/one_click_linux_gui/control b/release/one_click_linux_gui/control index 3c1c9e1b..c671b2b5 100644 --- a/release/one_click_linux_gui/control +++ b/release/one_click_linux_gui/control @@ -1,5 +1,5 @@ Package: peptdeep -Version: 1.1.3 +Version: 1.1.4 Architecture: all Maintainer: Mann Labs Description: peptdeep diff --git a/release/one_click_linux_gui/create_installer_linux.sh b/release/one_click_linux_gui/create_installer_linux.sh index 0da95226..9dfa7be6 100644 --- a/release/one_click_linux_gui/create_installer_linux.sh +++ b/release/one_click_linux_gui/create_installer_linux.sh @@ -17,7 +17,7 @@ python setup.py sdist bdist_wheel # Setting up the local package cd release/one_click_linux_gui # Make sure you include the required extra packages and always use the stable or very-stable options! -pip install "../../dist/peptdeep-1.1.3-py3-none-any.whl[stable]" +pip install "../../dist/peptdeep-1.1.4-py3-none-any.whl[stable]" if [ "$1" == "CPU" ]; then pip install torch -U --extra-index-url https://download.pytorch.org/whl/cpu diff --git a/release/one_click_macos_gui/Info.plist b/release/one_click_macos_gui/Info.plist index dcd168e7..76cff301 100644 --- a/release/one_click_macos_gui/Info.plist +++ b/release/one_click_macos_gui/Info.plist @@ -9,9 +9,9 @@ CFBundleIconFile alpha_logo.icns CFBundleIdentifier - peptdeep.1.1.3 + peptdeep.1.1.4 CFBundleShortVersionString - 1.1.3 + 1.1.4 CFBundleInfoDictionaryVersion 6.0 CFBundleName diff --git a/release/one_click_macos_gui/create_installer_macos.sh b/release/one_click_macos_gui/create_installer_macos.sh index 70b06781..115b44a0 100644 --- a/release/one_click_macos_gui/create_installer_macos.sh +++ b/release/one_click_macos_gui/create_installer_macos.sh @@ -20,7 +20,7 @@ python setup.py sdist bdist_wheel # Setting up the local package cd release/one_click_macos_gui -pip install "../../dist/peptdeep-1.1.3-py3-none-any.whl[stable]" +pip install "../../dist/peptdeep-1.1.4-py3-none-any.whl[stable]" # Creating the stand-alone pyinstaller folder pip install pyinstaller @@ -40,5 +40,5 @@ cp ../../LICENSE.txt Resources/LICENSE.txt cp ../logos/alpha_logo.png Resources/alpha_logo.png chmod 777 scripts/* -pkgbuild --root dist/peptdeep --identifier de.mpg.biochem.peptdeep.app --version 1.1.3 --install-location /Applications/peptdeep.app --scripts scripts peptdeep.pkg +pkgbuild --root dist/peptdeep --identifier de.mpg.biochem.peptdeep.app --version 1.1.4 --install-location /Applications/peptdeep.app --scripts scripts peptdeep.pkg productbuild --distribution distribution.xml --resources Resources --package-path peptdeep.pkg dist/peptdeep_gui_installer_macos.pkg diff --git a/release/one_click_macos_gui/distribution.xml b/release/one_click_macos_gui/distribution.xml index 6efcd7c2..14e3d546 100644 --- a/release/one_click_macos_gui/distribution.xml +++ b/release/one_click_macos_gui/distribution.xml @@ -1,6 +1,6 @@ - peptdeep 1.1.3 + peptdeep 1.1.4 diff --git a/release/one_click_windows_gui/create_installer_windows.sh b/release/one_click_windows_gui/create_installer_windows.sh index 535a3108..cac643b8 100644 --- a/release/one_click_windows_gui/create_installer_windows.sh +++ b/release/one_click_windows_gui/create_installer_windows.sh @@ -17,7 +17,7 @@ python setup.py sdist bdist_wheel # Setting up the local package cd release/one_click_windows_gui # Make sure you include the required extra packages and always use the stable or very-stable options! -pip install "../../dist/peptdeep-1.1.3-py3-none-any.whl[stable]" +pip install "../../dist/peptdeep-1.1.4-py3-none-any.whl[stable]" # Creating the stand-alone pyinstaller folder pip install pyinstaller diff --git a/release/one_click_windows_gui/peptdeep_innoinstaller.iss b/release/one_click_windows_gui/peptdeep_innoinstaller.iss index c7ad5977..ef8865b4 100644 --- a/release/one_click_windows_gui/peptdeep_innoinstaller.iss +++ b/release/one_click_windows_gui/peptdeep_innoinstaller.iss @@ -2,7 +2,7 @@ ; SEE THE DOCUMENTATION FOR DETAILS ON CREATING INNO SETUP SCRIPT FILES! #define MyAppName "peptdeep" -#define MyAppVersion "1.1.3" +#define MyAppVersion "1.1.4" #define MyAppPublisher "Max Planck Institute of Biochemistry and the University of Copenhagen, Mann Labs" #define MyAppURL "https://github.com/MannLabs/peptdeep" #define MyAppExeName "peptdeep_gui.exe" diff --git a/settings.ini b/settings.ini index 9d57e692..344a60f5 100644 --- a/settings.ini +++ b/settings.ini @@ -5,7 +5,7 @@ ### Python library ### repo = alphapeptdeep lib_name = peptdeep -version = 1.1.3 +version = 1.1.4 min_python = 3.7 license = apache2