refactor: move utils to deeppavlov (deeppavlov#744)

* refactor: move utils to deeppavlov

* fix: wrong path to utils; refactor: rename telegram_utils to telegram

* refactor: rename all occurences of ms_bot_framework_utils to ms_bot_framework and all occurences of server_utils to server

* Update README.md

You skipped one changing paths `utils/ -> deeppavlov/utils/` occurrence.

* fix: replace telegram to telegram_utils and server to server_utils in .json files

* fix: replace 'server' with 'server_utils' in deeppavlov/utils/server/server.py

* fix: replace 'server' with 'server_utils' and 'telegram' with 'telegram_utils' back in configs and README.md

* fix: remove 'utils' from deeppavlov/utils/settings/log_config.json

MANIFEST.in

@@ -4,4 +4,3 @@ include requirements.txt
 include deeppavlov/requirements/*.txt
 recursive-include deeppavlov *.json
 recursive-include deeppavlov *.md
-recursive-include utils *.json

README.md

@@ -139,17 +139,17 @@ or a name without the `.json` extension of one of the config files [provided](de
 For the `interactbot` mode you should specify Telegram bot token in `-t` parameter or in `TELEGRAM_TOKEN` environment variable.
 Also you should use `--no-default-skill` optional flag if your component implements an interface of DeepPavlov [*Skill*](deeppavlov/core/skill/skill.py) to skip its wrapping with DeepPavlov [*DefaultStatelessSkill*](deeppavlov/skills/default_skill/default_skill.py).
 If you want to get custom `/start` and `/help` Telegram messages for the running model you should:
-* Add section to [*utils/settings/models_info.json*](utils/settings/models_info.json) with your custom Telegram messages
-* In model config file specify `metadata.labels.telegram_utils` parameter with name which refers to the added section of [*utils/settings/models_info.json*](utils/settings/models_info.json)
+* Add section to [*deeppavlov/utils/settings/models_info.json*](deeppavlov/utils/settings/models_info.json) with your custom Telegram messages
+* In model config file specify `metadata.labels.telegram_utils` parameter with name which refers to the added section of [*deeppavlov/utils/settings/models_info.json*](deeppavlov/utils/settings/models_info.json)
 
 You can also serve DeepPavlov models for:
 * Microsoft Bot Framework ([see developer guide for the detailed instructions](http://docs.deeppavlov.ai/en/latest/devguides/ms_bot_integration.html)) 
 * Amazon Alexa ([see developer guide for the detailed instructions](http://docs.deeppavlov.ai/en/latest/devguides/amazon_alexa.html)) 
 
-For `riseapi` mode you should specify api settings (host, port, etc.) in [*utils/settings/server_config.json*](utils/settings/server_config.json) configuration file. If provided, values from *model_defaults* section override values for the same parameters from *common_defaults* section. Model names in *model_defaults* section should be similar to the class names of the models main component.
+For `riseapi` mode you should specify api settings (host, port, etc.) in [*deeppavlov/utils/settings/server_config.json*](deeppavlov/utils/settings/server_config.json) configuration file. If provided, values from *model_defaults* section override values for the same parameters from *common_defaults* section. Model names in *model_defaults* section should be similar to the class names of the models main component.
 Here is [detailed info on the DeepPavlov REST API](http://docs.deeppavlov.ai/en/latest/devguides/rest_api.html)
 
-All DeepPavlov settings files are stored in `utils/settings` by default. You can get full path to it with `python -m deeppavlov.settings settings`. Also you can move it with with `python -m deeppavlov.settings settings -p <new/configs/dir/path>` (all your configuration settings will be preserved) or move it to default location with `python -m deeppavlov.settings settings -d` (all your configuration settings will be RESET to default ones).
+All DeepPavlov settings files are stored in `deeppavlov/utils/settings` by default. You can get full path to it with `python -m deeppavlov.settings settings`. Also you can move it with with `python -m deeppavlov.settings settings -p <new/configs/dir/path>` (all your configuration settings will be preserved) or move it to default location with `python -m deeppavlov.settings settings -d` (all your configuration settings will be RESET to default ones).
 
 For `predict` you can specify path to input file with `-f` or `--input-file` parameter, otherwise, data will be taken
 from stdin.  

deeppavlov/agents/ecommerce_agent/ecommerce_agent.py

@@ -23,7 +23,7 @@
 from deeppavlov.core.commands.infer import build_model
 from deeppavlov.core.skill.skill import Skill
 from deeppavlov.deep import find_config
-from utils.ms_bot_framework_utils.server import run_ms_bot_framework_server
+from deeppavlov.utils.ms_bot_framework.server import run_ms_bot_framework_server
 
 parser = argparse.ArgumentParser()
 parser.add_argument("-i", "--ms-id", help="microsoft bot framework app id", type=str)

deeppavlov/core/common/paths.py

@@ -17,7 +17,7 @@
 from pathlib import Path
 
 _root_path = Path(__file__).resolve().parents[3]
-_default_settings_path: Path = _root_path / 'utils' / 'settings'
+_default_settings_path: Path = _root_path / 'deeppavlov' / 'utils' / 'settings'
 _settings_path = Path(os.getenv('DP_SETTINGS_PATH', _default_settings_path)).expanduser().resolve()
 if _settings_path.is_file():
     raise FileExistsError(f'DP_SETTINGS_PATH={_settings_path} is a file and not a directory')

deeppavlov/deep.py

@@ -22,12 +22,12 @@
 from deeppavlov.core.common.cross_validation import calc_cv_score
 from deeppavlov.core.common.file import find_config
 from deeppavlov.download import deep_download
-from utils.alexa.server import run_alexa_default_agent
-from utils.alice import start_alice_server
-from utils.ms_bot_framework_utils.server import run_ms_bf_default_agent
-from utils.pip_wrapper import install_from_config
-from utils.server_utils.server import start_model_server
-from utils.telegram_utils.telegram_ui import interact_model_by_telegram
+from deeppavlov.utils.alexa.server import run_alexa_default_agent
+from deeppavlov.utils.alice import start_alice_server
+from deeppavlov.utils.ms_bot_framework.server import run_ms_bf_default_agent
+from deeppavlov.utils.pip_wrapper import install_from_config
+from deeppavlov.utils.server.server import start_model_server
+from deeppavlov.utils.telegram.telegram_ui import interact_model_by_telegram
 
 log = getLogger(__name__)
 

utils/alexa/bot.py → deeppavlov/utils/alexa/bot.py

@@ -22,8 +22,8 @@
 from OpenSSL.crypto import X509
 
 from deeppavlov.agents.default_agent.default_agent import DefaultAgent
-from utils.alexa.conversation import Conversation
-from utils.alexa.ssl_tools import verify_cert, verify_signature
+from deeppavlov.utils.alexa.conversation import Conversation
+from deeppavlov.utils.alexa.ssl_tools import verify_cert, verify_signature
 
 REQUEST_TIMESTAMP_TOLERANCE_SECS = 150
 REFRESH_VALID_CERTS_PERIOD_SECS = 120

utils/alexa/server.py → deeppavlov/utils/alexa/server.py

@@ -29,7 +29,7 @@
 from deeppavlov.core.common.file import read_json
 from deeppavlov.core.common.paths import get_settings_path
 from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill
-from utils.alexa.bot import Bot
+from deeppavlov.utils.alexa.bot import Bot
 
 SERVER_CONFIG_FILENAME = 'server_config.json'
 

utils/alice/alice.py → deeppavlov/utils/alice/alice.py

@@ -29,7 +29,7 @@
 from deeppavlov.core.agent.rich_content import RichMessage
 from deeppavlov.core.common.paths import get_settings_path
 from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill
-from utils.server_utils.server import get_server_params
+from deeppavlov.utils.server.server import get_server_params
 
 SERVER_CONFIG_FILENAME = 'server_config.json'
 

utils/ms_bot_framework_utils/server.py → deeppavlov/utils/ms_bot_framework/server.py

@@ -14,7 +14,7 @@
 from deeppavlov.core.common.file import read_json
 from deeppavlov.core.common.paths import get_settings_path
 from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill
-from utils.ms_bot_framework_utils.bot import Bot
+from deeppavlov.utils.ms_bot_framework.bot import Bot
 
 SERVER_CONFIG_FILENAME = 'server_config.json'
 

utils/settings/log_config.json → deeppavlov/utils/settings/log_config.json

@@ -8,13 +8,6 @@
         "stderr"
       ],
       "propagate": true
-    },
-    "utils": {
-      "level": "DEBUG",
-      "handlers": [
-        "stderr"
-      ],
-      "propagate": true
     }
   },
   "formatters": {

docs/devguides/amazon_alexa.rst

@@ -179,7 +179,7 @@ with valid certificate from CA. `Here is the guide <https://developer.amazon.com
 for running custom skill service with self-signed certificates in test mode.
 
 Your intent and slot names defined in Alexa Development Console should be the same with values defined in
-DeepPavlov settings file ``utils/settings/server_config.json``. JSON examples from this guide use default values from
+DeepPavlov settings file ``deeppavlov/utils/settings/server_config.json``. JSON examples from this guide use default values from
 the settings file.
 
 DeepPavlov skill/component can be made available for Amazon Alexa as a REST service by:
@@ -204,5 +204,5 @@ for **each** conversation.
 You should use ``--no-default-skill`` optional flag if your component implements an interface of DeepPavlov *Skill*
 to skip its wrapping with DeepPavlov *DefaultStatelessSkill*.
 
-REST service properties (host, port, https options) are provided in ``utils/settings/server_config.json``. Please note,
+REST service properties (host, port, https options) are provided in ``deeppavlov/utils/settings/server_config.json``. Please note,
 that all command line parameters override corresponding config ones.

docs/devguides/ms_bot_integration.rst

@@ -104,7 +104,7 @@ for **each** conversation.
 You should use ``--no-default-skill`` optional flag if your component implements an interface of DeepPavlov *Skill*
 to skip its wrapping with DeepPavlov *DefaultStatelessSkill*.
 
-REST service properties (host, port) are provided in ``utils/settings/server_config.json``. You can also store your
+REST service properties (host, port) are provided in ``deeppavlov/utils/settings/server_config.json``. You can also store your
 app id and app secret in appropriate section of ``server_config.json``. Please note, that all command line parameters
 override corresponding config ones.
 

docs/devguides/rest_api.rst

@@ -9,7 +9,7 @@ inference as a REST web service. The general method is:
 (optional ``-d`` key is for dependencies download before service start)
 
 Web service properties (host, port, model endpoint, GET request
-arguments) are provided in ``utils/settings/server_config.json``,
+arguments) are provided in ``deeppavlov/utils/settings/server_config.json``,
 but port can be overridden with the ``-p`` key in command line.
 Properties from ``common_defaults`` section are used by default unless
 they are overridden by component-specific properties, provided in

docs/devguides/settings.rst

@@ -6,7 +6,7 @@ DeepPavlov provides some tools to facilitate its usage (e.g. dialog logging, set
 1. Settings files access and management
 ---------------------------------------
 
-Most of DeepPavlov settings are located in settings files, which in turn are located in a settings folder. Default settings folder location is ``utils/settings`` .
+Most of DeepPavlov settings are located in settings files, which in turn are located in a settings folder. Default settings folder location is ``deeppavlov/utils/settings`` .
 
 You can override a settings directory path by setting the ``DP_SETTINGS_PATH`` environment variable. Missing files will be added automatically when running any deeppavlov script.
 

docs/devguides/yandex_alice.rst

@@ -8,7 +8,7 @@ Pipelines
 
 Any DeepPavlov pipeline can be launched as a skill for Yandex.Alice.
 
-Configure host, port, model endpoint, GET request arguments in ``utils/server_config.json`` or see default values there.
+Configure host, port, model endpoint, GET request arguments in ``deeppavlov/utils/settings/server_config.json`` or see default values there.
 
 Use your own certificate for HTTPS if you have; otherwise, generate self-signed one like that:
 
@@ -42,7 +42,7 @@ You can also run :doc:`agents </apiref/agents>` as Alice skills:
     from deeppavlov.agents.default_agent.default_agent import DefaultAgent
     from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector
     from deeppavlov.skills.pattern_matching_skill import PatternMatchingSkill
-    from utils.alice import start_agent_server
+    from deeppavlov.utils.alice import start_agent_server
 
     skill_hello = PatternMatchingSkill(['Привет, мир!'], patterns=['привет', 'здравствуй', 'добрый день'])
     skill_bye = PatternMatchingSkill(['Пока, мир', 'Ещё увидимся'], patterns=['пока', 'чао', 'увидимся', 'до свидания'])

examples/tutorials/yandex_faq.ipynb

@@ -325,7 +325,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from utils.alice import start_agent_server\n",
+    "from deeppavlov.utils.alice import start_agent_server\n",
     "\n",
     "start_agent_server(agent, host='0.0.0.0', port=5000, endpoint='/faq', ssl_key='my.key', ssl_cert='my.crt')"
    ]

setup.py

@@ -45,7 +45,7 @@ def readme():
 
 setup(
     name='deeppavlov',
-    packages=find_packages(exclude=('tests', 'docs')),
+    packages=find_packages(exclude=('tests', 'docs', 'utils')),
     version=deeppavlov.__version__,
     description=deeppavlov.__description__,
     long_description=readme(),

tests/test_quick_start.py

@@ -22,7 +22,7 @@
 from deeppavlov.core.common.paths import get_settings_path
 from deeppavlov.core.data.utils import get_all_elems_from_json
 from deeppavlov.download import deep_download
-from utils.server_utils.server import get_server_params, SERVER_CONFIG_FILENAME
+from deeppavlov.utils.server.server import get_server_params, SERVER_CONFIG_FILENAME
 
 tests_dir = Path(__file__).parent
 test_configs_path = tests_dir / "deeppavlov" / "configs"