From 7bad1909bb61e5ed30a62f6b59dd7e1371e4b212 Mon Sep 17 00:00:00 2001 From: Eugene Golikov Date: Wed, 20 Mar 2019 21:43:52 +0300 Subject: [PATCH] refactor: move utils to deeppavlov (#744) * refactor: move utils to deeppavlov * fix: wrong path to utils; refactor: rename telegram_utils to telegram * refactor: rename all occurences of ms_bot_framework_utils to ms_bot_framework and all occurences of server_utils to server * Update README.md You skipped one changing paths `utils/ -> deeppavlov/utils/` occurrence. * fix: replace telegram to telegram_utils and server to server_utils in .json files * fix: replace 'server' with 'server_utils' in deeppavlov/utils/server/server.py * fix: replace 'server' with 'server_utils' and 'telegram' with 'telegram_utils' back in configs and README.md * fix: remove 'utils' from deeppavlov/utils/settings/log_config.json --- MANIFEST.in | 1 - README.md | 8 ++++---- deeppavlov/agents/ecommerce_agent/ecommerce_agent.py | 2 +- deeppavlov/core/common/paths.py | 2 +- deeppavlov/deep.py | 12 ++++++------ {utils => deeppavlov/utils}/__init__.py | 0 {utils => deeppavlov/utils}/alexa/__init__.py | 0 {utils => deeppavlov/utils}/alexa/bot.py | 4 ++-- {utils => deeppavlov/utils}/alexa/conversation.py | 0 {utils => deeppavlov/utils}/alexa/server.py | 2 +- {utils => deeppavlov/utils}/alexa/ssl_tools.py | 0 {utils => deeppavlov/utils}/alice/__init__.py | 0 {utils => deeppavlov/utils}/alice/alice.py | 2 +- .../utils/ms_bot_framework}/__init__.py | 0 .../utils/ms_bot_framework}/bot.py | 0 .../utils/ms_bot_framework}/conversation.py | 0 .../utils/ms_bot_framework}/server.py | 2 +- {utils => deeppavlov/utils}/pip_wrapper/__init__.py | 0 .../utils}/pip_wrapper/pip_wrapper.py | 0 .../utils/server}/__init__.py | 0 .../utils/server}/server.py | 0 {utils => deeppavlov/utils}/settings/__init__.py | 0 .../utils}/settings/dialog_logger_config.json | 0 {utils => deeppavlov/utils}/settings/log_config.json | 7 ------- .../utils}/settings/models_info.json | 0 .../utils}/settings/server_config.json | 0 .../utils/telegram}/__init__.py | 0 .../utils/telegram}/telegram_ui.py | 0 docs/devguides/amazon_alexa.rst | 4 ++-- docs/devguides/ms_bot_integration.rst | 2 +- docs/devguides/rest_api.rst | 2 +- docs/devguides/settings.rst | 2 +- docs/devguides/yandex_alice.rst | 4 ++-- examples/tutorials/yandex_faq.ipynb | 2 +- setup.py | 2 +- tests/test_quick_start.py | 2 +- 36 files changed, 27 insertions(+), 35 deletions(-) rename {utils => deeppavlov/utils}/__init__.py (100%) rename {utils => deeppavlov/utils}/alexa/__init__.py (100%) rename {utils => deeppavlov/utils}/alexa/bot.py (98%) rename {utils => deeppavlov/utils}/alexa/conversation.py (100%) rename {utils => deeppavlov/utils}/alexa/server.py (99%) rename {utils => deeppavlov/utils}/alexa/ssl_tools.py (100%) rename {utils => deeppavlov/utils}/alice/__init__.py (100%) rename {utils => deeppavlov/utils}/alice/alice.py (99%) rename {utils/ms_bot_framework_utils => deeppavlov/utils/ms_bot_framework}/__init__.py (100%) rename {utils/ms_bot_framework_utils => deeppavlov/utils/ms_bot_framework}/bot.py (100%) rename {utils/ms_bot_framework_utils => deeppavlov/utils/ms_bot_framework}/conversation.py (100%) rename {utils/ms_bot_framework_utils => deeppavlov/utils/ms_bot_framework}/server.py (99%) rename {utils => deeppavlov/utils}/pip_wrapper/__init__.py (100%) rename {utils => deeppavlov/utils}/pip_wrapper/pip_wrapper.py (100%) rename {utils/server_utils => deeppavlov/utils/server}/__init__.py (100%) rename {utils/server_utils => deeppavlov/utils/server}/server.py (100%) rename {utils => deeppavlov/utils}/settings/__init__.py (100%) rename {utils => deeppavlov/utils}/settings/dialog_logger_config.json (100%) rename {utils => deeppavlov/utils}/settings/log_config.json (88%) rename {utils => deeppavlov/utils}/settings/models_info.json (100%) rename {utils => deeppavlov/utils}/settings/server_config.json (100%) rename {utils/telegram_utils => deeppavlov/utils/telegram}/__init__.py (100%) rename {utils/telegram_utils => deeppavlov/utils/telegram}/telegram_ui.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index f28a9d4e47..662c28035e 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,4 +4,3 @@ include requirements.txt include deeppavlov/requirements/*.txt recursive-include deeppavlov *.json recursive-include deeppavlov *.md -recursive-include utils *.json \ No newline at end of file diff --git a/README.md b/README.md index 63501b5634..e740424fc0 100644 --- a/README.md +++ b/README.md @@ -139,17 +139,17 @@ or a name without the `.json` extension of one of the config files [provided](de For the `interactbot` mode you should specify Telegram bot token in `-t` parameter or in `TELEGRAM_TOKEN` environment variable. Also you should use `--no-default-skill` optional flag if your component implements an interface of DeepPavlov [*Skill*](deeppavlov/core/skill/skill.py) to skip its wrapping with DeepPavlov [*DefaultStatelessSkill*](deeppavlov/skills/default_skill/default_skill.py). If you want to get custom `/start` and `/help` Telegram messages for the running model you should: -* Add section to [*utils/settings/models_info.json*](utils/settings/models_info.json) with your custom Telegram messages -* In model config file specify `metadata.labels.telegram_utils` parameter with name which refers to the added section of [*utils/settings/models_info.json*](utils/settings/models_info.json) +* Add section to [*deeppavlov/utils/settings/models_info.json*](deeppavlov/utils/settings/models_info.json) with your custom Telegram messages +* In model config file specify `metadata.labels.telegram_utils` parameter with name which refers to the added section of [*deeppavlov/utils/settings/models_info.json*](deeppavlov/utils/settings/models_info.json) You can also serve DeepPavlov models for: * Microsoft Bot Framework ([see developer guide for the detailed instructions](http://docs.deeppavlov.ai/en/latest/devguides/ms_bot_integration.html)) * Amazon Alexa ([see developer guide for the detailed instructions](http://docs.deeppavlov.ai/en/latest/devguides/amazon_alexa.html)) -For `riseapi` mode you should specify api settings (host, port, etc.) in [*utils/settings/server_config.json*](utils/settings/server_config.json) configuration file. If provided, values from *model_defaults* section override values for the same parameters from *common_defaults* section. Model names in *model_defaults* section should be similar to the class names of the models main component. +For `riseapi` mode you should specify api settings (host, port, etc.) in [*deeppavlov/utils/settings/server_config.json*](deeppavlov/utils/settings/server_config.json) configuration file. If provided, values from *model_defaults* section override values for the same parameters from *common_defaults* section. Model names in *model_defaults* section should be similar to the class names of the models main component. Here is [detailed info on the DeepPavlov REST API](http://docs.deeppavlov.ai/en/latest/devguides/rest_api.html) -All DeepPavlov settings files are stored in `utils/settings` by default. You can get full path to it with `python -m deeppavlov.settings settings`. Also you can move it with with `python -m deeppavlov.settings settings -p ` (all your configuration settings will be preserved) or move it to default location with `python -m deeppavlov.settings settings -d` (all your configuration settings will be RESET to default ones). +All DeepPavlov settings files are stored in `deeppavlov/utils/settings` by default. You can get full path to it with `python -m deeppavlov.settings settings`. Also you can move it with with `python -m deeppavlov.settings settings -p ` (all your configuration settings will be preserved) or move it to default location with `python -m deeppavlov.settings settings -d` (all your configuration settings will be RESET to default ones). For `predict` you can specify path to input file with `-f` or `--input-file` parameter, otherwise, data will be taken from stdin. diff --git a/deeppavlov/agents/ecommerce_agent/ecommerce_agent.py b/deeppavlov/agents/ecommerce_agent/ecommerce_agent.py index c793645ef4..8f7fe615ea 100644 --- a/deeppavlov/agents/ecommerce_agent/ecommerce_agent.py +++ b/deeppavlov/agents/ecommerce_agent/ecommerce_agent.py @@ -23,7 +23,7 @@ from deeppavlov.core.commands.infer import build_model from deeppavlov.core.skill.skill import Skill from deeppavlov.deep import find_config -from utils.ms_bot_framework_utils.server import run_ms_bot_framework_server +from deeppavlov.utils.ms_bot_framework.server import run_ms_bot_framework_server parser = argparse.ArgumentParser() parser.add_argument("-i", "--ms-id", help="microsoft bot framework app id", type=str) diff --git a/deeppavlov/core/common/paths.py b/deeppavlov/core/common/paths.py index 3037edf393..b0dcee2fe3 100644 --- a/deeppavlov/core/common/paths.py +++ b/deeppavlov/core/common/paths.py @@ -17,7 +17,7 @@ from pathlib import Path _root_path = Path(__file__).resolve().parents[3] -_default_settings_path: Path = _root_path / 'utils' / 'settings' +_default_settings_path: Path = _root_path / 'deeppavlov' / 'utils' / 'settings' _settings_path = Path(os.getenv('DP_SETTINGS_PATH', _default_settings_path)).expanduser().resolve() if _settings_path.is_file(): raise FileExistsError(f'DP_SETTINGS_PATH={_settings_path} is a file and not a directory') diff --git a/deeppavlov/deep.py b/deeppavlov/deep.py index 137fcec1a9..4dd73302ea 100644 --- a/deeppavlov/deep.py +++ b/deeppavlov/deep.py @@ -22,12 +22,12 @@ from deeppavlov.core.common.cross_validation import calc_cv_score from deeppavlov.core.common.file import find_config from deeppavlov.download import deep_download -from utils.alexa.server import run_alexa_default_agent -from utils.alice import start_alice_server -from utils.ms_bot_framework_utils.server import run_ms_bf_default_agent -from utils.pip_wrapper import install_from_config -from utils.server_utils.server import start_model_server -from utils.telegram_utils.telegram_ui import interact_model_by_telegram +from deeppavlov.utils.alexa.server import run_alexa_default_agent +from deeppavlov.utils.alice import start_alice_server +from deeppavlov.utils.ms_bot_framework.server import run_ms_bf_default_agent +from deeppavlov.utils.pip_wrapper import install_from_config +from deeppavlov.utils.server.server import start_model_server +from deeppavlov.utils.telegram.telegram_ui import interact_model_by_telegram log = getLogger(__name__) diff --git a/utils/__init__.py b/deeppavlov/utils/__init__.py similarity index 100% rename from utils/__init__.py rename to deeppavlov/utils/__init__.py diff --git a/utils/alexa/__init__.py b/deeppavlov/utils/alexa/__init__.py similarity index 100% rename from utils/alexa/__init__.py rename to deeppavlov/utils/alexa/__init__.py diff --git a/utils/alexa/bot.py b/deeppavlov/utils/alexa/bot.py similarity index 98% rename from utils/alexa/bot.py rename to deeppavlov/utils/alexa/bot.py index f6978fa1e1..62926f84bf 100644 --- a/utils/alexa/bot.py +++ b/deeppavlov/utils/alexa/bot.py @@ -22,8 +22,8 @@ from OpenSSL.crypto import X509 from deeppavlov.agents.default_agent.default_agent import DefaultAgent -from utils.alexa.conversation import Conversation -from utils.alexa.ssl_tools import verify_cert, verify_signature +from deeppavlov.utils.alexa.conversation import Conversation +from deeppavlov.utils.alexa.ssl_tools import verify_cert, verify_signature REQUEST_TIMESTAMP_TOLERANCE_SECS = 150 REFRESH_VALID_CERTS_PERIOD_SECS = 120 diff --git a/utils/alexa/conversation.py b/deeppavlov/utils/alexa/conversation.py similarity index 100% rename from utils/alexa/conversation.py rename to deeppavlov/utils/alexa/conversation.py diff --git a/utils/alexa/server.py b/deeppavlov/utils/alexa/server.py similarity index 99% rename from utils/alexa/server.py rename to deeppavlov/utils/alexa/server.py index facf2c4e05..8164c10034 100644 --- a/utils/alexa/server.py +++ b/deeppavlov/utils/alexa/server.py @@ -29,7 +29,7 @@ from deeppavlov.core.common.file import read_json from deeppavlov.core.common.paths import get_settings_path from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from utils.alexa.bot import Bot +from deeppavlov.utils.alexa.bot import Bot SERVER_CONFIG_FILENAME = 'server_config.json' diff --git a/utils/alexa/ssl_tools.py b/deeppavlov/utils/alexa/ssl_tools.py similarity index 100% rename from utils/alexa/ssl_tools.py rename to deeppavlov/utils/alexa/ssl_tools.py diff --git a/utils/alice/__init__.py b/deeppavlov/utils/alice/__init__.py similarity index 100% rename from utils/alice/__init__.py rename to deeppavlov/utils/alice/__init__.py diff --git a/utils/alice/alice.py b/deeppavlov/utils/alice/alice.py similarity index 99% rename from utils/alice/alice.py rename to deeppavlov/utils/alice/alice.py index b53d7a6c42..d0ef655ded 100644 --- a/utils/alice/alice.py +++ b/deeppavlov/utils/alice/alice.py @@ -29,7 +29,7 @@ from deeppavlov.core.agent.rich_content import RichMessage from deeppavlov.core.common.paths import get_settings_path from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from utils.server_utils.server import get_server_params +from deeppavlov.utils.server.server import get_server_params SERVER_CONFIG_FILENAME = 'server_config.json' diff --git a/utils/ms_bot_framework_utils/__init__.py b/deeppavlov/utils/ms_bot_framework/__init__.py similarity index 100% rename from utils/ms_bot_framework_utils/__init__.py rename to deeppavlov/utils/ms_bot_framework/__init__.py diff --git a/utils/ms_bot_framework_utils/bot.py b/deeppavlov/utils/ms_bot_framework/bot.py similarity index 100% rename from utils/ms_bot_framework_utils/bot.py rename to deeppavlov/utils/ms_bot_framework/bot.py diff --git a/utils/ms_bot_framework_utils/conversation.py b/deeppavlov/utils/ms_bot_framework/conversation.py similarity index 100% rename from utils/ms_bot_framework_utils/conversation.py rename to deeppavlov/utils/ms_bot_framework/conversation.py diff --git a/utils/ms_bot_framework_utils/server.py b/deeppavlov/utils/ms_bot_framework/server.py similarity index 99% rename from utils/ms_bot_framework_utils/server.py rename to deeppavlov/utils/ms_bot_framework/server.py index 8ca9d2c3e9..82a6a9ebb2 100644 --- a/utils/ms_bot_framework_utils/server.py +++ b/deeppavlov/utils/ms_bot_framework/server.py @@ -14,7 +14,7 @@ from deeppavlov.core.common.file import read_json from deeppavlov.core.common.paths import get_settings_path from deeppavlov.skills.default_skill.default_skill import DefaultStatelessSkill -from utils.ms_bot_framework_utils.bot import Bot +from deeppavlov.utils.ms_bot_framework.bot import Bot SERVER_CONFIG_FILENAME = 'server_config.json' diff --git a/utils/pip_wrapper/__init__.py b/deeppavlov/utils/pip_wrapper/__init__.py similarity index 100% rename from utils/pip_wrapper/__init__.py rename to deeppavlov/utils/pip_wrapper/__init__.py diff --git a/utils/pip_wrapper/pip_wrapper.py b/deeppavlov/utils/pip_wrapper/pip_wrapper.py similarity index 100% rename from utils/pip_wrapper/pip_wrapper.py rename to deeppavlov/utils/pip_wrapper/pip_wrapper.py diff --git a/utils/server_utils/__init__.py b/deeppavlov/utils/server/__init__.py similarity index 100% rename from utils/server_utils/__init__.py rename to deeppavlov/utils/server/__init__.py diff --git a/utils/server_utils/server.py b/deeppavlov/utils/server/server.py similarity index 100% rename from utils/server_utils/server.py rename to deeppavlov/utils/server/server.py diff --git a/utils/settings/__init__.py b/deeppavlov/utils/settings/__init__.py similarity index 100% rename from utils/settings/__init__.py rename to deeppavlov/utils/settings/__init__.py diff --git a/utils/settings/dialog_logger_config.json b/deeppavlov/utils/settings/dialog_logger_config.json similarity index 100% rename from utils/settings/dialog_logger_config.json rename to deeppavlov/utils/settings/dialog_logger_config.json diff --git a/utils/settings/log_config.json b/deeppavlov/utils/settings/log_config.json similarity index 88% rename from utils/settings/log_config.json rename to deeppavlov/utils/settings/log_config.json index 0a42486186..b3d030928c 100644 --- a/utils/settings/log_config.json +++ b/deeppavlov/utils/settings/log_config.json @@ -8,13 +8,6 @@ "stderr" ], "propagate": true - }, - "utils": { - "level": "DEBUG", - "handlers": [ - "stderr" - ], - "propagate": true } }, "formatters": { diff --git a/utils/settings/models_info.json b/deeppavlov/utils/settings/models_info.json similarity index 100% rename from utils/settings/models_info.json rename to deeppavlov/utils/settings/models_info.json diff --git a/utils/settings/server_config.json b/deeppavlov/utils/settings/server_config.json similarity index 100% rename from utils/settings/server_config.json rename to deeppavlov/utils/settings/server_config.json diff --git a/utils/telegram_utils/__init__.py b/deeppavlov/utils/telegram/__init__.py similarity index 100% rename from utils/telegram_utils/__init__.py rename to deeppavlov/utils/telegram/__init__.py diff --git a/utils/telegram_utils/telegram_ui.py b/deeppavlov/utils/telegram/telegram_ui.py similarity index 100% rename from utils/telegram_utils/telegram_ui.py rename to deeppavlov/utils/telegram/telegram_ui.py diff --git a/docs/devguides/amazon_alexa.rst b/docs/devguides/amazon_alexa.rst index fade6dc97e..3c8ba8aa4f 100644 --- a/docs/devguides/amazon_alexa.rst +++ b/docs/devguides/amazon_alexa.rst @@ -179,7 +179,7 @@ with valid certificate from CA. `Here is the guide ` as Alice skills: from deeppavlov.agents.default_agent.default_agent import DefaultAgent from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector from deeppavlov.skills.pattern_matching_skill import PatternMatchingSkill - from utils.alice import start_agent_server + from deeppavlov.utils.alice import start_agent_server skill_hello = PatternMatchingSkill(['Привет, мир!'], patterns=['привет', 'здравствуй', 'добрый день']) skill_bye = PatternMatchingSkill(['Пока, мир', 'Ещё увидимся'], patterns=['пока', 'чао', 'увидимся', 'до свидания']) diff --git a/examples/tutorials/yandex_faq.ipynb b/examples/tutorials/yandex_faq.ipynb index 5a157ef7b2..c7f2485baf 100644 --- a/examples/tutorials/yandex_faq.ipynb +++ b/examples/tutorials/yandex_faq.ipynb @@ -325,7 +325,7 @@ "metadata": {}, "outputs": [], "source": [ - "from utils.alice import start_agent_server\n", + "from deeppavlov.utils.alice import start_agent_server\n", "\n", "start_agent_server(agent, host='0.0.0.0', port=5000, endpoint='/faq', ssl_key='my.key', ssl_cert='my.crt')" ] diff --git a/setup.py b/setup.py index 92bff64a59..87a55dbba4 100644 --- a/setup.py +++ b/setup.py @@ -45,7 +45,7 @@ def readme(): setup( name='deeppavlov', - packages=find_packages(exclude=('tests', 'docs')), + packages=find_packages(exclude=('tests', 'docs', 'utils')), version=deeppavlov.__version__, description=deeppavlov.__description__, long_description=readme(), diff --git a/tests/test_quick_start.py b/tests/test_quick_start.py index d3aceb1bba..35fca64845 100644 --- a/tests/test_quick_start.py +++ b/tests/test_quick_start.py @@ -22,7 +22,7 @@ from deeppavlov.core.common.paths import get_settings_path from deeppavlov.core.data.utils import get_all_elems_from_json from deeppavlov.download import deep_download -from utils.server_utils.server import get_server_params, SERVER_CONFIG_FILENAME +from deeppavlov.utils.server.server import get_server_params, SERVER_CONFIG_FILENAME tests_dir = Path(__file__).parent test_configs_path = tests_dir / "deeppavlov" / "configs"