feat: Skills and agent moved to deprecated (deeppavlov#1040)

* feat: Flask was completely replaced by Fastapi

- Alexa, Alice and ms_bot_framework modes now on FastAPI
- Flask removed from requirements
- Docs endpoint name was removed from REST server config file

* feat: added async to riseapi model and probe endpoints and to alice

* feat: added async to riseapi

* docs: added license and docstrings to data models of Alice ans Alexa

* refactor: skills and agent moved to utils.deprecated

* fix: removed data_model files form Alice and Alexa

* refactor: deprecated moved from utils to deeppavlov

Aiml and rasa skills moved from deprecated

* refactor: ms_bot's framework and alexa bots got parent class bot

agent generator moved to bot parent class

* feat: removed multi-instance bot option, run method moved to BaseBot

* refactor: OutGateway class removed, methods put to ms_bot_framework conv

* feat: bot attribute removed from ms_bot conversation class

* feat: common parent class for msbot and alexa conversations

timers moved to BaseConversation class

* refactor: removed multiple sessions from ms_bot

* ms_bot auth params moved to server config file

* fix: type `redirect_root_do_docs` in server.server

* refactor: removed `stateful` param from everywhere

* feat: removed default_skill_wrap parameter from everywhere

* feat: removed deprecated from Alexa connector

* feat: alexa works

* refactor: bots and conversations moved to utils.connector dir

* refactor: Bot added to Alice (removed skills, agent)

* refactor: removed deprecated structures from telegram connector

* refactor: configs refactored, server code simplified

* refactor: configs in bots and conversations

* refactor: conversations, bots

* refactor: dsl_skill moved from deprecated

* refactor: DialoLogger removed from deprecated

Dialog logging added to channel connectors

* refatcor: removed deprecated components from not deprecated skills

* refactor: Connectors docs and interfaces

* fix: fixed ConnectionResetError for requests in one session

* fix: fixed aiml and dsl tests

removed unnecessary imports

* refactor: deep.py, fix small typos

* docs: added breaking changes for 0.7.0

* docs: added links to readme.md

Co-Authored-By: Aleksei Lymar <[email protected]>

* docs: changes in breaking changes

* docs: removed irrelevant breaking change

* fix: wrong argument parsing in deep.py

* refactor: reduced line length

README.md

@@ -216,6 +216,18 @@ and others in the Integrations section for more info.
 
 ## Breaking Changes
 
+**Breaking changes in version 0.7.0**
+- in dialog logger config file [dialog_logger_config.json](deeppavlov/utils/settings/dialog_logger_config.json) `agent_name` parameter was renamed to `logger_name`,
+  the default value was changed
+- Agent, Skill, eCommerce Bot and Pattern Matching classes were moved to [deeppavlov.deprecated](deeppavlov/deprecated/)
+- [AIML Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/aiml_skill.html),
+  [RASA Skill](http://docs.deeppavlov.ai/en/0.7.0/features/skills/rasa_skill.html),
+  [Yandex Alice](http://docs.deeppavlov.ai/en/0.7.0/integrations/yandex_alice.html),
+  [Amazon Alexa](http://docs.deeppavlov.ai/en/0.7.0/integrations/amazon_alexa.html),
+  [Microsoft Bot Framework](http://docs.deeppavlov.ai/en/0.7.0/integrations/ms_bot.html) and
+  [Telegram integration](http://docs.deeppavlov.ai/en/0.7.0/integrations/telegram.html) interfaces were changed
+- `/start` and `/help` Telegram messages were moved from `models_info.json` to [connector_config.json](/deeppavlov/utils/settings/connector_config.json)
+
 **Breaking changes in version 0.6.0**
 - [REST API](http://docs.deeppavlov.ai/en/0.6.0/integrations/rest_api.html):
   - all models default endpoints were renamed to `/model`

deeppavlov/__init__.py

@@ -37,7 +37,7 @@ def evaluate_model(config: [str, Path, dict], download: bool = False, recursive:
 except ImportError:
     'Assuming that requirements are not yet installed'
 
-__version__ = '0.6.1'
+__version__ = '0.7.0'
 __author__ = 'Neural Networks and Deep Learning lab, MIPT'
 __description__ = 'An open source library for building end-to-end dialog systems and training chatbots.'
 __keywords__ = ['NLP', 'NER', 'SQUAD', 'Intents', 'Chatbot']

deeppavlov/configs/skills/aiml_skill.json

@@ -2,7 +2,6 @@
   "chainer": {
     "in": [
       "utterances_batch",
-      "history_batch",
       "states_batch"
     ],
     "out": [
@@ -19,7 +18,6 @@
         "null_confidence": 0.33,
         "in": [
           "utterances_batch",
-          "history_batch",
           "states_batch"
         ],
         "out": [

deeppavlov/core/common/chainer.py

@@ -28,7 +28,7 @@
 
 class Chainer(Component):
     """
-    Builds an agent/component pipeline from heterogeneous components (Rule-based/ML/DL). It allows to train
+    Builds a component pipeline from heterogeneous components (Rule-based/ML/DL). It allows to train
     and infer models in a pipeline as a whole.
 
     Attributes:

deeppavlov/core/common/registry.json

@@ -40,8 +40,8 @@
   "dstc2_reader": "deeppavlov.dataset_readers.dstc2_reader:DSTC2DatasetReader",
   "dstc_slotfilling": "deeppavlov.models.slotfill.slotfill:DstcSlotFillingNetwork",
   "ecommerce_preprocess": "deeppavlov.models.preprocessors.ecommerce_preprocess:EcommercePreprocess",
-  "ecommerce_skill_bleu": "deeppavlov.skills.ecommerce_skill.bleu_retrieve:EcommerceSkillBleu",
-  "ecommerce_skill_tfidf": "deeppavlov.skills.ecommerce_skill.tfidf_retrieve:EcommerceSkillTfidf",
+  "ecommerce_skill_bleu": "deeppavlov.deprecated.skills.ecommerce_skill.bleu_retrieve:EcommerceSkillBleu",
+  "ecommerce_skill_tfidf": "deeppavlov.deprecated.skills.ecommerce_skill.tfidf_retrieve:EcommerceSkillTfidf",
   "elmo_embedder": "deeppavlov.models.embedders.elmo_embedder:ELMoEmbedder",
   "elmo_file_paths_iterator": "deeppavlov.dataset_iterators.elmo_file_paths_iterator:ELMoFilePathsIterator",
   "elmo_model": "deeppavlov.models.elmo.elmo:ELMo",

deeppavlov/deep.py

@@ -1,18 +1,16 @@
-"""
-Copyright 2017 Neural Networks and Deep Learning lab, MIPT
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-"""
+# Copyright 2017 Neural Networks and Deep Learning lab, MIPT
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 import argparse
 from logging import getLogger
@@ -22,21 +20,21 @@
 from deeppavlov.core.common.cross_validation import calc_cv_score
 from deeppavlov.core.common.file import find_config
 from deeppavlov.download import deep_download
-from deeppavlov.utils.alexa.server import run_alexa_default_agent
+from deeppavlov.utils.alexa import start_alexa_server
 from deeppavlov.utils.alice import start_alice_server
-from deeppavlov.utils.ms_bot_framework.server import run_ms_bf_default_agent
+from deeppavlov.utils.ms_bot_framework import start_ms_bf_server
 from deeppavlov.utils.pip_wrapper import install_from_config
-from deeppavlov.utils.server.server import start_model_server
-from deeppavlov.utils.socket.socket import start_socket_server
-from deeppavlov.utils.telegram.telegram_ui import interact_model_by_telegram
+from deeppavlov.utils.server import start_model_server
+from deeppavlov.utils.socket import start_socket_server
+from deeppavlov.utils.telegram import interact_model_by_telegram
 
 log = getLogger(__name__)
 
 parser = argparse.ArgumentParser()
 
 parser.add_argument("mode", help="select a mode, train or interact", type=str,
                     choices={'train', 'evaluate', 'interact', 'predict', 'interactbot', 'interactmsbot',
-                             'alexa', 'riseapi', 'risesocket', 'download', 'install', 'crossval'})
+                             'alexa', 'alice', 'riseapi', 'risesocket', 'download', 'install', 'crossval'})
 parser.add_argument("config_path", help="path to a pipeline json config", type=str)
 
 parser.add_argument("-e", "--start-epoch-num", dest="start_epoch_num", default=None,
@@ -50,79 +48,59 @@
 parser.add_argument("--folds", help="number of folds", type=int, default=5)
 
 parser.add_argument("-t", "--token", default=None,  help="telegram bot token", type=str)
+
 parser.add_argument("-i", "--ms-id", default=None, help="microsoft bot framework app id", type=str)
 parser.add_argument("-s", "--ms-secret", default=None, help="microsoft bot framework app secret", type=str)
 
-parser.add_argument("--multi-instance", action="store_true", help="allow rising of several instances of the model")
-parser.add_argument("--stateful", action="store_true", help="interact with a stateful model")
-parser.add_argument("--no-default-skill", action="store_true", help="do not wrap with default skill")
-
-parser.add_argument("--https", action="store_true", help="run model in https mode")
+parser.add_argument("--https", action="store_true", default=None, help="run model in https mode")
 parser.add_argument("--key", default=None, help="ssl key", type=str)
 parser.add_argument("--cert", default=None, help="ssl certificate", type=str)
 
 parser.add_argument("-p", "--port", default=None, help="api port", type=int)
+
 parser.add_argument("--socket-type", default='TCP', type=str, choices={"TCP", "UNIX"})
 parser.add_argument("--socket-file", default="/tmp/deeppavlov_socket.s", type=str)
 
-parser.add_argument("--api-mode", help="rest api mode: 'basic' with batches or 'alice' for  Yandex.Dialogs format",
-                    type=str, default='basic', choices={'basic', 'alice'})
-
 
 def main():
     args = parser.parse_args()
-
     pipeline_config_path = find_config(args.config_path)
-    https = args.https
-    ssl_key = args.key
-    ssl_cert = args.cert
 
     if args.download or args.mode == 'download':
         deep_download(pipeline_config_path)
 
-    multi_instance = args.multi_instance
-    stateful = args.stateful
-
     if args.mode == 'train':
-        train_evaluate_model_from_config(pipeline_config_path, recursive=args.recursive, 
+        train_evaluate_model_from_config(pipeline_config_path,
+                                         recursive=args.recursive,
                                          start_epoch_num=args.start_epoch_num)
     elif args.mode == 'evaluate':
         train_evaluate_model_from_config(pipeline_config_path, to_train=False, start_epoch_num=args.start_epoch_num)
     elif args.mode == 'interact':
         interact_model(pipeline_config_path)
     elif args.mode == 'interactbot':
-        token = args.token
-        interact_model_by_telegram(model_config=pipeline_config_path,
-                                   token=token,
-                                   default_skill_wrap=not args.no_default_skill)
+        interact_model_by_telegram(model_config=pipeline_config_path, token=args.token)
     elif args.mode == 'interactmsbot':
-        ms_id = args.ms_id
-        ms_secret = args.ms_secret
-        run_ms_bf_default_agent(model_config=pipeline_config_path,
-                                app_id=ms_id,
-                                app_secret=ms_secret,
-                                multi_instance=multi_instance,
-                                stateful=stateful,
-                                port=args.port,
-                                https=https,
-                                ssl_key=ssl_key,
-                                ssl_cert=ssl_cert,
-                                default_skill_wrap=not args.no_default_skill)
+        start_ms_bf_server(model_config=pipeline_config_path,
+                           app_id=args.ms_id,
+                           app_secret=args.ms_secret,
+                           port=args.port,
+                           https=args.https,
+                           ssl_key=args.key,
+                           ssl_cert=args.cert)
     elif args.mode == 'alexa':
-        run_alexa_default_agent(model_config=pipeline_config_path,
-                                multi_instance=multi_instance,
-                                stateful=stateful,
-                                port=args.port,
-                                https=https,
-                                ssl_key=ssl_key,
-                                ssl_cert=ssl_cert,
-                                default_skill_wrap=not args.no_default_skill)
+        start_alexa_server(model_config=pipeline_config_path,
+                           port=args.port,
+                           https=args.https,
+                           ssl_key=args.key,
+                           ssl_cert=args.cert)
+    elif args.mode == 'alice':
+        start_alice_server(model_config=pipeline_config_path,
+                           port=args.port,
+                           https=args.https,
+                           ssl_key=args.key,
+                           ssl_cert=args.cert)
     elif args.mode == 'riseapi':
-        alice = args.api_mode == 'alice'
-        if alice:
-            start_alice_server(pipeline_config_path, https, ssl_key, ssl_cert, port=args.port)
-        else:
-            start_model_server(pipeline_config_path, https, ssl_key, ssl_cert, port=args.port)
+        start_model_server(pipeline_config_path, args.https, args.key, args.cert, port=args.port)
     elif args.mode == 'risesocket':
         start_socket_server(pipeline_config_path, args.socket_type, port=args.port, socket_file=args.socket_file)
     elif args.mode == 'predict':
@@ -133,8 +111,7 @@ def main():
         if args.folds < 2:
             log.error('Minimum number of Folds is 2')
         else:
-            n_folds = args.folds
-            calc_cv_score(pipeline_config_path, n_folds=n_folds, is_loo=False)
+            calc_cv_score(pipeline_config_path, n_folds=args.folds, is_loo=False)
 
 
 if __name__ == "__main__":

deeppavlov/deprecated/agent/__init__.py

@@ -0,0 +1,4 @@
+from .agent import Agent, SkillWrapper
+from .filter import Filter
+from .processor import Processor
+from .rich_content import RichControl, RichMessage

deeppavlov/core/agent/agent.py → deeppavlov/deprecated/agent/agent.py

@@ -14,10 +14,10 @@
 
 from abc import ABCMeta, abstractmethod
 from collections import defaultdict
-from typing import List, Dict, Tuple, Optional, Union
+from typing import List, Dict, Tuple, Optional
 
-from deeppavlov.core.agent.dialog_logger import DialogLogger
 from deeppavlov.core.models.component import Component
+from deeppavlov.utils.connector.dialog_logger import DialogLogger
 
 
 class Agent(Component, metaclass=ABCMeta):

deeppavlov/deprecated/agents/default_agent/__init__.py

@@ -0,0 +1 @@
+from .default_agent import DefaultAgent

deeppavlov/agents/default_agent/default_agent.py → deeppavlov/deprecated/agents/default_agent/default_agent.py

@@ -14,11 +14,9 @@
 
 from typing import List, Optional
 
-from deeppavlov.agents.filters.transparent_filter import TransparentFilter
-from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector
-from deeppavlov.core.agent.agent import Agent
-from deeppavlov.core.agent.filter import Filter
-from deeppavlov.core.agent.processor import Processor
+from deeppavlov.deprecated.agent import Agent, Filter, Processor
+from deeppavlov.deprecated.agents.filters import TransparentFilter
+from deeppavlov.deprecated.agents.processors import HighestConfidenceSelector
 from deeppavlov.core.models.component import Component
 
 
@@ -35,7 +33,8 @@ class DefaultAgent(Agent):
     a) To define set of skills it uses;
     b) To implement skills Filter;
     c) To implement Processor.
-    You can refer to :class:`deeppavlov.core.skill.Skill`, :class:`deeppavlov.core.agent.Filter`, :class:`deeppavlov.core.agent.Processor` base classes to get more info.
+    You can refer to :class:`deeppavlov.deprecated.skill.Skill`, :class:`deeppavlov.deprecated.agent.Filter`,
+    :class:`deeppavlov.deprecated.agent.Processor` base classes to get more info.
 
     Args:
         skills: List of initiated agent skills or components instances.
@@ -53,7 +52,7 @@ def __init__(self, skills: List[Component], skills_processor: Optional[Processor
         self.skills_filter = skills_filter or TransparentFilter(len(skills))
         self.skills_processor = skills_processor or HighestConfidenceSelector()
 
-    def _call(self, utterances_batch: list, utterances_ids: Optional[list]=None) -> list:
+    def _call(self, utterances_batch: list, utterances_ids: Optional[list] = None) -> list:
         """
         Processes batch of utterances and returns corresponding responses batch.
 

deeppavlov/deprecated/agents/ecommerce_agent/__init__.py

@@ -0,0 +1 @@
+from .ecommerce_agent import EcommerceAgent

deeppavlov/agents/ecommerce_agent/ecommerce_agent.py → deeppavlov/deprecated/agents/ecommerce_agent/ecommerce_agent.py

@@ -17,12 +17,11 @@
 from logging import getLogger
 from typing import List, Dict, Any
 
-from deeppavlov.agents.rich_content.default_rich_content import PlainText, ButtonsFrame, Button
-from deeppavlov.core.agent.agent import Agent
-from deeppavlov.core.agent.rich_content import RichMessage
 from deeppavlov.core.commands.infer import build_model
-from deeppavlov.core.skill.skill import Skill
 from deeppavlov.deep import find_config
+from deeppavlov.deprecated.agent import Agent, RichMessage
+from deeppavlov.deprecated.agents.rich_content import PlainText, ButtonsFrame, Button
+from deeppavlov.deprecated.skill import Skill
 from deeppavlov.utils.ms_bot_framework.server import run_ms_bot_framework_server
 
 parser = argparse.ArgumentParser()
@@ -192,8 +191,7 @@ def main():
     args = parser.parse_args()
     run_ms_bot_framework_server(agent_generator=make_agent,
                                 app_id=args.ms_id,
-                                app_secret=args.ms_secret,
-                                stateful=True)
+                                app_secret=args.ms_secret)
 
 
 if __name__ == '__main__':

deeppavlov/deprecated/agents/filters/__init__.py

@@ -0,0 +1 @@
+from .transparent_filter import TransparentFilter

deeppavlov/agents/filters/transparent_filter.py → deeppavlov/deprecated/agents/filters/transparent_filter.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from deeppavlov.core.agent.filter import Filter
+from deeppavlov.deprecated.agent import Filter
 
 
 class TransparentFilter(Filter):

deeppavlov/agents/hello_bot_agent/hello_bot_agent.py → deeppavlov/deprecated/agents/hello_bot_agent/hello_bot_agent.py

@@ -12,9 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from deeppavlov.agents.default_agent.default_agent import DefaultAgent
-from deeppavlov.agents.processors.highest_confidence_selector import HighestConfidenceSelector
-from deeppavlov.skills.pattern_matching_skill import PatternMatchingSkill
+from deeppavlov.deprecated.agents.default_agent import DefaultAgent
+from deeppavlov.deprecated.agents.processors import HighestConfidenceSelector
+from deeppavlov.deprecated.skills.pattern_matching_skill import PatternMatchingSkill
 
 
 def make_hello_bot_agent() -> DefaultAgent:

deeppavlov/deprecated/agents/processors/__init__.py

@@ -0,0 +1,3 @@
+from .default_rich_content_processor import DefaultRichContentWrapper
+from .highest_confidence_selector import HighestConfidenceSelector
+from .random_selector import RandomSelector

deeppavlov/agents/processors/default_rich_content_processor.py → deeppavlov/deprecated/agents/processors/default_rich_content_processor.py

@@ -12,9 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from deeppavlov.agents.rich_content.default_rich_content import PlainText
-from deeppavlov.core.agent.processor import Processor
-from deeppavlov.core.agent.rich_content import RichMessage
+from deeppavlov.deprecated.agent import Processor, RichMessage
+from deeppavlov.deprecated.agents.rich_content import PlainText
 
 
 class DefaultRichContentWrapper(Processor):

deeppavlov/agents/processors/highest_confidence_selector.py → deeppavlov/deprecated/agents/processors/highest_confidence_selector.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-from deeppavlov.core.agent.processor import Processor
+from deeppavlov.deprecated.agent import Processor
 
 
 class HighestConfidenceSelector(Processor):

deeppavlov/agents/processors/random_selector.py → deeppavlov/deprecated/agents/processors/random_selector.py

@@ -14,7 +14,7 @@
 
 import random
 
-from deeppavlov.core.agent.processor import Processor
+from deeppavlov.deprecated.agent import Processor
 
 
 class RandomSelector(Processor):

deeppavlov/deprecated/agents/rich_content/__init__.py

@@ -0,0 +1 @@
+from .default_rich_content import Button, ButtonsFrame, PlainText

deeppavlov/agents/rich_content/default_rich_content.py → deeppavlov/deprecated/agents/rich_content/default_rich_content.py

@@ -14,7 +14,7 @@
 
 from typing import Optional
 
-from deeppavlov.core.agent.rich_content import RichControl
+from deeppavlov.deprecated.agent import RichControl
 
 
 class PlainText(RichControl):

deeppavlov/deprecated/skill/__init__.py

@@ -0,0 +1 @@
+from .skill import Skill

deeppavlov/deprecated/skills/default_skill/__init__.py

@@ -0,0 +1 @@
+from .default_skill import DefaultStatelessSkill