fix agentfabric bugs

apps/agentfabric/app.py

@@ -621,11 +621,17 @@ def preview_send_message(chatbot, input, _state, uuid_str, _user_token):
 
         # get chat history from memory
         history = user_memory.get_history()
+        # skip
+        filtered_files = [
+            item for item in append_files
+            if not item.lower().endswith(('.jpeg', '.png', '.jpg', '.wav',
+                                          '.gif', '.mp3'))
+        ]
 
         use_llm = True if len(user_agent.function_list) else False
         ref_doc = user_memory.run(
             query=input.text,
-            url=append_files,
+            url=filtered_files,
             max_token=4000,
             top_k=2,
             checked=True,

apps/agentfabric/appBot.py

@@ -150,9 +150,19 @@ def send_message(chatbot, input, _state, _user_token):
         # get short term memory history
         history = user_memory.get_history()
 
+        # skip
+        filtered_files = [
+            item for item in append_files
+            if not item.lower().endswith(('.jpeg', '.png', '.jpg', '.wav',
+                                          '.gif', '.mp3'))
+        ]
+
         use_llm = True if len(user_agent.function_list) else False
         ref_doc = user_memory.run(
-            query=input.text, url=append_files, checked=True, use_llm=use_llm)
+            query=input.text,
+            url=filtered_files,
+            checked=True,
+            use_llm=use_llm)
 
         response = ''
         try:

apps/agentfabric/requirements.txt

@@ -1,7 +1,7 @@
 gradio==4.36.1
 markdown-cjk-spacing
 mdx_truly_sane_lists
-modelscope-agent>=0.6.6
+http://modelscope-agent.oss-cn-hangzhou.aliyuncs.com/releases/v0.7.4/modelscope_agent-0.7.4-py3-none-any.whl
 modelscope_studio>=0.4.0
 moviepy
 pymdown-extensions

apps/agentfabric/server.py

@@ -4,6 +4,7 @@
 import time
 import traceback
 from functools import wraps
+from werkzeug.utils import secure_filename
 
 import json
 import requests
@@ -93,7 +94,8 @@ def builder_chat(uuid_str):
     for file in files:
         ci_dir = get_user_ci_dir()
         os.makedirs(ci_dir, exist_ok=True)
-        file_path = os.path.join(ci_dir, uuid_str + '_' + file.filename)
+        secure_fn = secure_filename(file.filename)
+        file_path = os.path.join(ci_dir, uuid_str + '_' + secure_fn)
         if '../' in file_path:
             raise Exception('Access not allowed.')
         file.save(file_path)
@@ -125,7 +127,7 @@ def generate():
                         assert isinstance(exec_result, Config)
                         builder_cfg = exec_result.to_dict()
                         save_builder_configuration(builder_cfg, uuid_str)
-                        # app.session_manager.clear_user_bot(uuid_str)
+                        app.session_manager.clear_user_bot(uuid_str)
                         res = json.dumps(
                             {
                                 'data': response,
@@ -225,7 +227,8 @@ def import_builder(uuid_str):
             return jsonify({'error': 'No selected file'}), 400
 
         # 保存文件到服务器的文件系统
-        file_path = os.path.join(IMPORT_ZIP_TEMP_DIR, uuid_str, file.filename)
+        secure_fn = secure_filename(file.filename)
+        file_path = os.path.join(IMPORT_ZIP_TEMP_DIR, uuid_str, secure_fn)
         if '../' in file_path:
             raise Exception('Access not allowed.')
         os.makedirs(os.path.dirname(file_path), exist_ok=True)
@@ -318,7 +321,8 @@ def save_builder_config(uuid_str):
     if not os.path.exists(upload_dir):
         os.makedirs(upload_dir)
     for file in files:
-        file_path = os.path.join(upload_dir, file.filename)
+        secure_fn = secure_filename(file.filename)
+        file_path = os.path.join(upload_dir, secure_fn)
         if '../' in file_path:
             raise Exception('Access not allowed.')
         file.save(file_path)
@@ -353,7 +357,7 @@ def save_builder_config(uuid_str):
                 error=str(e),
                 details={'error_traceback': traceback.format_exc()})
     save_builder_configuration(builder_cfg=builder_config, uuid_str=uuid_str)
-    # app.session_manager.clear_user_bot(uuid_str)
+    app.session_manager.clear_user_bot(uuid_str)
 
     return jsonify({'success': True, 'request_id': request_id_var.get('')})
 
@@ -424,10 +428,11 @@ def generate():
                 f'load history method: time consumed {time.time() - start_time}'
             )
 
-            # skip image upsert
+            # skip
             filtered_files = [
                 item for item in file_paths
-                if not item.lower().endswith(('.jpeg', '.png', '.jpg'))
+                if not item.lower().endswith(('.jpeg', '.png', '.jpg', '.wav',
+                                              '.gif', '.mp3'))
             ]
 
             use_llm = True if len(user_agent.function_list) else False
@@ -495,7 +500,15 @@ def generate():
             stack_trace = stack_trace.replace('\n', '\\n')
             logger.error(
                 f'preview_chat_generate_error: {str(e)}, {stack_trace}')
-            raise e
+            error_info = f'data: Please check your configuration and try again. Error: {str(e)}, {stack_trace}\n\n'
+            res = json.dumps(
+                {
+                    'data': error_info,
+                    'is_final': True,
+                    'request_id': request_id_var.get('')
+                },
+                ensure_ascii=False)
+            yield f'data: {res}\n\n'
 
     return Response(generate(), mimetype='text/event-stream')
 

apps/agentfabric/server_utils.py

@@ -137,20 +137,32 @@ def get_user_bot(
             self,
             builder_id,
             session,
-            renew=False,
             user_token=None) -> Tuple[RolePlay, MemoryWithRetrievalKnowledge]:
         unique_id = builder_id + '_' + session
         user_agent = self.user_bots[unique_id]
-        if renew or user_agent is None:
+        if user_agent is None:
             logger.info(f'init_user_chatbot_agent: {builder_id} {session}')
             user_agent = init_user_chatbot_agent(
                 builder_id, session, use_tool_api=True, user_token=user_token)
             self.user_bots[unique_id] = user_agent
         return user_agent
 
-    def clear_user_bot(self, builder_id, session):
+    def renew_user_bot(
+            self,
+            builder_id,
+            session='default',
+            user_token=None):
+        self.clear_user_bot(builder_id, session)
+        unique_id = builder_id + '_' + session
+        logger.info(f'reinit_user_chatbot_agent: {builder_id} {session}')
+        user_agent = init_user_chatbot_agent(
+            builder_id, session, use_tool_api=True, user_token=user_token)
+        self.user_bots[unique_id] = user_agent
+
+    def clear_user_bot(self, builder_id, session='default'):
         unique_id = builder_id + '_' + session
         user_agent = self.user_bots[unique_id]
+        logger.info(f'clear_user_chatbot_agent: {builder_id} {session}')
         if user_agent is not None:
             self.user_bots.delete_key(unique_id)
         shutil.rmtree(

apps/agentfabric/user_core.py

@@ -54,6 +54,11 @@ def init_user_chatbot_agent(uuid_str='',
         user_token=user_token,
         openapi_list=openapi_plugin_list)
 
+    logger.query_info(
+        uuid=uuid_str,
+        message=
+        f'init {builder_cfg.name} chatbot end.')
+
     # build memory
     preview_history_dir = get_user_preview_history_dir(uuid_str, session)
     storage_path = preview_history_dir

modelscope_agent/agent.py

@@ -193,11 +193,32 @@ def _register_tool(self,
         Returns:
 
         """
-        tool_name = tool
+        tool_name = None
         tool_cfg = {}
+
+        # # Check if the tool is a dictionary and extract the tool name and configuration from it.
         if isinstance(tool, dict):
-            tool_name = next(iter(tool))
-            tool_cfg = tool[tool_name]
+            try:
+                tool_name = next(iter(tool))
+                tool_cfg = tool[tool_name]
+            except StopIteration:
+                # If the tool is an empty dictionary, proceed to register the next tool.
+                print(
+                    'Empty tool dictionary provided, skipping the registration of the current tool'
+                )
+                return
+
+        # If the tool is a string, assign it directly to tool_name.
+        elif isinstance(tool, str):
+            tool_name = tool
+
+        # If the tool_name is empty, skip the registration of the current tool.
+        if not tool_name:
+            print(
+                'No tool name provided, skipping the registration of the current tool'
+            )
+            return
+
         if tool_name not in TOOL_REGISTRY and not self.use_tool_api:
             raise NotImplementedError
         if tool_name not in self.function_list:

modelscope_agent/agents/multi_role_play.py

@@ -30,7 +30,7 @@
 Action: 工具的名称，必须是[{tool_names}]之一
 Action Input: 工具的输入
 Observation: <result>工具返回的结果</result>
-Answer: 根据Observation总结本次工具调用返回的结果，如果结果中出现url，请使用如下格式展示出来：![图片](url)
+Answer: 根据Observation总结本次工具调用返回的结果。如果结果中出现url，请严格遵循Observation中url的格式进行展示。
 
 """
 
@@ -59,10 +59,9 @@
 Action: The name of the tool, must be one of [{tool_names}]
 Action Input: Tool input
 Observation: <result>Tool returns result</result>
-Answer: Summarize the results of this tool call based on Observation. If the result contains url, %s
+Answer: Summarize the results of this tool call based on Observation.
 
-""" % ('You can call zero or more times according to your needs:',
-       'please display it in the following format:![Image](URL)')
+""" % ('You can call zero or more times according to your needs:')
 
 PROMPT_TEMPLATE_EN = """{role_prompt}"""
 

modelscope_agent/llm/utils/function_call_with_raw_prompt.py

@@ -21,7 +21,7 @@
 Action: 工具的名称，必须是[{tool_names}]之一
 Action Input: 工具的输入
 Observation: <result>工具返回的结果</result>
-Answer: 根据Observation总结本次工具调用返回的结果，如果结果中出现url，请使用如下格式展示出来：![图片](url)
+Answer: 根据Observation总结本次工具调用返回的结果。如果结果中出现url，请严格遵循Observation中url的格式进行展示。
 
 """
 
@@ -47,7 +47,7 @@
 ...
 Observation: <result>工具N返回的结果</result>
 
-Answer: 根据Observation总结本次工具调用返回的结果，如果结果中出现url，请使用如下格式展示出来：![图片](url)
+Answer: 根据Observation总结本次工具调用返回的结果
 
 """
 
@@ -64,10 +64,9 @@
 Action: The name of the tool, must be one of [{tool_names}]
 Action Input: Tool input
 Observation: <result>Tool returns result</result>
-Answer: Summarize the results of this tool call based on Observation. If the result contains url, %s
+Answer: Summarize the results of this tool call based on Observation.
 
-""" % ('You can call zero or more times according to your needs:',
-       'please display it in the following format:![Image](URL)')
+""" % ('You can call zero or more times according to your needs:')
 
 TOOL_TEMPLATE_EN_PARALLEL = """
 # Tools
@@ -90,10 +89,9 @@
 Observation: <result>Tool 1 returns result</result>
 ...
 Observation: <result>Tool N returns result</result>
-Answer: Summarize the results of this tool call based on Observation. If the result contains url, %s
+Answer: Summarize the results of this tool call based on Observation.
 
-""" % ('You can call zero or more times according to your needs:',
-       'please display it in the following format:![Image](URL)')
+""" % ('You can call zero or more times according to your needs:')
 
 TOOL_TEMPLATE = {
     'zh': TOOL_TEMPLATE_ZH,

modelscope_agent/memory/memory_with_rag.py

@@ -38,6 +38,7 @@ def __init__(self,
             llm=llm,
             cache_dir=storage_path,
             use_cache=use_knowledge_cache,
+            documents=[],
             **kwargs)
 
     @enable_rag_callback

modelscope_agent/tools/base.py

@@ -1,5 +1,6 @@
 import os
 import time
+import traceback
 from abc import ABC, abstractmethod
 from copy import deepcopy
 from typing import Dict, List, Optional, Union
@@ -335,6 +336,12 @@ def __init__(self,
         self.tenant_id = tenant_id
         self._register_tool()
 
+        if not self.tool_name:
+            print(
+                "Skipping tool registration and status check because 'tool_name' is not defined or empty."
+            )
+            return  # When tool_name is an empty string, skip registration and status check.
+
         max_retry = 10
         while max_retry > 0:
             status = self._check_tool_status()
@@ -368,6 +375,13 @@ def parse_service_response(response):
 
     def _register_tool(self):
         try:
+            # Check if `tool_name` is defined and not empty.
+            if not self.tool_name:
+                print(
+                    "Skipping tool registration because 'tool_name' is not defined or empty."
+                )
+                return  # Return directly, skipping registration.
+
             service_token = os.getenv('TOOL_MANAGER_AUTH', '')
             headers = {
                 'Content-Type': 'application/json',
@@ -422,8 +436,8 @@ def _check_tool_status(self):
             return result['status']
         except Exception as e:
             raise RuntimeError(
-                f'Get error during checking status from tool manager service with detail {e}'
-            )
+                f'Get error during checking status from tool manager service: {self.tool_name} '
+                f'{self.tenant_id}. detail {e}, {traceback.format_exc()}')
 
     def _get_tool_info(self):
         try:
@@ -442,10 +456,16 @@ def _get_tool_info(self):
                 },
                 headers=headers)
             response.raise_for_status()
-            return ToolServiceProxy.parse_service_response(response)
+            tool_info = ToolServiceProxy.parse_service_response(response)
+            # check required params
+            _ = tool_info['name']
+            _ = tool_info['description']
+            _ = tool_info['parameters']
+            return tool_info
         except Exception as e:
             raise RuntimeError(
-                f'Get error during getting tool info from tool manager service with detail {e}'
+                f'Get error during getting tool info from tool manager, '
+                f'{self.tool_name} {self.tenant_id}. detail {e}, {traceback.format_exc()}'
             )
 
     def call(self, params: str, **kwargs):

modelscope_agent/tools/dashscope_tools/image_generation.py

@@ -47,7 +47,7 @@ def call(self, params: str, **kwargs) -> str:
         prompt = params['text']
         if prompt is None:
             return None
-        seed = kwargs.get('seed', None)
+        seed = kwargs.get('seed', 1234)
         model = kwargs.get('model', 'wanx-v1')
         extra_input = {}
         lora_index = params.get('lora_index', None)

modelscope_agent/tools/utils/openapi_utils.py

@@ -189,13 +189,17 @@ def swagger_to_openapi(swagger_data):
 
 def openapi_schema_convert(schema: dict, auth: dict = {}):
     config_data = {}
-    host = schema.get('host', '')
+    host = schema.get('host', '') if schema else ''  # Check if schema is None
+
     if host:
-        schema = swagger_to_openapi(schema)
+        schema = swagger_to_openapi(schema) if schema else {
+        }  # Call only if schema is not None
 
-    schema = jsonref.replace_refs(schema)
+    schema = jsonref.replace_refs(schema) if schema else {
+    }  # Call only if schema is not None
 
-    servers = schema.get('servers', [])
+    servers = schema.get('servers', []) if schema else [
+    ]  # Check if schema is None
 
     if servers:
         servers_url = servers[0].get('url')

modelscope_agent/tools/utils/output_wrapper.py

@@ -178,7 +178,7 @@ def __init__(self, video, **kwargs) -> None:
 
         self._raw_data = video
 
-        if self._path.endswith('.gif'):
+        if self._path.split('?')[0].endswith('.gif'):
             self._repr = f'![IMAGEGEN]({self._path})'
         else:
             self._repr = f'<video src="{self._path}"/>'