[FEAT][Flow.run() img = None for conditional img inputs, BaseMultiModalModel, and multi modal swarms of manufacturing agents

Author: Kye

example.py

@@ -1,9 +1,21 @@
+import os
+
+from dotenv import load_dotenv
+
+# Import the OpenAIChat model and the Flow struct
 from swarms.models import OpenAIChat
 from swarms.structs import Flow
 
+# Load the environment variables
+load_dotenv()
+
+# Get the API key from the environment
+api_key = os.environ.get("OPENAI_API_KEY")
+
 # Initialize the language model
 llm = OpenAIChat(
     temperature=0.5,
+    openai_api_key=api_key,
 )
 
 

multi_modal_auto_agent.py

@@ -1,5 +1,8 @@
 from swarms.structs import Flow
 from swarms.models.gpt4_vision_api import GPT4VisionAPI
+from swarms.prompts.multi_modal_autonomous_instruction_prompt import (
+    MULTI_MODAL_AUTO_AGENT_SYSTEM_PROMPT_1,
+)
 
 
 llm = GPT4VisionAPI()
@@ -10,6 +13,7 @@
 ## Initialize the workflow
 flow = Flow(
     llm=llm,
+    sop=MULTI_MODAL_AUTO_AGENT_SYSTEM_PROMPT_1,
     max_loops="auto",
 )
 

playground/demos/idea_2_img/main.py

@@ -0,0 +1,7 @@
+"""
+Idea 2 img
+
+task -> gpt4 text -> dalle3 img -> gpt4vision img + text analyze img -> dalle3 img -> loop    
+    
+"""
+from swarms.models.gpt4_vision_api import GPT4VisionAPI

playground/demos/swarm_of_mma_manufacturing/main.py

@@ -0,0 +1,15 @@
+"""
+Swarm of multi modal autonomous agents for manufacturing!
+---------------------------------------------------------    
+Health Security agent: Agent that monitors the health of working conditions: input image of factory output: health safety index 0.0 - 1.0 being the highest
+Quality Control agent: Agent that monitors the quality of the product: input image of product output: quality index 0.0 - 1.0 being the highest
+Productivity agent: Agent that monitors the productivity of the factory: input image of factory output: productivity index 0.0 - 1.0 being the highest
+Safety agent: Agent that monitors the safety of the factory: input image of factory output: safety index 0.0 - 1.0 being the highest
+Security agent: Agent that monitors the security of the factory: input image of factory output: security index 0.0 - 1.0 being the highest
+Sustainability agent: Agent that monitors the sustainability of the factory: input image of factory output: sustainability index 0.0 - 1.0 being the highest
+Efficiency agent: Agent that monitors the efficiency of the factory: input image of factory output: efficiency index 0.0 - 1.0 being the highest    
+
+
+Flow:
+health security agent -> quality control agent -> productivity agent -> safety agent -> security agent -> sustainability agent -> efficiency agent 
+"""

swarms/models/base_multimodal_model.py

@@ -1,3 +1,4 @@
+from abc import abstractmethod
 import asyncio
 import base64
 import concurrent.futures
@@ -7,8 +8,8 @@
 from typing import List, Optional, Tuple
 
 import requests
-from ABC import abstractmethod
 from PIL import Image
+from termcolor import colored
 
 
 class BaseMultiModalModel:
@@ -37,7 +38,6 @@ def __init__(
         self.retries = retries
         self.chat_history = []
 
-    
     @abstractmethod
     def __call__(self, text: str, img: str):
         """Run the model"""
@@ -61,17 +61,17 @@ def get_img_from_web(self, img: str):
         except requests.RequestException as error:
             print(f"Error fetching image from {img} and error: {error}")
             return None
-        
+
     def encode_img(self, img: str):
         """Encode the image to base64"""
         with open(img, "rb") as image_file:
             return base64.b64encode(image_file.read()).decode("utf-8")
-    
+
     def get_img(self, img: str):
         """Get the image from the path"""
         image_pil = Image.open(img)
         return image_pil
-    
+
     def clear_chat_history(self):
         """Clear the chat history"""
         self.chat_history = []
@@ -87,11 +87,11 @@ def run_many(
         Args:
             tasks (List[str]): List of tasks
             imgs (List[str]): List of image paths
-        
+
         Returns:
             List[str]: List of responses
-            
-        
+
+
         """
         # Instantiate the thread pool executor
         with ThreadPoolExecutor(max_workers=self.max_workers) as executor:
@@ -101,7 +101,6 @@ def run_many(
         for result in results:
             print(result)
 
-
     def run_batch(self, tasks_images: List[Tuple[str, str]]) -> List[str]:
         """Process a batch of tasks and images"""
         with concurrent.futures.ThreadPoolExecutor() as executor:
@@ -133,11 +132,11 @@ async def run_batch_async_with_retries(
             for task, img in tasks_images
         ]
         return await asyncio.gather(*futures)
-    
+
     def unique_chat_history(self):
         """Get the unique chat history"""
         return list(set(self.chat_history))
-    
+
     def run_with_retries(self, task: str, img: str):
         """Run the model with retries"""
         for i in range(self.retries):
@@ -146,7 +145,7 @@ def run_with_retries(self, task: str, img: str):
             except Exception as error:
                 print(f"Error with the request {error}")
                 continue
-    
+
     def run_batch_with_retries(self, tasks_images: List[Tuple[str, str]]):
         """Run the model with retries"""
         for i in range(self.retries):
@@ -188,28 +187,37 @@ def get_generation_time(self) -> float:
         if self.start_time and self.end_time:
             return self.end_time - self.start_time
         return 0
-    
+
     def get_chat_history(self):
         """Get the chat history"""
         return self.chat_history
-    
+
     def get_unique_chat_history(self):
         """Get the unique chat history"""
         return list(set(self.chat_history))
-    
+
     def get_chat_history_length(self):
         """Get the chat history length"""
         return len(self.chat_history)
-    
+
     def get_unique_chat_history_length(self):
         """Get the unique chat history length"""
         return len(list(set(self.chat_history)))
-    
+
     def get_chat_history_tokens(self):
         """Get the chat history tokens"""
         return self._num_tokens()
-    
+
     def print_beautiful(self, content: str, color: str = "cyan"):
         """Print Beautifully with termcolor"""
         content = colored(content, color)
-        print(content)
+        print(content)
+
+    def stream(self, content: str):
+        """Stream the output
+
+        Args:
+            content (str): _description_
+        """
+        for chunk in content:
+            print(chunk)

swarms/models/gpt4_vision_api.py

@@ -1,6 +1,7 @@
-import logging 
+import logging
 import asyncio
 import base64
+from typing import Optional
 import concurrent.futures
 from termcolor import colored
 import json
@@ -12,6 +13,13 @@
 import requests
 from dotenv import load_dotenv
 
+
+try:
+    import cv2
+except ImportError:
+    print("OpenCV not installed. Please install OpenCV to use this model.")
+    raise ImportError
+
 # Load environment variables
 load_dotenv()
 openai_api_key = os.getenv("OPENAI_API_KEY")
@@ -59,7 +67,8 @@ def __init__(
         max_workers: int = 10,
         max_tokens: str = 300,
         openai_proxy: str = "https://api.openai.com/v1/chat/completions",
-        beautify: bool = False
+        beautify: bool = False,
+        streaming_enabled: Optional[bool] = False,
     ):
         super().__init__()
         self.openai_api_key = openai_api_key
@@ -69,6 +78,7 @@ def __init__(
         self.max_tokens = max_tokens
         self.openai_proxy = openai_proxy
         self.beautify = beautify
+        self.streaming_enabled = streaming_enabled
 
         if self.logging_enabled:
             logging.basicConfig(level=logging.DEBUG)
@@ -123,14 +133,101 @@ def run(self, task: str, img: str):
             out = response.json()
             content = out["choices"][0]["message"]["content"]
 
+            if self.streaming_enabled:
+                content = self.stream_response(content)
+            else:
+                pass
+
             if self.beautify:
                 content = colored(content, "cyan")
+                print(content)
             else:
                 print(content)
+
         except Exception as error:
             print(f"Error with the request: {error}")
             raise error
 
+    def video_prompt(self, frames):
+        """
+        SystemPrompt is a class that generates a prompt for the user to respond to.
+        The prompt is generated based on the current state of the system.
+
+        Parameters
+        ----------
+        frames : list
+            A list of base64 frames
+
+        Returns
+        -------
+        PROMPT : str
+            The system prompt
+
+        Examples
+        --------
+
+        >>> from swarms.models import GPT4VisionAPI
+        >>> llm = GPT4VisionAPI()
+        >>> video = "video.mp4"
+        >>> base64_frames = llm.process_video(video)
+        >>> prompt = llm.video_prompt(base64_frames)
+        >>> print(prompt)
+
+        """
+        PROMPT = f"""
+        These are frames from a video that I want to upload. Generate a compelling description that I can upload along with the video:
+        
+        {frames}
+        """
+        return PROMPT
+
+    def stream_response(self, content: str):
+        """Stream the response of the output
+
+        Args:
+            content (str): _description_
+        """
+        for chunk in content:
+            print(chunk)
+
+    def process_video(self, video: str):
+        """
+        Process a video into a list of base64 frames
+
+        Parameters
+        ----------
+        video : str
+            The path to the video file
+
+        Returns
+        -------
+        base64_frames : list
+            A list of base64 frames
+
+        Examples
+        --------
+        >>> from swarms.models import GPT4VisionAPI
+        >>> llm = GPT4VisionAPI()
+        >>> video = "video.mp4"
+        >>> base64_frames = llm.process_video(video)
+
+        """
+        video = cv2.VideoCapture(video)
+
+        base64_frames = []
+        while video.isOpened():
+            success, frame = video.read()
+            if not success:
+                break
+            _, buffer = cv2.imencode(".jpg", frame)
+            base64_frames.append(base64.b64encode(buffer).decode("utf-8"))
+
+        video.release()
+        print(len(base64_frames), "frames read.")
+
+        for img in base64_frames:
+            base64.b64decode(img.encode("utf-8"))
+
     def __call__(self, task: str, img: str):
         """Run the model."""
         try:
@@ -168,10 +265,17 @@ def __call__(self, task: str, img: str):
             out = response.json()
             content = out["choices"][0]["message"]["content"]
 
+            if self.streaming_enabled:
+                content = self.stream_response(content)
+            else:
+                pass
+
             if self.beautify:
                 content = colored(content, "cyan")
+                print(content)
             else:
                 print(content)
+
         except Exception as error:
             print(f"Error with the request: {error}")
             raise error

swarms/models/kosmos_two.py

@@ -24,7 +24,7 @@ class Kosmos:
     ----------
     model_name : str
         Path to the pretrained model
-    
+
     Examples
     --------
     >>> kosmos = Kosmos()

swarms/models/whisperx_model.py

@@ -99,7 +99,9 @@ def transcribe_youtube_video(self):
             print("The key 'segments' is not found in the result.")
 
     def transcribe(self, audio_file):
-        model = whisperx_model.load_model("large-v2", self.device, self.compute_type)
+        model = whisperx_model.load_model(
+            "large-v2", self.device, self.compute_type
+        )
         audio = whisperx_model.load_audio(audio_file)
         result = model.transcribe(audio, batch_size=self.batch_size)