image get ocr info (#3)

* functionality

* debug

debug

* debug

Author: ThousandOfWind

.gitignore

@@ -154,3 +154,7 @@ cython_debug/
 __pycache__/
 
 output/
+
+*.ipynb
+
+outputTEMP_MPY_wvf_snd.mp3

ExampleVideoGen.py

@@ -1,5 +1,5 @@
 import os
-from tools.tools import getCurrentTimeAsFolder
+from tools.tools import current_time_as_folder
 from openai import AzureOpenAI
 from tools.openai_adapter import OpenaiAdapter
 from tools.speech_adapter import SpeechServiceAdapter, DefaultMaleSpeaker
@@ -13,7 +13,7 @@
     bing_search_api=os.getenv('BING_SEARCH_ENDPOINT'), 
     bing_search_key=os.getenv('BING_SEARCH_KEY')
 )
-newsList = bing.newsCategoryTrending(ChinaCategory.Sports.value, Market.China.value)
+newsList = bing.news_category_trending(ChinaCategory.Sports.value, Market.China.value)
 news = newsList[0]
 
 oai = OpenaiAdapter(openai_client=AzureOpenAI(
@@ -23,6 +23,6 @@
 ))
 speech = SpeechServiceAdapter(os.getenv('SPEECH_HOST'), os.getenv('SPEECH_REGION'), os.getenv('SPEECH_KEY'), DefaultMaleSpeaker)
 
-director = AIDirector(oai, speech, bing, '/System/Library/Fonts/Supplemental/Arial Unicode.ttf')
+director = AIDirector(oai, speech, bing)
 
-director.news2Video(news, folderPath=getCurrentTimeAsFolder())
+director.news2Video(news, folderPath=current_time_as_folder())

ExampleVideoGenWithAvatar.py

@@ -1,11 +1,13 @@
 import os
-from tools.tools import getCurrentTimeAsFolder
+from tools.tools import current_time_as_folder
 from openai import AzureOpenAI
 from tools.openai_adapter import OpenaiAdapter
 from tools.speech_adapter import SpeechServiceAdapter, DefaultFemaleSpeaker
 from tools.bing_search_adapter import BingSearchAdapter, ChinaCategory, Market
 from workers.AIDirector import AIDirector
 from dotenv import load_dotenv
+from configs.directorConfig import DirectorConfig
+
 
 load_dotenv()
 
@@ -17,10 +19,13 @@
 oai = OpenaiAdapter(openai_client=client)
 speech = SpeechServiceAdapter(os.getenv('SPEECH_HOST'), os.getenv('SPEECH_REGION'), os.getenv('SPEECH_KEY'), DefaultFemaleSpeaker)
 bing = BingSearchAdapter(bing_search_api=os.getenv('BING_SEARCH_ENDPOINT'), bing_search_key=os.getenv('BING_SEARCH_KEY'))
-director = AIDirector(oai, speech, bing, '/System/Library/Fonts/Supplemental/Arial Unicode.ttf')
+config = DirectorConfig({
+    "use_avatar": True
+})
+director = AIDirector(oai, speech, bing, config=config)
 
-folderPath = getCurrentTimeAsFolder()
-newsList = bing.newsCategoryTrending(ChinaCategory.Military.value, Market.China.value)
+folderPath = current_time_as_folder()
+newsList = bing.news_category_trending(ChinaCategory.Military.value, Market.China.value)
 director.news2Video(newsList[2], folderPath, with_avatar=True)
 
 

ExampleWepageVedioGen.py

@@ -0,0 +1,30 @@
+import os
+from tools.tools import current_time_as_folder
+from openai import AzureOpenAI
+from tools.openai_adapter import OpenaiAdapter
+from tools.speech_adapter import SpeechServiceAdapter, DefaultFemaleSpeaker
+from tools.bing_search_adapter import BingSearchAdapter
+from workers.AIDirector import AIDirector
+from dotenv import load_dotenv
+import easyocr
+from configs.directorConfig import DirectorConfig
+
+load_dotenv()
+
+client = AzureOpenAI(
+    api_version="2023-12-01-preview",
+    azure_endpoint=os.getenv('OPANAI_API_ENDPOINT'),
+    api_key=os.getenv('OPANAI_API_KEY'),
+)
+oai = OpenaiAdapter(openai_client=client)
+speech = SpeechServiceAdapter(os.getenv('SPEECH_HOST'), os.getenv('SPEECH_REGION'), os.getenv('SPEECH_KEY'), DefaultFemaleSpeaker)
+bing = BingSearchAdapter(bing_search_api=os.getenv('BING_SEARCH_ENDPOINT'), bing_search_key=os.getenv('BING_SEARCH_KEY'))
+reader = easyocr.Reader(['ch_sim','en'])
+director = AIDirector(oai, speech, bing, reader, config=DirectorConfig({
+    "use_ocr":True,
+    "use_image_in_webpage": True
+}))
+
+folderPath = current_time_as_folder()
+director.webpage2Video("https://azure.microsoft.com/zh-cn/products/ai-services/?activetab=pivot:azureopenai%E6%9C%8D%E5%8A%A1tab", folderPath)
+

README.md

@@ -229,7 +229,7 @@ director.news2Video(newsList[0], folderPath, with_avatar=True)
 
 - [ ] webpage to script
   - [x] news webpage to script
-  - [ ] any webpage to script
+  - [x] any webpage to script
   - [ ] any topic to script
 - [x] Collect/Generate multimedia resource for a script
   - [x] Text to speech
@@ -245,11 +245,13 @@ director.news2Video(newsList[0], folderPath, with_avatar=True)
   - [ ] Add BGM
   - [ ] Fix Avatar background issue, Avatar position and size auto-adjust
   - [ ] Different length
+  - [ ] Any size
 - [ ] [Current on going] Go deeper into content
-  - [ ] Download image/video in webpage
-  - [ ] Add OCR when review image for news
+  - [x] Download image/video in webpage
+  - [x] Add OCR when review image for news
   - [ ] Search related information
   - [ ] Draw table / chart if need
+  - [ ] RAG on knowledge
 - [ ] UX
   - [ ] UI Design
   - [ ] GUI
@@ -260,7 +262,9 @@ director.news2Video(newsList[0], folderPath, with_avatar=True)
   - [ ] Async methods
   - [ ] More Comments
   - [ ] Error handling
-  - [ ] Cost statistic
+  - [ ] Cost analysis
+  - [ ] Test
+  - [ ] Name of variables
 - [ ] Integrate social media
 - [ ] Integrate Lang Chain
 - [ ] Onboard GPT store

configs/config.py

@@ -0,0 +1,10 @@
+import copy
+
+class Config(object):
+    def __init__(self, conf:dir):
+        self._config = copy.deepcopy(conf) # set it to conf
+
+    def get_property(self, property_name):
+        if property_name not in self._config.keys(): # we don't want KeyError
+            return None  # just return None if not found
+        return self._config[property_name]

configs/directorConfig.py

@@ -0,0 +1,30 @@
+from configs.config import Config
+
+class DirectorConfig(Config):
+    def __init__(self, conf: dir):
+        super().__init__(conf)
+
+    @property
+    def path_to_font(self) -> str:
+        return self.get_property("path_to_font") or '/System/Library/Fonts/Supplemental/Arial Unicode.ttf'
+    
+    @property
+    def video_shape(self) -> (int, int):
+        return self.get_property("video_shape") or tuple((720, 1280))
+    
+    @property
+    def use_avatar(self) -> bool:
+        return self.get_property("use_avatar") or False
+    
+    @property
+    def use_image_in_webpage(self) -> bool:
+        return self.get_property("use_image_in_webpage") or False
+    
+    @property
+    def use_ocr(self) -> bool:
+        return self.get_property("use_ocr") or False
+    
+    @property
+    def use_dalle(self) -> bool:
+        return self.get_property("use_dalle") or False
+    

models/image.py

@@ -0,0 +1,48 @@
+from enum import Enum
+
+class ImageInfo:
+    def __init__(self, path:str, raw_description:str='', ai_description:str='', provider:str=''):
+        self.path = path
+        self.raw_description = raw_description
+        self.ai_description = ai_description
+        self.provider = provider
+    
+    def toJSON(self):
+        return {
+            "path": self.path,
+            "raw_description": self.raw_description,
+            "ai_description": self.ai_description,
+            "provider": self.provider
+        }
+    
+    @property
+    def description(self):
+        return " ".join([
+            "description: `{}`".format(self.raw_description) if self.raw_description else " ",
+            "ocr result: `{}`".format(self.ai_description) if self.ai_description else " "
+        ])
+
+class ImageEncodingFormatEnum(Enum):
+    JPEG = 'jpeg'
+    PNG = 'png'
+    GIF = 'gif'
+    SVG = 'svg+xml'
+
+ImageTypeSuffix = {
+    ImageEncodingFormatEnum.JPEG.value: [
+        'jpg',
+        'jpeg',
+        'jfif',
+        'pjpeg',
+        'pjp'
+    ],
+    ImageEncodingFormatEnum.PNG.value: [
+        'png'
+    ],
+    ImageEncodingFormatEnum.GIF.value: [
+        'gif'
+    ],
+    ImageEncodingFormatEnum.SVG.value: [
+        'svg'
+    ]
+}

models/webpage.py

@@ -0,0 +1,17 @@
+from bs4 import BeautifulSoup
+from models.image import ImageInfo
+from typing import List
+
+class WebpageInfo:
+    def __init__(self, soup:BeautifulSoup):
+        self.soup = soup
+        self.title_text = self.soup.title.text if self.soup.title else ''
+        self.content = self.soup.text
+        self.images:List[ImageInfo] = []
+
+    def toJSON(self):
+        return {
+            "title": self.title_text,
+            "content": self.content,
+            "images": [image.toJSON() for image in self.images]
+        }

prompts/newsWebsiteToScript.json

@@ -1,6 +1,6 @@
 [
 	{
 		"role": "system",
-		"content": "你是著名的网络新闻主播。你不用介绍你自己或者当前频道，但需要说明消息源。你需要根据在线新闻网站提供的信息写200字左右新闻播报稿件，着重实时故事描述，避免评论。你懂得如何取悦观众，语言必须流畅，事实要清楚明了符合原新闻和常识。在开头就点明新闻爆点，吸引流量。巧妙利用谐音增加趣味性。观点一针见血直击要害。提出问题引发听众的思考。"
+		"content": "你是著名的网络新闻主播。不用介绍你自己或者当前频道。根据在线新闻网站提供的信息写200字左右新闻播报稿件，着重实时故事描述，避免评论。你懂得如何取悦观众，语言必须流畅，事实要清楚明了符合原新闻和常识。在开头就点明新闻爆点，吸引流量。巧妙利用谐音增加趣味性。观点一针见血直击要害。提出问题引发听众的思考。"
 	}
 ]