Fix matrix image uploads

.github/workflows/daily-update.yml

@@ -11,6 +11,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - run: |
+        # Setup Cloudflare WARP
+        curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | sudo gpg --yes --dearmor --output /usr/share/keyrings/cloudflare-warp-archive-keyring.gpg
+        # Add this repo to your apt repositories
+        echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp-archive-keyring.gpg] https://pkg.cloudflareclient.com/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflare-client.list
+        # Install
+        sudo apt-get update && sudo apt-get install cloudflare-warp
+        warp-cli --accept-tos register
+        warp-cli --accept-tos connect
+        curl https://www.cloudflare.com/cdn-cgi/trace/
     - uses: actions/checkout@v2
     - name: Set up Python 3.9
       uses: actions/setup-python@v1

.github/workflows/weekly-update.yml

@@ -11,6 +11,16 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
+    - run: |
+        # Setup Cloudflare WARP
+        curl -fsSL https://pkg.cloudflareclient.com/pubkey.gpg | sudo gpg --yes --dearmor --output /usr/share/keyrings/cloudflare-warp-archive-keyring.gpg
+        # Add this repo to your apt repositories
+        echo "deb [signed-by=/usr/share/keyrings/cloudflare-warp-archive-keyring.gpg] https://pkg.cloudflareclient.com/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/cloudflare-client.list
+        # Install
+        sudo apt-get update && sudo apt-get install cloudflare-warp
+        warp-cli --accept-tos register
+        warp-cli --accept-tos connect
+        curl https://www.cloudflare.com/cdn-cgi/trace/
     - uses: actions/checkout@v2
     - name: Set up Python 3.9
       uses: actions/setup-python@v1

main.py

@@ -4,16 +4,20 @@
 
 import asyncio
 import logging
+import mimetypes
 import os
 import sys
+import tempfile
 from multiprocessing import JoinableQueue, Process
 from time import sleep
 
+import aiofiles
 import discord
 import simplematrixbotlib as botlib
 import telegram
 from discord.ext import commands
 from dotenv import load_dotenv
+from nio.responses import UploadResponse
 from praw import Reddit
 
 import feeds
@@ -28,6 +32,7 @@
                               create_reddit_matrix_post,
                               create_reddit_telegram_post)
 from util.load_config import load_config
+from util.requests import fetch_image_to_file
 
 logging.basicConfig(level=logging.INFO)
 
@@ -220,6 +225,26 @@ def run_matrix_bot(queue: JoinableQueue):
         password=os.getenv('MATRIX_PASSWORD'))
     bot = botlib.Bot(creds)
 
+    async def get_matrix_content_uri(image_url: str):
+        image_filepath = os.path.join(tempfile.gettempdir(), os.path.basename(image_url))
+        if not fetch_image_to_file(image_url, image_filepath):
+            return None
+        mime_type = mimetypes.guess_type(image_filepath)[0]
+
+        file_stat = await aiofiles.os.stat(image_filepath)
+        async with aiofiles.open(image_filepath, "r+b") as file:
+            resp, _ = await bot.api.async_client.upload(
+                file,
+                content_type=mime_type,
+                filename=os.path.basename(image_filepath),
+                filesize=file_stat.st_size,
+                encrypt=False,
+            )
+        if not isinstance(resp, UploadResponse):
+            logging.error("Failed Upload Response: %s", resp)
+            return None
+        return resp.content_uri
+
     @bot.listener.on_startup
     async def on_startup(room_id: str):
         if room_id != matrix_space:
@@ -232,11 +257,21 @@ async def on_startup(room_id: str):
             source, channel, post = message
             if source == 'reddit':
                 matrix_post = create_reddit_matrix_post(channel, post)
+                if matrix_post.image_url:
+                    matrix_content_uri = await get_matrix_content_uri(matrix_post.image_url)
+                    if matrix_content_uri:
+                        matrix_post.markdown = matrix_post.markdown.replace(
+                            matrix_post.image_url, matrix_content_uri)
                 await bot.api.send_markdown_message(
                     room_id=matrix_rooms['reddit'][channel['category']],
                     message=matrix_post.markdown)
             elif source == 'blog':
                 matrix_post = create_blog_matrix_post(post)
+                if matrix_post.image_url:
+                    matrix_content_uri = await get_matrix_content_uri(matrix_post.image_url)
+                    if matrix_content_uri:
+                        matrix_post.markdown = matrix_post.markdown.replace(
+                            matrix_post.image_url, matrix_content_uri)
                 await bot.api.send_markdown_message(
                     room_id=matrix_rooms['blog'][channel],
                     message=matrix_post.markdown)

util/embed_utils.py

@@ -327,6 +327,7 @@ class MatrixPost:
     Matrix post
     '''
     markdown: str
+    image_url: str = None
 
 
 def create_forum_matrix_post(forum, post):
@@ -335,7 +336,7 @@ def create_forum_matrix_post(forum, post):
     '''
     config = load_config('forums.json')[forum]
 
-    message = f'💬 {config["publisher"]}'
+    message = f'***\n\n💬 {config["publisher"]}'
     message += f'\n\n**[{post["title"]}]({post["url"]})**'
     if post["description"]:
         if len(post["description"]) > 1000:
@@ -356,11 +357,11 @@ def create_blog_matrix_post(post):
     '''
     Create blog post embed from JSON
     '''
-    message = f'📰 {post["publisher"]}'
+    message = f'***\n\n📰 {post["publisher"]}'
     message += f'\n\n**[{post["title"]}]({post["url"]})**'
     image_url = post["image"] if 'image' in post and len(post['image']) > 0 else ''
     if image_url:
-        message += f'\n\n![{post["title"]}]({image_url})'
+        message += f'\n\n![]({image_url})'
     if post["description"]:
         if len(post["description"]) > 1000:
             post["description"] = post["description"][:1000] + '...'
@@ -370,25 +371,27 @@ def create_blog_matrix_post(post):
         message = message.replace('\n\n\n', '\n\n')
 
     return MatrixPost(
-        markdown=message
+        markdown=message,
+        image_url=image_url
     )
 
 
 def create_reddit_matrix_post(subreddit, post):
     '''
     Create reddit post embed from subreddit JSON
     '''
-    message = f'💬 r/{subreddit["subreddit"]}'
+    message = f'***\n\n💬 r/{subreddit["subreddit"]}'
     message += f'\n\n**[{post["title"]}]({post["url"]})**'
     image_url = post["media"]["url"] if 'media' in post and 'url' in post['media'] else ''
     if image_url:
-        message += f'\n\n![{post["title"]}]({image_url})'
+        message += f'\n\n![]({image_url})'
     if post["selftext"]:
         if len(post["selftext"]) > 1000:
             post["selftext"] = post["selftext"][:1000] + '...'
         message += f'\n\n{post["selftext"].strip()}'
     message += '\n‎'
 
     return MatrixPost(
-        markdown=message
+        markdown=message,
+        image_url=image_url
     )

util/requests.py

@@ -34,3 +34,30 @@ def fetch_from_server(url: str):
     except Exception as exception:  # pylint: disable=broad-except
         logging.error('Cannot connect to %s - %s', url, str(exception)[:30])
         return None
+
+
+def fetch_image_to_file(url: str, filename: str):
+    '''
+    Fetch image from server and save to file
+    '''
+    try:
+        response = requests.get(
+            url,
+            headers={
+                'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7)'
+                'AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36'
+            },
+            timeout=25,
+            verify=False
+        )
+        if response.status_code == 200:
+            with open(filename, 'wb') as file:
+                file.write(response.content)
+            return True
+        logging.error(
+            'Request to %s failed - %d: %s', url, response.status_code, response.reason
+        )
+        return False
+    except Exception as exception:  # pylint: disable=broad-except
+        logging.error('Cannot connect to %s - %s', url, str(exception)[:30])
+        return False