Serve image assets from Khoj domain, not directly from S3 bucket (#734)

- Serve generated images from Khoj domain instead of directly from AWS S3 - Rename assets URL from Khoj S3 bucket to assets.khoj.dev
khoj-ai · Jun 2, 2024 · cf8c9c2 · cf8c9c2
2 parents 5bb3689 + 7823ef0
commit cf8c9c2
Show file tree

Hide file tree

Showing 8 changed files with 71 additions and 7 deletions.
diff --git a/documentation/docs/data-sources/share_your_data.md b/documentation/docs/data-sources/share_your_data.md
@@ -13,4 +13,4 @@ There are several ways you can get started with sharing your data with the Khoj
 - Setup the sync options for either [Obsidian](/clients/obsidian) or [Emacs](/clients/emacs) to automatically sync your documents with Khoj. This is best if you are already using these tools and want to leverage Khoj's AI capabilities.
 - Configure your [Notion](/data-sources/notion_integration) or [Github](/data-sources/github_integration) to sync with Khoj. By providing your credentials, you can keep the data synced in the background.
 
-![demo of dragging and dropping a file](https://khoj-web-bucket.s3.amazonaws.com/drag_drop_file.gif)
+![demo of dragging and dropping a file](https://assets.khoj.dev/drag_drop_file.gif)
diff --git a/documentation/docusaurus.config.js b/documentation/docusaurus.config.js
@@ -80,7 +80,7 @@ const config = {
  {name: 'og:type', content: 'website'},
  {name: 'og:site_name', content: 'Khoj Documentation'},
  {name: 'og:description', content: 'Quickly get started with using or self-hosting Khoj'},
- {name: 'og:image', content: 'https://khoj-web-bucket.s3.amazonaws.com/link_preview_docs.png'},
+ {name: 'og:image', content: 'https://assets.khoj.dev/link_preview_docs.png'},
  {name: 'og:url', content: 'https://docs.khoj.dev'},
  {name: 'keywords', content: 'khoj, khoj ai, chatgpt, open ai, open source, productivity'}
  ],

diff --git a/src/interface/desktop/search.html b/src/interface/desktop/search.html
@@ -2,7 +2,7 @@
  <head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
- <meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png">
+ <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
  <title>Khoj - Search</title>
 
  <link rel="icon" type="image/png" sizes="128x128" href="./assets/icons/favicon-128x128.png">

diff --git a/src/khoj/database/adapters/__init__.py b/src/khoj/database/adapters/__init__.py
@@ -493,7 +493,7 @@ async def aget_client_application_by_id(client_id: str, client_secret: str):
 
 class AgentAdapters:
  DEFAULT_AGENT_NAME = "Khoj"
- DEFAULT_AGENT_AVATAR = "https://khoj-web-bucket.s3.amazonaws.com/lamp-128.png"
+ DEFAULT_AGENT_AVATAR = "https://assets.khoj.dev/lamp-128.png"
  DEFAULT_AGENT_SLUG = "khoj"
 
  @staticmethod

diff --git a/src/khoj/database/management/commands/change_generated_images_url.py b/src/khoj/database/management/commands/change_generated_images_url.py
@@ -0,0 +1,61 @@
+from django.core.management.base import BaseCommand
+from tqdm import tqdm
+
+from khoj.database.models import Conversation
+from khoj.utils.helpers import ImageIntentType, is_none_or_empty
+
+
+class Command(BaseCommand):
+ help = "Serve Khoj generated images from a different URL."
+
+ def add_arguments(self, parser):
+ # Pass Source URL
+ parser.add_argument(
+ "--source",
+ action="store",
+ help="URL from which generated images are currently served.",
+ )
+ # Pass Destination URL
+ parser.add_argument("--destination", action="store", help="URL to serve generated image from going forward.")
+
+ # Add a new argument 'reverse' to the command
+ parser.add_argument(
+ "--reverse",
+ action="store_true",
+ help="Revert to serve generated images from source instead of destination URL.",
+ )
+
+ def handle(self, *args, **options):
+ updated_count = 0
+ if not options.get("source") or not options.get("destination"):
+ self.stdout.write(
+ self.style.ERROR(
+ "Set --source, --destination args to migrate serving images from source to destination URL."
+ )
+ )
+ return
+
+ destination = options["source"] if options["reverse"] else options["destination"]
+ source = options["destination"] if options["reverse"] else options["source"]
+ for conversation in Conversation.objects.all():
+ conversation_updated = False
+ for chat in tqdm(conversation.conversation_log.get("chat", []), desc="Processing Conversations"):
+ if (
+ chat.get("by", "") == "khoj"
+ and not is_none_or_empty(chat.get("message"))
+ and chat.get("message", "").startswith(source)
+ and chat.get("intent", {}).get("type", "") == ImageIntentType.TEXT_TO_IMAGE2.value
+ and chat.get("message", "").endswith(".webp")
+ ):
+ # Convert source url to destination url
+ chat["message"] = chat["message"].replace(source, destination)
+ conversation_updated = True
+ updated_count += 1
+
+ if conversation_updated:
+ print(f"Save the updated conversation {conversation.id} to the database.")
+ conversation.save()
+
+ if updated_count > 0:
+ success = f"Successfully converted {updated_count} image URLs from {source} to {destination}.".strip()
+ self.stdout.write(self.style.SUCCESS(success))
diff --git a/src/khoj/interface/web/chat.html b/src/khoj/interface/web/chat.html
@@ -3,7 +3,7 @@
  <head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1.0 maximum-scale=1.0">
- <meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png">
+ <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
  <title>Khoj - Chat</title>
 
  <link rel="stylesheet" href="/static/assets/khoj.css?v={{ khoj_version }}">

diff --git a/src/khoj/interface/web/login.html b/src/khoj/interface/web/login.html
@@ -7,7 +7,7 @@
  <link rel="icon" type="image/png" sizes="128x128" href="/static/assets/icons/favicon-128x128.png">
  <link rel="manifest" href="/static/khoj.webmanifest">
  <link rel="stylesheet" href="/static/assets/khoj.css">
- <meta property="og:image" content="https://khoj-web-bucket.s3.amazonaws.com/khoj_hero.png">
+ <meta property="og:image" content="https://assets.khoj.dev/khoj_hero.png">
  </head>
 
  <body>

diff --git a/src/khoj/routers/storage.py b/src/khoj/routers/storage.py
@@ -6,6 +6,9 @@
 
 AWS_ACCESS_KEY = os.getenv("AWS_ACCESS_KEY")
 AWS_SECRET_KEY = os.getenv("AWS_SECRET_KEY")
+# S3 supports serving assets via your domain. Khoj expects this to be used in production. To enable it:
+# 1. Your bucket name for images should be of the form sub.domain.tld. For example, generated.khoj.dev
+# 2. Add CNAME entry to your domain's DNS records pointing to the S3 bucket. For example, CNAME generated.khoj.dev generated-khoj-dev.s3.amazonaws.com
 AWS_UPLOAD_IMAGE_BUCKET_NAME = os.getenv("AWS_IMAGE_UPLOAD_BUCKET")
 
 aws_enabled = AWS_ACCESS_KEY is not None and AWS_SECRET_KEY is not None and AWS_UPLOAD_IMAGE_BUCKET_NAME is not None
@@ -25,7 +28,7 @@ def upload_image(image: bytes, user_id: uuid.UUID):
  image_key = f"{user_id}/{uuid.uuid4()}.webp"
  try:
  s3_client.put_object(Bucket=AWS_UPLOAD_IMAGE_BUCKET_NAME, Key=image_key, Body=image, ACL="public-read")
- url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}.s3.amazonaws.com/{image_key}"
+ url = f"https://{AWS_UPLOAD_IMAGE_BUCKET_NAME}/{image_key}"
  return url
  except Exception as e:
  logger.error(f"Failed to upload image to S3: {e}")