Skip to content

Commit

Permalink
Generate dynamic diagrams (via Excalidraw) (#940)
Browse files Browse the repository at this point in the history 
Add support for generating dynamic diagrams in flow with Excalidraw (https://github.com/excalidraw/excalidraw). This happens in three steps:
1. Default information collection & intent determination step.
2. Improving the overall guidance of the prompt for generating a JSON, Excalidraw-compatible declaration.
3. Generation of the diagram to output to the final UI.

Add support in the web UI.
  • Loading branch information
@sabaimran
sabaimran authored Oct 22, 2024
1 parent 1e993d5 commit 0dad421
Show file tree
Hide file tree
Showing 24 changed files with 689 additions and 100 deletions.
14 changes: 10 additions & 4 deletions src/interface/desktop/chat.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -326,7 +326,7 @@
entries.forEach(entry => {
// If the element is in the viewport, fetch the remaining message and unobserve the element
if (entry.isIntersecting) {
fetchRemainingChatMessages(chatHistoryUrl, headers);
fetchRemainingChatMessages(chatHistoryUrl, headers, chatBody.dataset.conversation_id, hostURL);
observer.unobserve(entry.target);
}
});
Expand All @@ -342,7 +342,11 @@
new Date(chat_log.created),
chat_log.onlineContext,
chat_log.intent?.type,
chat_log.intent?.["inferred-queries"]);
chat_log.intent?.["inferred-queries"],
chatBody.dataset.conversationId ?? "",
hostURL,
);

chatBody.appendChild(messageElement);

// When the 4th oldest message is within viewing distance (~60% scrolled up)
Expand Down Expand Up @@ -421,7 +425,7 @@
}
}

function fetchRemainingChatMessages(chatHistoryUrl, headers) {
function fetchRemainingChatMessages(chatHistoryUrl, headers, conversationId, hostURL) {
// Create a new IntersectionObserver
let observer = new IntersectionObserver((entries, observer) => {
entries.forEach(entry => {
Expand All @@ -435,7 +439,9 @@
new Date(chat_log.created),
chat_log.onlineContext,
chat_log.intent?.type,
chat_log.intent?.["inferred-queries"]
chat_log.intent?.["inferred-queries"],
chatBody.dataset.conversationId ?? "",
hostURL,
);
entry.target.replaceWith(messageElement);

Expand Down
15 changes: 13 additions & 2 deletions src/interface/desktop/chatutils.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,11 +189,19 @@ function processOnlineReferences(referenceSection, onlineContext) { //same
return numOnlineReferences;
}

function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null) { //same
function renderMessageWithReference(message, by, context=null, dt=null, onlineContext=null, intentType=null, inferredQueries=null, conversationId=null, hostURL=null) {
let chatEl;
if (intentType?.includes("text-to-image")) {
let imageMarkdown = generateImageMarkdown(message, intentType, inferredQueries);
chatEl = renderMessage(imageMarkdown, by, dt, null, false, "return");
} else if (intentType === "excalidraw") {
let domain = hostURL ?? "https://app.khoj.dev/";

if (!domain.endsWith("/")) domain += "/";

let excalidrawMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in the web app at ${domain}chat?conversationId=${conversationId}`;

chatEl = renderMessage(excalidrawMessage, by, dt, null, false, "return");
} else {
chatEl = renderMessage(message, by, dt, null, false, "return");
}
Expand Down Expand Up @@ -312,7 +320,6 @@ function formatHTMLMessage(message, raw=false, willReplace=true) { //same
}

function createReferenceSection(references, createLinkerSection=false) {
console.log("linker data: ", createLinkerSection);
let referenceSection = document.createElement('div');
referenceSection.classList.add("reference-section");
referenceSection.classList.add("collapsed");
Expand Down Expand Up @@ -417,7 +424,11 @@ function handleImageResponse(imageJson, rawResponse) {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
} else if (imageJson.intentType === "excalidraw") {
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in the web app`;
rawResponse += redirectMessage;
}

if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
}
Expand Down
16 changes: 13 additions & 3 deletions src/interface/obsidian/src/chat_view.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -484,12 +484,13 @@ export class KhojChatView extends KhojPaneView {
dt?: Date,
intentType?: string,
inferredQueries?: string[],
conversationId?: string,
) {
if (!message) return;

let chatMessageEl;
if (intentType?.includes("text-to-image")) {
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries);
if (intentType?.includes("text-to-image") || intentType === "excalidraw") {
let imageMarkdown = this.generateImageMarkdown(message, intentType, inferredQueries, conversationId);
chatMessageEl = this.renderMessage(chatEl, imageMarkdown, sender, dt);
} else {
chatMessageEl = this.renderMessage(chatEl, message, sender, dt);
Expand All @@ -509,14 +510,18 @@ export class KhojChatView extends KhojPaneView {
chatMessageBodyEl.appendChild(this.createReferenceSection(references));
}

generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[]) {
generateImageMarkdown(message: string, intentType: string, inferredQueries?: string[], conversationId?: string): string {
let imageMarkdown = "";
if (intentType === "text-to-image") {
imageMarkdown = `![](data:image/png;base64,${message})`;
} else if (intentType === "text-to-image2") {
imageMarkdown = `![](${message})`;
} else if (intentType === "text-to-image-v3") {
imageMarkdown = `![](data:image/webp;base64,${message})`;
} else if (intentType === "excalidraw") {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}chat?conversationId=${conversationId}`;
imageMarkdown = redirectMessage;
}
if (inferredQueries) {
imageMarkdown += "\n\n**Inferred Query**:";
Expand Down Expand Up @@ -884,6 +889,7 @@ export class KhojChatView extends KhojPaneView {
new Date(chatLog.created),
chatLog.intent?.type,
chatLog.intent?.["inferred-queries"],
chatBodyEl.dataset.conversationId ?? "",
);
// push the user messages to the chat history
if(chatLog.by === "you"){
Expand Down Expand Up @@ -1354,6 +1360,10 @@ export class KhojChatView extends KhojPaneView {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
} else if (imageJson.intentType === "excalidraw") {
const domain = this.setting.khojUrl.endsWith("/") ? this.setting.khojUrl : `${this.setting.khojUrl}/`;
const redirectMessage = `Hey, I'm not ready to show you diagrams yet here. But you can view it in ${domain}`;
rawResponse += redirectMessage;
}
if (inferredQuery) {
rawResponse += `\n\n**Inferred Query**:\n\n${inferredQuery}`;
Expand Down
2 changes: 1 addition & 1 deletion src/interface/web/app/chat/chat.module.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ div.titleBar {
div.chatBoxBody {
display: grid;
height: 100%;
width: 70%;
width: 95%;
margin: auto;
}

Expand Down
9 changes: 8 additions & 1 deletion src/interface/web/app/chat/layout.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,14 @@ export default function RootLayout({
child-src 'none';
object-src 'none';"
></meta>
<body className={inter.className}>{children}</body>
<body className={inter.className}>
{children}
<script
dangerouslySetInnerHTML={{
__html: `window.EXCALIDRAW_ASSET_PATH = 'https://assets.khoj.dev/@excalidraw/excalidraw/dist/';`,
}}
/>
</body>
</html>
);
}
7 changes: 4 additions & 3 deletions src/interface/web/app/chat/page.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,6 @@ import { useIPLocationData, useIsMobileWidth, welcomeConsole } from "../common/u
import ChatInputArea, { ChatOptions } from "../components/chatInputArea/chatInputArea";
import { useAuthenticatedData } from "../common/auth";
import { AgentData } from "../agents/page";
import { DotsThreeVertical } from "@phosphor-icons/react";
import { Button } from "@/components/ui/button";

interface ChatBodyDataProps {
chatOptionsData: ChatOptions | null;
Expand All @@ -43,6 +41,8 @@ function ChatBodyData(props: ChatBodyDataProps) {
const setQueryToProcess = props.setQueryToProcess;
const onConversationIdChange = props.onConversationIdChange;

const chatHistoryCustomClassName = props.isMobileWidth ? "w-full" : "w-4/6";

useEffect(() => {
if (image) {
props.setImage64(encodeURIComponent(image));
Expand Down Expand Up @@ -103,10 +103,11 @@ function ChatBodyData(props: ChatBodyDataProps) {
setAgent={setAgentMetadata}
pendingMessage={processingMessage ? message : ""}
incomingMessages={props.streamedMessages}
customClassName={chatHistoryCustomClassName}
/>
</div>
<div
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit`}
className={`${styles.inputBox} p-1 md:px-2 shadow-md bg-background align-middle items-center justify-center dark:bg-neutral-700 dark:border-0 dark:shadow-sm rounded-t-2xl rounded-b-none md:rounded-xl h-fit ${chatHistoryCustomClassName} mr-auto ml-auto`}
>
<ChatInputArea
agentColor={agentMetadata?.color}
Expand Down
69 changes: 35 additions & 34 deletions src/interface/web/app/common/chatFunctions.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,10 +5,10 @@ export interface RawReferenceData {
onlineContext?: OnlineContext;
}

export interface ResponseWithReferences {
context?: Context[];
online?: OnlineContext;
response?: string;
export interface ResponseWithIntent {
intentType: string;
response: string;
inferredQueries?: string[];
}

interface MessageChunk {
Expand Down Expand Up @@ -49,10 +49,14 @@ export function convertMessageChunkToJson(chunk: string): MessageChunk {
function handleJsonResponse(chunkData: any) {
const jsonData = chunkData as any;
if (jsonData.image || jsonData.detail) {
let responseWithReference = handleImageResponse(chunkData, true);
if (responseWithReference.response) return responseWithReference.response;
let responseWithIntent = handleImageResponse(chunkData, true);
return responseWithIntent;
} else if (jsonData.response) {
return jsonData.response;
return {
response: jsonData.response,
intentType: "",
inferredQueries: [],
};
} else {
throw new Error("Invalid JSON response");
}
Expand Down Expand Up @@ -80,16 +84,29 @@ export function processMessageChunk(
return { context, onlineContext };
} else if (chunk.type === "message") {
const chunkData = chunk.data;
// Here, handle if the response is a JSON response with an image, but the intentType is excalidraw
if (chunkData !== null && typeof chunkData === "object") {
currentMessage.rawResponse += handleJsonResponse(chunkData);
let responseWithIntent = handleJsonResponse(chunkData);

if (responseWithIntent.intentType && responseWithIntent.intentType === "excalidraw") {
currentMessage.rawResponse = responseWithIntent.response;
} else {
currentMessage.rawResponse += responseWithIntent.response;
}

currentMessage.intentType = responseWithIntent.intentType;
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
} else if (
typeof chunkData === "string" &&
chunkData.trim()?.startsWith("{") &&
chunkData.trim()?.endsWith("}")
) {
try {
const jsonData = JSON.parse(chunkData.trim());
currentMessage.rawResponse += handleJsonResponse(jsonData);
let responseWithIntent = handleJsonResponse(jsonData);
currentMessage.rawResponse += responseWithIntent.response;
currentMessage.intentType = responseWithIntent.intentType;
currentMessage.inferredQueries = responseWithIntent.inferredQueries;
} catch (e) {
currentMessage.rawResponse += JSON.stringify(chunkData);
}
Expand All @@ -111,42 +128,26 @@ export function processMessageChunk(
return { context, onlineContext };
}

export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithReferences {
export function handleImageResponse(imageJson: any, liveStream: boolean): ResponseWithIntent {
let rawResponse = "";

if (imageJson.image) {
const inferredQuery = imageJson.inferredQueries?.[0] ?? "generated image";

// If response has image field, response is a generated image.
if (imageJson.intentType === "text-to-image") {
rawResponse += `![generated_image](data:image/png;base64,${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image2") {
rawResponse += `![generated_image](${imageJson.image})`;
} else if (imageJson.intentType === "text-to-image-v3") {
rawResponse = `![](data:image/webp;base64,${imageJson.image})`;
}
if (inferredQuery && !liveStream) {
rawResponse += `\n\n${inferredQuery}`;
}
// If response has image field, response may be a generated image
rawResponse = imageJson.image;
}

let reference: ResponseWithReferences = {};
let responseWithIntent: ResponseWithIntent = {
intentType: imageJson.intentType,
response: rawResponse,
inferredQueries: imageJson.inferredQueries,
};

if (imageJson.context && imageJson.context.length > 0) {
const rawReferenceAsJson = imageJson.context;
if (rawReferenceAsJson instanceof Array) {
reference.context = rawReferenceAsJson;
} else if (typeof rawReferenceAsJson === "object" && rawReferenceAsJson !== null) {
reference.online = rawReferenceAsJson;
}
}
if (imageJson.detail) {
// The detail field contains the improved image prompt
rawResponse += imageJson.detail;
}

reference.response = rawResponse;
return reference;
return responseWithIntent;
}

export function modifyFileFilterForConversation(
Expand Down
5 changes: 5 additions & 0 deletions src/interface/web/app/common/iconUtils.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import {
Oven,
Gavel,
Broadcast,
KeyReturn,
} from "@phosphor-icons/react";
import { Markdown, OrgMode, Pdf, Word } from "@/app/components/logo/fileLogo";

Expand Down Expand Up @@ -193,6 +194,10 @@ export function getIconForSlashCommand(command: string, customClassName: string
}

if (command.includes("default")) {
return <KeyReturn className={className} />;
}

if (command.includes("diagram")) {
return <Shapes className={className} />;
}

Expand Down
7 changes: 1 addition & 6 deletions src/interface/web/app/components/chatHistory/chatHistory.module.css
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,7 @@ div.chatHistory {
display: flex;
flex-direction: column;
height: 100%;
}

div.chatLayout {
height: 80vh;
overflow-y: auto;
margin: 0 auto;
margin: auto;
}

div.agentIndicator a {
Expand Down
Loading

0 comments on commit 0dad421

Please sign in to comment.