Implemend PDF-chat feature

package.json

@@ -61,6 +61,7 @@
  "nanoid": "^4.0.2",
  "openid-client": "^5.4.2",
  "parquetjs": "^0.11.2",
+ "pdfjs-dist": "^4.0.269",
  "postcss": "^8.4.31",
  "serpapi": "^1.1.1",
  "tailwind-scrollbar": "^3.0.0",

src/lib/buildPrompt.ts

@@ -2,7 +2,8 @@ import type { BackendModel } from "./server/models";
 import type { Message } from "./types/Message";
 import { format } from "date-fns";
 import type { WebSearch } from "./types/WebSearch";
-import { downloadFile } from "./server/files/downloadFile";
+import type { PdfSearch } from "./types/PdfSearch";
+import { downloadImgFile } from "./server/files/downloadFile";
 import type { Conversation } from "./types/Conversation";
 
 interface buildPromptOptions {
@@ -11,6 +12,7 @@ interface buildPromptOptions {
  model: BackendModel;
  locals?: App.Locals;
  webSearch?: WebSearch;
+ pdfSearch?: PdfSearch;
  preprompt?: string;
  files?: File[];
 }
@@ -19,6 +21,7 @@ export async function buildPrompt({
  messages,
  model,
  webSearch,
+ pdfSearch,
  preprompt,
  id,
 }: buildPromptOptions): Promise<string> {
@@ -47,6 +50,31 @@ export async function buildPrompt({
  `,
  },
  ];
+ }else if (pdfSearch && pdfSearch.context) {
+ const lastMsg = messages.slice(-1)[0];
+ const messagesWithoutLastUsrMsg = messages.slice(0, -1);
+ const previousUserMessages = messages.filter((el) => el.from === "user").slice(0, -1);
+
+ const previousQuestions =
+ previousUserMessages.length > 0
+ ? `Previous questions: \n${previousUserMessages
+ .map(({ content }) => `- ${content}`)
+ .join("\n")}`
+ : "";
+
+ messages = [
+ ...messagesWithoutLastUsrMsg,
+ {
+ from: "user",
+ content: `Below are the information I extracted from a PDF file that might be useful:
+ =====================
+ ${pdfSearch.context}
+ =====================
+ ${previousQuestions}
+ Answer the question: ${lastMsg.content} 
+ `,
+ },
+ ];
  }
 
  // section to handle potential files input
@@ -60,7 +88,7 @@ export async function buildPrompt({
  const markdowns = await Promise.all(
  el.files.map(async (hash) => {
  try {
- const { content: image, mime } = await downloadFile(hash, id);
+ const { content: image, mime } = await downloadImgFile(hash, id);
  const b64 = image.toString("base64");
  return `![](data:${mime};base64,${b64})})`;
  } catch (e) {

src/lib/components/OpenPdfSearchResults.svelte

@@ -0,0 +1,114 @@
+<script lang="ts">
+ import type { PdfSearchUpdate } from "$lib/types/MessageUpdate";
+ import CarbonCaretRight from "~icons/carbon/caret-right";
+
+ import CarbonCheckmark from "~icons/carbon/checkmark-filled";
+ import CarbonError from "~icons/carbon/error-filled";
+
+ import EosIconsLoading from "~icons/eos-icons/loading";
+
+ export let loading = false;
+ export let classNames = "";
+ export let pdfSearchMessages: PdfSearchUpdate[] = [];
+
+ let detailsOpen: boolean;
+ let error: boolean;
+ $: error = pdfSearchMessages[pdfSearchMessages.length - 1]?.messageType === "error";
+</script>
+
+<details
+ class="flex w-fit rounded-xl border border-gray-200 bg-white shadow-sm dark:border-gray-800 dark:bg-gray-900 {classNames} max-w-full"
+ bind:open={detailsOpen}
+>
+ <summary
+ class="align-center flex cursor-pointer select-none list-none py-1 pl-2.5 pr-2 align-text-top transition-all"
+ >
+ {#if error}
+ <CarbonError class="my-auto text-red-700 dark:text-red-500" />
+ {:else if loading}
+ <EosIconsLoading class="my-auto text-gray-500" />
+ {:else}
+ <CarbonCheckmark class="my-auto text-gray-500" />
+ {/if}
+ <span class="px-2 font-medium" class:text-red-700={error} class:dark:text-red-500={error}
+ >PDF search
+ </span>
+ <div class="my-auto transition-all" class:rotate-90={detailsOpen}>
+ <CarbonCaretRight />
+ </div>
+ </summary>
+
+ <div class="content px-5 pb-5 pt-4">
+ {#if pdfSearchMessages.length === 0}
+ <div class="mx-auto w-fit">
+ <EosIconsLoading class="mb-3 h-4 w-4" />
+ </div>
+ {:else}
+ <ol>
+ {#each pdfSearchMessages as message}
+ {#if message.messageType === "update"}
+ <li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+ <div class="flex items-start">
+ <div
+ class="-ml-1.5 h-3 w-3 flex-none rounded-full bg-gray-200 dark:bg-gray-600 {loading
+ ? 'group-last:animate-pulse group-last:bg-gray-300 group-last:dark:bg-gray-500'
+ : ''}"
+ />
+ <h3 class="text-md -mt-1.5 pl-2.5 text-gray-800 dark:text-gray-100">
+ {message.message}
+ </h3>
+ </div>
+ {#if message.args}
+ <p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+ {message.args}
+ </p>
+ {/if}
+ </li>
+ {:else if message.messageType === "error"}
+ <li class="group border-l pb-6 last:!border-transparent last:pb-0 dark:border-gray-800">
+ <div class="flex items-start">
+ <CarbonError
+ class="-ml-1.5 h-3 w-3 flex-none scale-110 text-red-700 dark:text-red-500"
+ />
+ <h3 class="text-md -mt-1.5 pl-2.5 text-red-700 dark:text-red-500">
+ {message.message}
+ </h3>
+ </div>
+ {#if message.args}
+ <p class="mt-1.5 pl-4 text-gray-500 dark:text-gray-400">
+ {message.args}
+ </p>
+ {/if}
+ </li>
+ {/if}
+ {/each}
+ </ol>
+ {/if}
+ </div>
+</details>
+
+<style>
+ @keyframes grow {
+ 0% {
+ font-size: 0;
+ opacity: 0;
+ }
+ 30% {
+ font-size: 1em;
+ opacity: 0;
+ }
+ 100% {
+ opacity: 1;
+ }
+ }
+
+ details[open] .content {
+ animation-name: grow;
+ animation-duration: 300ms;
+ animation-delay: 0ms;
+ }
+
+ details summary::-webkit-details-marker {
+ display: none;
+ }
+</style>

src/lib/components/UploadBtn.svelte

@@ -1,23 +1,52 @@
 <script lang="ts">
+ import {createEventDispatcher} from "svelte";
  import CarbonUpload from "~icons/carbon/upload";
 
  export let classNames = "";
+ export let multimodal = false;
  export let files: File[];
- let filelist: FileList;
+ export let uploadingPdf = false;
+ 
+ const accept = multimodal ? "image/*,.pdf" : ".pdf";
+ const label = multimodal ? "Upload image or PDF" : "Upload PDF";
+ let fileInput: HTMLInputElement;
 
- $: if (filelist) {
- files = Array.from(filelist);
+ const dispatch = createEventDispatcher<{
+ uploadpdf: File;
+ }>();
+
+ function onChange() {
+ if(!fileInput.files){
+ return;
+ }
+ 
+ const file = fileInput.files?.[0];
+ if (file?.type === "application/pdf") {
+ // pdf upload
+ dispatch("uploadpdf", file);
+ }else{
+ // image files for multimodal models
+ files = Array.from(fileInput.files);
+ }
  }
 </script>
 
 <button
  class="btn relative h-8 rounded-lg border bg-white px-3 py-1 text-sm text-gray-500 shadow-sm transition-all hover:bg-gray-100 dark:border-gray-600 dark:bg-gray-700 dark:text-gray-300 dark:hover:bg-gray-600 {classNames}"
+ class:animate-pulse={uploadingPdf}
+ class:pointer-events-none={uploadingPdf}
 >
  <input
- bind:files={filelist}
+ bind:this={fileInput}
+ on:change={onChange}
  class="absolute w-full cursor-pointer opacity-0"
  type="file"
- accept="image/*"
+ {accept}
  />
- <CarbonUpload class="mr-2 text-xs " /> Upload image
+ <CarbonUpload class="mr-2 text-xs " />
+ {#if uploadingPdf}
+ Processing PDF file
+ {:else}
+ {label}
+ {/if}
 </button>

src/lib/components/chat/ChatMessage.svelte

@@ -17,7 +17,8 @@
  import type { Model } from "$lib/types/Model";
 
  import OpenWebSearchResults from "../OpenWebSearchResults.svelte";
- import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+ import OpenPdfSearchResults from "../OpenPdfSearchResults.svelte";
+ import type { RAGUpdate, WebSearchUpdate, PdfSearchUpdate } from "$lib/types/MessageUpdate";
 
  function sanitizeMd(md: string) {
  let ret = md
@@ -49,7 +50,7 @@
  export let readOnly = false;
  export let isTapped = false;
 
- export let webSearchMessages: WebSearchUpdate[];
+ export let RAGMessages: RAGUpdate[];
 
  const dispatch = createEventDispatcher<{
  retry: { content: string; id: Message["id"] };
@@ -108,9 +109,15 @@
 
  let searchUpdates: WebSearchUpdate[] = [];
 
- $: searchUpdates = ((webSearchMessages.length > 0
- ? webSearchMessages
+ $: searchUpdates = ((RAGMessages.filter(({type}) => type === "webSearch").length > 0
+ ? RAGMessages.filter(({type}) => type === "webSearch")
  : message.updates?.filter(({ type }) => type === "webSearch")) ?? []) as WebSearchUpdate[];
+ 
+ let pdfUpdates: PdfSearchUpdate[] = [];
+
+ $: pdfUpdates = ((RAGMessages.filter(({type}) => type === "pdfSearch").length > 0
+ ? RAGMessages.filter(({type}) => type === "pdfSearch")
+ : message.updates?.filter(({ type }) => type === "pdfSearch")) ?? []) as PdfSearchUpdate[];
 
  $: downloadLink =
  message.from === "user" ? `${$page.url.pathname}/message/${message.id}/prompt` : undefined;
@@ -153,7 +160,14 @@
  loading={!(searchUpdates[searchUpdates.length - 1]?.messageType === "sources")}
  />
  {/if}
- {#if !message.content && (webSearchIsDone || (webSearchMessages && webSearchMessages.length === 0))}
+ {#if pdfUpdates && pdfUpdates.length > 0}
+ <OpenPdfSearchResults
+ classNames={tokens.length ? "mb-3.5" : ""}
+ pdfSearchMessages={pdfUpdates}
+ loading={!(pdfUpdates[pdfUpdates.length - 1]?.messageType === "done")}
+ />
+ {/if}
+ {#if !message.content && (webSearchIsDone || (RAGMessages && RAGMessages.length === 0))}
  <IconLoading />
  {/if}
 

src/lib/components/chat/ChatMessages.svelte

@@ -8,7 +8,7 @@
  import type { LayoutData } from "../../../routes/$types";
  import ChatIntroduction from "./ChatIntroduction.svelte";
  import ChatMessage from "./ChatMessage.svelte";
- import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+ import type { RAGUpdate } from "$lib/types/MessageUpdate";
  import { browser } from "$app/environment";
  import SystemPromptModal from "../SystemPromptModal.svelte";
  import { page } from "$app/stores";
@@ -25,7 +25,7 @@
 
  let chatContainer: HTMLElement;
 
- export let webSearchMessages: WebSearchUpdate[] = [];
+ export let RAGMessages: RAGUpdate[] = [];
 
  async function scrollToBottom() {
  await tick();
@@ -40,7 +40,7 @@
 
 <div
  class="scrollbar-custom mr-1 h-full overflow-y-auto"
- use:snapScrollToBottom={messages.length ? [...messages, ...webSearchMessages] : false}
+ use:snapScrollToBottom={messages.length ? [...messages, ...RAGMessages] : false}
  bind:this={chatContainer}
 >
  <div class="mx-auto flex h-full max-w-3xl flex-col gap-6 px-5 pt-6 sm:gap-8 xl:max-w-4xl">
@@ -54,7 +54,7 @@
  {isAuthor}
  {readOnly}
  model={currentModel}
- webSearchMessages={i === messages.length - 1 ? webSearchMessages : []}
+ RAGMessages={i === messages.length - 1 ? RAGMessages : []}
  on:retry
  on:vote
  />
@@ -65,7 +65,7 @@
  <ChatMessage
  message={{ from: "assistant", content: "", id: randomUUID() }}
  model={currentModel}
- {webSearchMessages}
+ {RAGMessages}
  />
  {/if}
  <div class="h-44 flex-none" />

src/lib/components/chat/ChatWindow.svelte

@@ -15,7 +15,7 @@
  import type { Model } from "$lib/types/Model";
  import WebSearchToggle from "../WebSearchToggle.svelte";
  import LoginModal from "../LoginModal.svelte";
- import type { WebSearchUpdate } from "$lib/types/MessageUpdate";
+ import type { RAGUpdate } from "$lib/types/MessageUpdate";
  import { page } from "$app/stores";
  import DisclaimerModal from "../DisclaimerModal.svelte";
  import FileDropzone from "./FileDropzone.svelte";
@@ -30,9 +30,10 @@
  export let shared = false;
  export let currentModel: Model;
  export let models: Model[];
- export let webSearchMessages: WebSearchUpdate[] = [];
+ export let RAGMessages: RAGUpdate[] = [];
  export let preprompt: string | undefined = undefined;
  export let files: File[] = [];
+ export let uploadingPdf: boolean = false;
 
  $: isReadOnly = !models.some((model) => model.id === currentModel.id);
 
@@ -94,7 +95,7 @@
  {messages}
  readOnly={isReadOnly}
  isAuthor={!shared}
- {webSearchMessages}
+ {RAGMessages}
  {preprompt}
  on:message={(ev) => {
  if ($page.data.loginRequired) {
@@ -153,9 +154,8 @@
  content: messages[messages.length - 1].content,
  })}
  />
- {:else if currentModel.multimodal}
- <UploadBtn bind:files classNames="ml-auto" />
  {/if}
+ <UploadBtn bind:files on:uploadpdf classNames="ml-auto" multimodal={currentModel.multimodal} {uploadingPdf} />
  </div>
  <form
  on:dragover={onDragOver}