[Assistants] Filter on names (#841)

* [Assistants] Filter on names * Add `$text` index on `assistant.name` (#844) * add maxlength * better experience * use `$meta: "textScore"` * Update src/routes/assistants/+page.server.ts Co-authored-by: Eliott C. <[email protected]> * null, not undefined * [Assistants] Filter on names (using searchTokens) (#873) Filter with `searchTokens` * input * rm extra whitespace * hide UI before migration * rm ad-hoc migration --------- Co-authored-by: Eliott C. <[email protected]> Co-authored-by: Nathan Sarrazin <[email protected]> Co-authored-by: Victor Mustar <[email protected]>
huggingface · Mar 5, 2024 · 10dbbd6 · 10dbbd6
1 parent 714ff2c
commit 10dbbd6
Show file tree

Hide file tree

Showing 8 changed files with 98 additions and 4 deletions.
diff --git a/src/lib/server/database.ts b/src/lib/server/database.ts
@@ -117,6 +117,7 @@ client.on("open", () => {
  assistants.createIndex({ userCount: 1 }).catch(console.error);
  assistants.createIndex({ featured: 1, userCount: -1 }).catch(console.error);
  assistants.createIndex({ modelId: 1, userCount: -1 }).catch(console.error);
+ assistants.createIndex({ searchTokens: 1 }).catch(console.error);
  reports.createIndex({ assistantId: 1 }).catch(console.error);
  reports.createIndex({ createdBy: 1, assistantId: 1 }).catch(console.error);
 });
diff --git a/src/lib/types/Assistant.ts b/src/lib/types/Assistant.ts
@@ -14,4 +14,5 @@ export interface Assistant extends Timestamps {
  preprompt: string;
  userCount?: number;
  featured?: boolean;
+ searchTokens: string[];
 }
diff --git a/src/lib/utils/debounce.ts b/src/lib/utils/debounce.ts
@@ -0,0 +1,17 @@
+/**
+ * A debounce function that works in both browser and Nodejs.
+ * For pure Nodejs work, prefer the `Debouncer` class.
+ */
+export function debounce<T extends unknown[]>(
+ callback: (...rest: T) => unknown,
+ limit: number
+): (...rest: T) => void {
+ let timer: ReturnType<typeof setTimeout>;
+
+ return function (...rest) {
+ clearTimeout(timer);
+ timer = setTimeout(() => {
+ callback(...rest);
+ }, limit);
+ };
+}
diff --git a/src/lib/utils/searchTokens.ts b/src/lib/utils/searchTokens.ts
@@ -0,0 +1,33 @@
+const PUNCTUATION_REGEX = /\p{P}/gu;
+
+function removeDiacritics(s: string, form: "NFD" | "NFKD" = "NFD"): string {
+ return s.normalize(form).replace(/[\u0300-\u036f]/g, "");
+}
+
+export function generateSearchTokens(value: string): string[] {
+ const fullTitleToken = removeDiacritics(value)
+ .replace(PUNCTUATION_REGEX, "")
+ .replaceAll(/\s+/g, "")
+ .toLowerCase();
+ return [
+ ...new Set([
+ ...removeDiacritics(value)
+ .split(/\s+/)
+ .map((word) => word.replace(PUNCTUATION_REGEX, "").toLowerCase())
+ .filter((word) => word.length),
+ ...(fullTitleToken.length ? [fullTitleToken] : []),
+ ]),
+ ];
+}
+
+function escapeForRegExp(s: string): string {
+ return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); // $& means the whole matched string
+}
+
+export function generateQueryTokens(query: string): RegExp[] {
+ return removeDiacritics(query)
+ .split(/\s+/)
+ .map((word) => word.replace(PUNCTUATION_REGEX, "").toLowerCase())
+ .filter((word) => word.length)
+ .map((token) => new RegExp(`^${escapeForRegExp(token)}`));
+}
diff --git a/src/routes/assistants/+page.server.ts b/src/routes/assistants/+page.server.ts
@@ -3,6 +3,7 @@ import { ENABLE_ASSISTANTS } from "$env/static/private";
 import { collections } from "$lib/server/database.js";
 import type { Assistant } from "$lib/types/Assistant";
 import type { User } from "$lib/types/User";
+import { generateQueryTokens } from "$lib/utils/searchTokens.js";
 import { error, redirect } from "@sveltejs/kit";
 import type { Filter } from "mongodb";
 
@@ -16,6 +17,7 @@ export const load = async ({ url, locals }) => {
  const modelId = url.searchParams.get("modelId");
  const pageIndex = parseInt(url.searchParams.get("p") ?? "0");
  const username = url.searchParams.get("user");
+ const query = url.searchParams.get("q")?.trim() ?? null;
  const createdByCurrentUser = locals.user?.username && locals.user.username === username;
 
  let user: Pick<User, "_id"> | null = null;
@@ -34,6 +36,7 @@ export const load = async ({ url, locals }) => {
  ...(modelId && { modelId }),
  ...(!createdByCurrentUser && { userCount: { $gt: 1 } }),
  ...(user ? { createdById: user._id } : { featured: true }),
+ ...(query && { searchTokens: { $all: generateQueryTokens(query) } }),
  };
  const assistants = await collections.assistants
  .find(filter)
@@ -49,5 +52,6 @@ export const load = async ({ url, locals }) => {
  selectedModel: modelId ?? "",
  numTotalItems,
  numItemsPerPage: NUM_PER_PAGE,
+ query,
  };
 };
diff --git a/src/routes/assistants/+page.svelte b/src/routes/assistants/+page.svelte
@@ -4,6 +4,7 @@
  import { PUBLIC_APP_ASSETS, PUBLIC_ORIGIN } from "$env/static/public";
  import { isHuggingChat } from "$lib/utils/isHuggingChat";
 
+ import { tick } from "svelte";
  import { goto } from "$app/navigation";
  import { base } from "$app/paths";
  import { page } from "$app/stores";
@@ -14,16 +15,22 @@
  import CarbonArrowUpRight from "~icons/carbon/arrow-up-right";
  import CarbonEarthAmerica from "~icons/carbon/earth-americas-filled";
  import CarbonUserMultiple from "~icons/carbon/user-multiple";
+ import CarbonSearch from "~icons/carbon/search";
  import Pagination from "$lib/components/Pagination.svelte";
  import { formatUserCount } from "$lib/utils/formatUserCount";
  import { getHref } from "$lib/utils/getHref";
+ import { debounce } from "$lib/utils/debounce";
  import { useSettingsStore } from "$lib/stores/settings";
 
  export let data: PageData;
 
  $: assistantsCreator = $page.url.searchParams.get("user");
  $: createdByMe = data.user?.username && data.user.username === assistantsCreator;
 
+ const SEARCH_DEBOUNCE_DELAY = 400;
+ let filterInputEl: HTMLInputElement;
+ let searchDisabled = false;
+
  const onModelChange = (e: Event) => {
  const newUrl = getHref($page.url, {
  newKeys: { modelId: (e.target as HTMLSelectElement).value },
@@ -32,6 +39,18 @@
  goto(newUrl);
  };
 
+ const filterOnName = debounce(async (e: Event) => {
+ searchDisabled = true;
+ const value = (e.target as HTMLInputElement).value;
+ const newUrl = getHref($page.url, { newKeys: { q: value } });
+ await goto(newUrl);
+ setTimeout(async () => {
+ searchDisabled = false;
+ await tick();
+ filterInputEl.focus();
+ }, 0);
+ }, SEARCH_DEBOUNCE_DELAY);
+
  const settings = useSettingsStore();
 </script>
 
@@ -99,7 +118,7 @@
  {assistantsCreator}'s Assistants
  <a
  href={getHref($page.url, {
- existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p"] },
+ existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p", "q"] },
  })}
  class="group"
  ><CarbonClose
@@ -119,7 +138,7 @@
  {:else}
  <a
  href={getHref($page.url, {
- existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p"] },
+ existingKeys: { behaviour: "delete", keys: ["user", "modelId", "p", "q"] },
  })}
  class="flex items-center gap-1.5 rounded-full border px-3 py-1 {!assistantsCreator
  ? 'border-gray-300 bg-gray-50 dark:border-gray-600 dark:bg-gray-700 dark:text-white'
@@ -132,16 +151,31 @@
  <a
  href={getHref($page.url, {
  newKeys: { user: data.user.username },
- existingKeys: { behaviour: "delete", keys: ["modelId", "p"] },
+ existingKeys: { behaviour: "delete", keys: ["modelId", "p", "q"] },
  })}
- class="flex items-center gap-1.5 rounded-full border px-3 py-1 {assistantsCreator &&
+ class="flex items-center gap-1.5 truncate rounded-full border px-3 py-1 {assistantsCreator &&
  createdByMe
  ? 'border-gray-300 bg-gray-50 dark:border-gray-600 dark:bg-gray-700 dark:text-white'
  : 'border-transparent text-gray-400 hover:text-gray-800 dark:hover:text-gray-300'}"
  >{data.user.username}
  </a>
  {/if}
  {/if}
+ <div
+ class="relative ml-auto flex hidden h-[30px] w-40 items-center rounded-full border px-2 has-[:focus]:border-gray-400 sm:w-64 dark:border-gray-600"
+ >
+ <CarbonSearch class="pointer-events-none absolute left-2 text-xs text-gray-400" />
+ <input
+ class="h-[30px] w-full bg-transparent pl-5 focus:outline-none"
+ placeholder="Filter by name"
+ value={data.query}
+ on:input={filterOnName}
+ bind:this={filterInputEl}
+ maxlength="150"
+ type="search"
+ disabled={searchDisabled}
+ />
+ </div>
  </div>
 
  <div class="mt-8 grid grid-cols-2 gap-3 sm:gap-5 md:grid-cols-3 lg:grid-cols-4">

diff --git a/src/routes/settings/assistants/[assistantId]/edit/+page.server.ts b/src/routes/settings/assistants/[assistantId]/edit/+page.server.ts
@@ -8,6 +8,7 @@ import { z } from "zod";
 import { sha256 } from "$lib/utils/sha256";
 
 import sharp from "sharp";
+import { generateSearchTokens } from "$lib/utils/searchTokens";
 
 const newAsssistantSchema = z.object({
  name: z.string().min(1),
@@ -130,6 +131,7 @@ export const actions: Actions = {
  exampleInputs,
  avatar: deleteAvatar ? undefined : hash ?? assistant.avatar,
  updatedAt: new Date(),
+ searchTokens: generateSearchTokens(parse.data.name),
  },
  }
  );

diff --git a/src/routes/settings/assistants/new/+page.server.ts b/src/routes/settings/assistants/new/+page.server.ts
@@ -7,6 +7,7 @@ import { ObjectId } from "mongodb";
 import { z } from "zod";
 import { sha256 } from "$lib/utils/sha256";
 import sharp from "sharp";
+import { generateSearchTokens } from "$lib/utils/searchTokens";
 
 const newAsssistantSchema = z.object({
  name: z.string().min(1),
@@ -99,6 +100,7 @@ export const actions: Actions = {
  updatedAt: new Date(),
  userCount: 1,
  featured: false,
+ searchTokens: generateSearchTokens(parse.data.name),
  });
 
  // add insertedId to user settings