Count system prompt tokens

src/lib/components/AssistantSettings.svelte

@@ -12,6 +12,7 @@
 
  import { useSettingsStore } from "$lib/stores/settings";
  import { isHuggingChat } from "$lib/utils/isHuggingChat";
+ import TokensCounter from "./TokensCounter.svelte";
 
  type ActionData = {
  error: boolean;
@@ -28,6 +29,10 @@
  export let models: Model[] = [];
 
  let files: FileList | null = null;
+ let selectedModel: Model | undefined = models.find(
+ (model) => assistant?.modelId && assistant.modelId === model.id
+ );
+ let enteredSystemPrompt = "";
 
  const settings = useSettingsStore();
 
@@ -238,7 +243,12 @@
 
  <label>
  <div class="mb-1 font-semibold">Model</div>
- <select name="modelId" class="w-full rounded-lg border-2 border-gray-200 bg-gray-100 p-2">
+ <select
+ name="modelId"
+ class="w-full rounded-lg border-2 border-gray-200 bg-gray-100 p-2"
+ on:change={(e) =>
+ (selectedModel = models.find((model) => model.id === e.currentTarget.value))}
+ >
  {#each models.filter((model) => !model.unlisted) as model}
  <option
  value={model.id}
@@ -390,12 +400,20 @@
 
  <div class="col-span-1 flex h-full flex-col">
  <span class="mb-1 text-sm font-semibold"> Instructions (system prompt) </span>
- <textarea
- name="preprompt"
- class="mb-20 min-h-[8lh] flex-1 rounded-lg border-2 border-gray-200 bg-gray-100 p-2 text-sm"
- placeholder="You'll act as..."
- value={assistant?.preprompt ?? ""}
- />
+ <div class="relative mb-20 flex min-h-[8lh] flex-1 grow flex-col">
+ <textarea
+ name="preprompt"
+ class="flex-1 rounded-lg border-2 border-gray-200 bg-gray-100 p-2 text-sm"
+ placeholder="You'll act as..."
+ value={assistant?.preprompt ?? ""}
+ on:input={(e) => (enteredSystemPrompt = e.currentTarget.value)}
+ />
+ <TokensCounter
+ classNames="absolute bottom-2 right-2"
+ prompt={enteredSystemPrompt}
+ model={selectedModel}
+ />
+ </div>
  <p class="text-xs text-red-500">{getError("preprompt", form)}</p>
  </div>
  </div>

src/lib/components/TokensCounter.svelte

@@ -0,0 +1,50 @@
+<script lang="ts">
+ import type { Model } from "$lib/types/Model";
+ import { AutoTokenizer } from "@xenova/transformers";
+
+ export let classNames = "";
+ export let prompt = "";
+ export let model: Model | undefined = undefined;
+
+ let nTokens = 0;
+ let isDisabled = false;
+ let modelId = model?.id;
+
+ async function tokenizeText() {
+ if (isDisabled || !model || !prompt) {
+ nTokens = 0;
+ return;
+ }
+
+ try {
+ const tokenizer = await AutoTokenizer.from_pretrained(model.id);
+ const { input_ids } = await tokenizer(prompt);
+ nTokens = input_ids.size;
+ isDisabled = false;
+ } catch (err) {
+ isDisabled = true;
+ console.error("Error while counting tokens: ", err);
+ }
+ }
+
+ $: {
+ if (typeof window !== "undefined" && model) {
+ tokenizeText();
+ }
+ }
+
+ $: {
+ if (model?.id !== modelId) {
+ // reset
+ modelId = model?.id;
+ nTokens = 0;
+ isDisabled = false;
+ }
+ }
+</script>
+
+{#if model?.parameters?.max_new_tokens && nTokens}
+ <p class="text-sm opacity-60 hover:opacity-80 {classNames}">
+ {nTokens}/{model.parameters.max_new_tokens}
+ </p>
+{/if}

src/routes/settings/(nav)/[...model]/+page.svelte

@@ -5,6 +5,7 @@
  import type { BackendModel } from "$lib/server/models";
  import { useSettingsStore } from "$lib/stores/settings";
  import CopyToClipBoardBtn from "$lib/components/CopyToClipBoardBtn.svelte";
+ import TokensCounter from "$lib/components/TokensCounter.svelte";
  import CarbonArrowUpRight from "~icons/carbon/arrow-up-right";
  import CarbonLink from "~icons/carbon/link";
 
@@ -99,7 +100,7 @@
  {isActive ? "Active model" : "Activate"}
  </button>
 
- <div class="flex w-full flex-col gap-2">
+ <div class="relative flex w-full flex-col gap-2">
  <div class="flex w-full flex-row content-between">
  <h3 class="mb-1.5 text-lg font-semibold text-gray-800">System Prompt</h3>
  {#if hasCustomPreprompt}
@@ -117,5 +118,10 @@
  class="w-full resize-none rounded-md border-2 bg-gray-100 p-2"
  bind:value={$settings.customPrompts[$page.params.model]}
  />
+ <TokensCounter
+ classNames="absolute bottom-2 right-2"
+ prompt={$settings.customPrompts[$page.params.model]}
+ {model}
+ />
  </div>
 </div>