Merge pull request #114 from jackschedel/2.1.0

context fix + token count in default
jackschedel · Feb 12, 2024 · b563ce1 · b563ce1
2 parents d5bfb23 + 9d4b936
commit b563ce1
Show file tree

Hide file tree

Showing 4 changed files with 14 additions and 21 deletions.
diff --git a/src/components/ChatConfigMenu/ChatConfigMenu.tsx b/src/components/ChatConfigMenu/ChatConfigMenu.tsx
@@ -5,6 +5,7 @@ import { useTranslation } from 'react-i18next';
 import PopupModal from '@components/PopupModal';
 import {
   FrequencyPenaltySlider,
+  MaxTokenSlider,
   PresencePenaltySlider,
   TemperatureSlider,
   TopPSlider,
@@ -48,6 +49,7 @@ const ChatConfigPopup = ({
   const [_modelSelection, _setModelSelection] = useState<number>(
     config.model_selection
   );
+  const [_maxTokens, _setMaxTokens] = useState<number>(config.max_tokens);
   const [_maxToken, _setMaxToken] = useState<number>(config.max_tokens);
   const [_temperature, _setTemperature] = useState<number>(config.temperature);
   const [_topP, _setTopP] = useState<number>(config.top_p);
@@ -97,6 +99,11 @@ const ChatConfigPopup = ({
           _setModel={_setModelSelection}
           showHidden={true}
         />
+        <MaxTokenSlider
+          _maxToken={_maxTokens}
+          _setMaxToken={_setMaxTokens}
+          _model={_modelSelection}
+        />
         <DefaultSystemChat
           _systemMessage={_systemMessage}
           _setSystemMessage={_setSystemMessage}

diff --git a/src/constants/chat.ts b/src/constants/chat.ts
@@ -95,7 +95,7 @@ export const defaultUserMaxContext = 8000;
 
 export const _defaultChatConfig: ConfigInterface = {
   model_selection: 0,
-  temperature: 1,
+  temperature: 0.7,
   presence_penalty: 0,
   top_p: 1,
   frequency_penalty: 0,

diff --git a/src/hooks/useSubmit.ts b/src/hooks/useSubmit.ts
@@ -89,7 +89,6 @@ const useSubmit = () => {
 
       const messages = limitMessageTokens(
         chats[currentChatIndex].messages,
-        modelDef.model_max_context,
         modelDef.model,
         modelDef.model_max_tokens,
         config.max_tokens

diff --git a/src/utils/messageUtils.ts b/src/utils/messageUtils.ts
@@ -63,34 +63,21 @@ const countTokens = (messages: MessageInterface[], model: string) => {
 
 export const limitMessageTokens = (
   messages: MessageInterface[],
-  context_limit: number = 4096,
   model: string,
-  max_model_token: number = 4096,
-  token_limit: number = 4096
+  max_context: number = 4096,
+  max_tokens: number = 2048
 ): MessageInterface[] => {
   const limitedMessages: MessageInterface[] = [];
+  max_context -= max_tokens;
   let tokenCount = 0;
 
-  if (max_model_token < context_limit) {
-    context_limit = max_model_token;
-  }
-
-  let wholeTokenCount = 0;
-  for (let i = 0; i < messages.length; i++) {
-    wholeTokenCount += countTokens([messages[i]], model);
-  }
-
-  if (token_limit < context_limit + wholeTokenCount) {
-    context_limit = max_model_token - token_limit;
-  }
-
   const isSystemFirstMessage = messages[0]?.role === 'system';
   let retainSystemMessage = false;
 
   // Check if the first message is a system message and if it fits within the token limit
   if (isSystemFirstMessage) {
     const systemTokenCount = countTokens([messages[0]], model);
-    if (systemTokenCount < context_limit) {
+    if (systemTokenCount < max_context) {
       tokenCount += systemTokenCount;
       retainSystemMessage = true;
     }
@@ -100,7 +87,7 @@ export const limitMessageTokens = (
   // until the token limit is reached (excludes first message)
   for (let i = messages.length - 1; i >= 1; i--) {
     const count = countTokens([messages[i]], model);
-    if (count + tokenCount > context_limit) break;
+    if (count + tokenCount > max_context) break;
     tokenCount += count;
     limitedMessages.unshift({ ...messages[i] });
   }
@@ -112,7 +99,7 @@ export const limitMessageTokens = (
   } else if (!isSystemFirstMessage) {
     // Check if the first message (non-system) can fit within the limit
     const firstMessageTokenCount = countTokens([messages[0]], model);
-    if (firstMessageTokenCount + tokenCount < context_limit) {
+    if (firstMessageTokenCount + tokenCount < max_context) {
       limitedMessages.unshift({ ...messages[0] });
     }
   }