Add temp, top_p, top_k to envs and main

Author: Belluxx

example-gemma2-2B.env

@@ -12,6 +12,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=8000
 N_GENERATE=2000
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-gemma2-9B-27B.env

@@ -12,6 +12,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=8000
 N_GENERATE=2000
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-llama3instruct.env

@@ -13,6 +13,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=8000
 N_GENERATE=2000
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-openchat.env

@@ -12,6 +12,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=4096
 N_GENERATE=1024
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-phi3.env

@@ -12,6 +12,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=4096
 N_GENERATE=1024
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-qwen2.env

@@ -14,6 +14,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=4096
 N_GENERATE=1024
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-readerlm.env

@@ -12,6 +12,9 @@ AGENT_ASSISTANT="assistant"
 REAL_TIME=1
 N_CTX=64000
 N_GENERATE=16000
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

example-zephyr.env

@@ -12,6 +12,9 @@ ASSISTANT_INITIAL_MESSAGE="Hello, do you have any question?"
 REAL_TIME=1
 N_CTX=4096
 N_GENERATE=1024
+TEMPERATURE=0.8
+TOP_P=0.9
+TOP_K=40
 SEED=69
 USE_MMAP=1
 USE_MLOCK=1

main.py

@@ -48,6 +48,9 @@ def get_env_and_check(key: str, required: bool = True) -> str:
 REAL_TIME =                 bool(int(get_env_and_check('REAL_TIME')))
 N_CTX =                     int(get_env_and_check('N_CTX'))
 N_GENERATE =                int(get_env_and_check('N_GENERATE'))
+TEMPERATURE =               float(get_env_and_check('TEMPERATURE'))
+TOP_P =                     float(get_env_and_check('TOP_P'))
+TOP_K =                     int(get_env_and_check('TOP_K'))
 SEED =                      int(get_env_and_check('SEED'))
 USE_MMAP =                  bool(int(get_env_and_check('USE_MMAP')))
 USE_MLOCK =                 bool(int(get_env_and_check('USE_MLOCK')))
@@ -159,6 +162,9 @@ def file_to_markdown(file_path: str) -> str:
         bot=BOT,
         eos=EOS,
         n_generate=N_GENERATE,
+        temperature=TEMPERATURE,
+        top_p=TOP_P,
+        top_k=TOP_K,
         debug=DEBUG
     )