Update app.py

app.py

@@ -16,6 +16,8 @@ ROLE_TOKENS = {
     "system": SYSTEM_TOKEN
 }
 
+CONTEXT_SIZE = 4096
+
 # Create a lock object
 lock = threading.Lock()
 
@@ -38,16 +40,16 @@ model_name = "ggml-model-q4_1.gguf"
 snapshot_download(repo_id=repo_name, local_dir=".", allow_patterns=model_name)
 
 model = Llama(
-        model_path=model_name,
-        n_ctx=4096,
-        n_parts=1,
-        #n_batch=100,
-        logits_all=True,
-        #n_threads=12,
-        verbose=True,
-        n_gpu_layers=35,
-        n_gqa=8       #must be set for 70b models
-    )
+    model_path=model_name,
+    n_ctx=CONTEXT_SIZE,
+    n_parts=1,
+    #n_batch=100,
+    logits_all=True,
+    #n_threads=12,
+    verbose=True,
+    n_gpu_layers=35,
+    n_gqa=8       #must be set for 70b models
+)
 
 
 def get_message_tokens(model, role, content):
@@ -189,14 +191,13 @@ def generate_response():
     
     tokens = []
     
-    for message in messages:#[:-1]:
+    for message in messages:
         if message.get("from") == "assistant":
             message_tokens = get_message_tokens(model=model, role="bot", content=message.get("content", ""))
         else:
             message_tokens = get_message_tokens(model=model, role="user", content=message.get("content", ""))
     
         tokens.extend(message_tokens)
-        #LINEBREAK_TOKEN)
         
     #app.logger.info('model.eval start')
     #model.eval(tokens)
@@ -210,12 +211,13 @@ def generate_response():
             
     tokens.extend([model.token_bos(), BOT_TOKEN, LINEBREAK_TOKEN])
 
+    
     app.logger.info('Prompt:')
-    app.logger.info(model.detokenize(tokens).decode("utf-8", errors="ignore"))
+    app.logger.info(model.detokenize(tokens[:CONTEXT_SIZE]).decode("utf-8", errors="ignore"))
 
     app.logger.info('Generate started')
     generator = model.generate(
-        tokens,
+        tokens[:CONTEXT_SIZE],
         top_k=top_k,
         top_p=top_p,
         temp=temperature,