Update app.py

app.py

@@ -45,9 +45,9 @@ model = Llama(
     n_parts=1,
     #n_batch=100,
     logits_all=True,
-    #n_threads=12,
+    n_threads=12,
     verbose=True,
-    #n_gpu_layers=35,
+    n_gpu_layers=35,
     n_gqa=8       #must be set for 70b models
 )
 
@@ -129,12 +129,12 @@ def generate_search_request():
     parameters = data.get("parameters", {})
     
     # Extract parameters from the request
-    temperature = 0.01
+    temperature = parameters.get("temperature", 0.01)
     truncate = parameters.get("truncate", 1000)
     max_new_tokens = parameters.get("max_new_tokens", 1024)
-    top_p = 0.8
+    top_p = parameters.get("top_p", 0.85)
     repetition_penalty = parameters.get("repetition_penalty", 1.2)
-    top_k = 20
+    top_k = parameters.get("top_k", 30)
     return_full_text = parameters.get("return_full_text", False)
 
     
@@ -168,12 +168,12 @@ def generate_response():
     parameters = data.get("parameters", {})
     
     # Extract parameters from the request
-    temperature = 0.02#parameters.get("temperature", 0.01)
+    temperature = parameters.get("temperature", 0.01)
     truncate = parameters.get("truncate", 1000)
     max_new_tokens = parameters.get("max_new_tokens", 1024)
-    top_p = 80#parameters.get("top_p", 0.85)
+    top_p = parameters.get("top_p", 0.85)
     repetition_penalty = parameters.get("repetition_penalty", 1.2)
-    top_k = 25#parameters.get("top_k", 30)
+    top_k = parameters.get("top_k", 30)
     return_full_text = parameters.get("return_full_text", False)