Update app.py

app.py

@@ -20,6 +20,7 @@ ROLE_TOKENS = {
 
 CONTEXT_SIZE = 4000
 ENABLE_GPU = True
+GPU_LAYERS = 83
 
 # Create a lock object
 lock = threading.Lock()
@@ -50,7 +51,7 @@ model = None
 model_path = snapshot_download(repo_id=repo_name, allow_patterns=model_name) + '/' + model_name
 app.logger.info('Model path: ' + model_path)
 
-def init_model(context_size, enable_gpu=False, gpu_layer_number=83):
+def init_model(context_size, enable_gpu=False, gpu_layer_number=35):
     global model
     
     if model is not None:
@@ -83,7 +84,7 @@ def init_model(context_size, enable_gpu=False, gpu_layer_number=83):
         )
         return model
 
-init_model(CONTEXT_SIZE, ENABLE_GPU, 35)
+init_model(CONTEXT_SIZE, ENABLE_GPU, GPU_LAYERS)
 
 def get_message_tokens(model, role, content):
     message_tokens = model.tokenize(content.encode("utf-8"))
@@ -138,7 +139,7 @@ def handler_change_context_size():
     stop_generation = True
 
     new_size = int(request.args.get('size', CONTEXT_SIZE))
-    init_model(new_size, enable_gpu=ENABLE_GPU)
+    init_model(new_size, enable_gpu=ENABLE_GPU, GPU_LAYERS)
     
     return Response('Size changed', content_type='text/plain')