Update app.py

app.py

@@ -1,7 +1,7 @@
 import os
 import subprocess
 
-subprocess.run('pip install flash-attn==2.7.0.post2 --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
+#subprocess.run('pip install flash-attn==2.7.0.post2 --no-build-isolation', env={'FLASH_ATTENTION_SKIP_CUDA_BUILD': "TRUE"}, shell=True)
 
 import threading
 
@@ -11,6 +11,9 @@ import spaces
 import gradio as gr
 import torch
 from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+from kernels import get_kernel
+
+vllm_flash_attn3 = get_kernel("kernels-community/vllm-flash-attn3")
 
 #torch._dynamo.config.disable = True
 
@@ -25,7 +28,7 @@ def load_model():
         MODEL_ID,
         torch_dtype=torch.bfloat16,  # if device == "cuda" else torch.float32,
         device_map="auto",  # if device == "cuda" else None,
-        attn_implementation="flash_attention_2",
+        attn_implementation=vllm_flash_attn3,
     )  # .cuda()
     print(f"Selected device:", device)
     return model, tokenizer, device