Updated app

app.py

@@ -1,10 +1,21 @@
 import gradio as gr
-from mlx_lm import load, generate
+from llama_cpp import Llama
+from huggingface_hub import hf_hub_download
 
-model, tokenizer = load("astegaras/Mistral-7B-FineTuned-MLX")
+# Download GGUF to local file
+model_path = hf_hub_download(
+    repo_id="astegaras/Llama3.2_3B",
+    filename="model-Q4_K_M.gguf"
+)
 
-def respond(prompt):
-    return generate(model, tokenizer, prompt)
+llm = Llama(
+    model_path=model_path,
+    n_ctx=2048,
+    n_gpu_layers=0,
+)
 
-gr.Interface(fn=respond, inputs="text", outputs="text").launch()
+def respond(prompt):
+    out = llm(prompt, max_tokens=256)
+    return out["choices"][0]["text"]
 
+gr.Interface(fn=respond, inputs="text", outputs="text").launch()