fix: app.py with qwen backup

app.py

@@ -1,16 +1,47 @@
 import json
+import os
 
 import gradio as gr
-from distilabel.llms import InferenceEndpointsLLM
+from distilabel.llms import InferenceEndpointsLLM, LlamaCppLLM
 from distilabel.steps.tasks.argillalabeller import ArgillaLabeller
 
-llm = InferenceEndpointsLLM(
+file_path = os.path.join(os.path.dirname(__file__), "Qwen2-5-0.5B-Instruct-f16.gguf")
+download_url = "https://huggingface.co/gaianet/Qwen2.5-0.5B-Instruct-GGUF/resolve/main/Qwen2.5-0.5B-Instruct-Q5_K_S.gguf?download=true"
+
+
+if not os.path.exists(file_path):
+    import requests
+    import tqdm
+
+    response = requests.get(download_url, stream=True)
+    total_length = int(response.headers.get("content-length"))
+
+    with open(file_path, "wb") as f:
+        for chunk in tqdm.tqdm(
+            response.iter_content(chunk_size=1024 * 1024),
+            total=total_length / (1024 * 1024),
+            unit="KB",
+            unit_scale=True,
+        ):
+            f.write(chunk)
+
+
+llm_cpp = LlamaCppLLM(
+    model_path=file_path,
+    n_gpu_layers=-1,
+    n_ctx=1000 * 114,
+    generation_kwargs={"max_new_tokens": 1000 * 14},
+)
+task_cpp = ArgillaLabeller(llm=llm_cpp)
+task_cpp.load()
+
+llm_ep = InferenceEndpointsLLM(
     model_id="meta-llama/Meta-Llama-3.1-8B-Instruct",
     tokenizer_id="meta-llama/Meta-Llama-3.1-8B-Instruct",
     generation_kwargs={"max_new_tokens": 1000},
 )
-task = ArgillaLabeller(llm=llm)
-task.load()
+task_ep = ArgillaLabeller(llm=llm_ep)
+task_ep.load()
 
 
 def load_examples():
@@ -49,11 +80,17 @@ def process_records_gradio(records, fields, question, example_records=None):
         if example_records:
             runtime_parameters["example_records"] = example_records
 
-        task.set_runtime_parameters(runtime_parameters)
-
+        task_ep.set_runtime_parameters(runtime_parameters)
+        task_cpp.set_runtime_parameters(runtime_parameters)
         results = []
-        output = task.process(inputs=[{"record": record} for record in records])
-        output = next(output)
+        try:
+            output = next(
+                task_ep.process(inputs=[{"record": record} for record in records])
+            )
+        except Exception:
+            output = next(
+                task_cpp.process(inputs=[{"record": record} for record in records])
+            )
         for idx in range(len(records)):
             entry = output[idx]
             if entry["suggestions"]:
@@ -115,7 +152,7 @@ interface = gr.Interface(
         gr.Code(label="Question (JSON, optional)", language="json"),
     ],
     examples=examples,
-    cache_examples=True,
+    cache_examples=False,
     outputs=gr.Code(label="Suggestions", language="json", lines=10),
     title="Distilabel - ArgillaLabeller - Record Processing Interface",
     description=description,