fix

app.py

@@ -68,10 +68,12 @@ def load_model():
     tokenizer.pad_token = tokenizer.eos_token
     tokenizer.padding_side = "right"
     
-    base_model_instance = AutoModel.from_pretrained(BASE_MODEL, device_map="auto", torch_dtype=torch.float16)
-    model = PeftModel.from_pretrained(base_model_instance, CUR_MODEL)
+    base_model_instance = AutoModel.from_pretrained("orionweller/repllama-instruct-hard-positives-v2-joint-full-weights", max_memory={"cpu": "12GiB"}, torch_dtype=torch.float16, offload_state_dict=True)
+    # base_model_instance = AutoModel.from_pretrained(BASE_MODEL)
+    # model = PeftModel.from_pretrained(base_model_instance, CUR_MODEL)
     model.eval()
 
+
 def load_faiss_index(dataset_name):
     index_path = f"{dataset_name}/faiss_index.bin"
     if os.path.exists(index_path):