try float32 for inference

llm_toolkit/eval_logical_reasoning.py

@@ -29,7 +29,13 @@ max_new_tokens = int(os.getenv("MAX_NEW_TOKENS", 16))
 repetition_penalty = float(os.getenv("REPETITION_PENALTY", 1.0))
 
 dtype = (
-    torch.bfloat16 if os.getenv("USE_BF16_FOR_INFERENCE") == "true" else torch.float16
+    torch.float32
+    if os.getenv("USE_FLOAT32_FOR_INFERENCE") == "true"
+    else (
+        torch.bfloat16
+        if os.getenv("USE_BF16_FOR_INFERENCE") == "true"
+        else torch.float16
+    )
 )
 
 print(model_name, adapter_name_or_path, load_in_4bit, data_path, results_path)

scripts/eval-mgtv-best.sh

@@ -13,17 +13,17 @@ cat /etc/os-release
 lscpu
 grep MemTotal /proc/meminfo
 
-#pip install -r requirements.txt
+pip install -r requirements.txt
 #cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes]
 
-pip install transformers==4.41.2
-
-export USING_LLAMA_FACTORY=true
+# pip install transformers==4.41.2
+# export USING_LLAMA_FACTORY=true
 
 export MODEL_NAME=internlm/internlm2_5-7b-chat-1m
 export ADAPTER_NAME_OR_PATH=inflaton-ai/InternLM_2_5-7b_LoRA-Adapter
 export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
 export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_internlm_best.csv
+export USE_FLOAT32_FOR_INFERENCE=true
 
 echo "Eval $MODEL_NAME with $ADAPTER_NAME_OR_PATH"
 python llm_toolkit/eval_logical_reasoning.py