ready for internlm r3

llama-factory/config/internlm2_5_7b_lora_sft_bf16_p2_full_r3.yaml

@@ -0,0 +1,46 @@
+### model
+model_name_or_path: internlm/internlm2_5-7b-chat-1m
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_target: all
+# quantization_bit: 4                     # use 4-bit QLoRA
+loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
+# use_unsloth: true                       # use UnslothAI's LoRA optimization for 2x faster training
+upcast_layernorm: true
+
+### dataset
+dataset: alpaca_mgtv_p2
+template: intern2
+cutoff_len: 8192
+max_samples: 25000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/internlm2_5_7b/lora/sft_bf16_p2_full_r3
+logging_steps: 10
+save_steps: 35
+plot_loss: true
+# overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 16
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 2.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+val_size: 0.1
+per_device_eval_batch_size: 1
+eval_strategy: steps
+eval_steps: 35
+
+report_to: wandb
+run_name: internlm2_5_7b_p2_l40_r3 # optional

results/mgtv-llama3_p1_en_full_metrics.csv

@@ -1,5 +1,5 @@
 epoch,model,accuracy,precision,recall,f1
-0.0,meta-llama/Meta-Llama-3-8B-Instruct_torch.bfloat16_lf,0.13333333333333333,0.5430486329272943,0.13333333333333333,0.17807889451865855
+0.0,meta-llama/Meta-Llama-3-8B-Instruct_torch.bfloat16_lf,0.139,0.5741012854071863,0.13899999999999998,0.18649813199749535
 0.3333333333333333,meta-llama/Meta-Llama-3-8B-Instruct/checkpoint-117_torch.bfloat16_lf,0.6486666666666666,0.6525934632970077,0.6486666666666666,0.6312721163517108
 0.6666666666666666,meta-llama/Meta-Llama-3-8B-Instruct/checkpoint-234_torch.bfloat16_lf,0.561,0.6897096276142071,0.561,0.6083393704375663
 1.0,meta-llama/Meta-Llama-3-8B-Instruct/checkpoint-351_torch.bfloat16_lf,0.621,0.686842945161901,0.621,0.6417441253605001

results/mgtv-llama3_p1_r3_full_metrics.csv

@@ -0,0 +1,7 @@
+epoch,model,accuracy,precision,recall,f1
+0.0,hfl/llama-3-chinese-8b-instruct-v3_torch.bfloat16_lf,0.4563333333333333,0.6744501777722286,0.4563333333333333,0.5301217655617138
+0.2,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-35_torch.bfloat16_lf,0.6406666666666667,0.7652410757316818,0.6406666666666667,0.686506602559313
+0.4,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-70_torch.bfloat16_lf,0.7223333333333334,0.761495200766395,0.7223333333333334,0.7296692772947554
+0.6,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-105_torch.bfloat16_lf,0.6256666666666667,0.7694288569564225,0.6256666666666667,0.6747421032154606
+0.8,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-140_torch.bfloat16_lf,0.7173333333333334,0.7746925589996599,0.7173333333333334,0.7391047683054164
+1.0,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-175_torch.bfloat16_lf,0.688,0.7678475683985698,0.688,0.7181965107150645

results/mgtv-llama3_p2_en_full_metrics.csv

@@ -1,5 +1,5 @@
 epoch,model,accuracy,precision,recall,f1
-0.0,meta-llama/Meta-Llama-3-8B-Instruct_torch.bfloat16_lf,0.17533333333333334,0.5690657625063544,0.17533333333333334,0.21427445914454485
+0.0,meta-llama/Meta-Llama-3-8B-Instruct_torch.bfloat16_lf,0.15466666666666667,0.5218515852800517,0.15466666666666667,0.17611762474007195
 0.3333333333333333,meta-llama/Meta-Llama-3-8B-Instruct/checkpoint-117_torch.bfloat16_lf,0.6203333333333333,0.663582082981778,0.6203333333333333,0.6363626392286635
 0.6666666666666666,meta-llama/Meta-Llama-3-8B-Instruct/checkpoint-234_torch.bfloat16_lf,0.5613333333333334,0.7000506187405509,0.5613333333333334,0.6113039056178092
 1.0,meta-llama/Meta-Llama-3-8B-Instruct/checkpoint-351_torch.bfloat16_lf,0.6203333333333333,0.6819200833733873,0.6203333333333333,0.6405153767205392

results/mgtv-llama3_p2_r3_full_metrics.csv

@@ -0,0 +1,4 @@
+epoch,model,accuracy,precision,recall,f1
+0.0,hfl/llama-3-chinese-8b-instruct-v3_torch.bfloat16_lf,0.25066666666666665,0.6852419041932336,0.25066666666666665,0.32636449818329016
+0.2,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-35_torch.bfloat16_lf,0.7283333333333334,0.7722393813259697,0.7283333333333334,0.7426450360790026
+0.4,hfl/llama-3-chinese-8b-instruct-v3/checkpoint-70_torch.bfloat16_lf,0.741,0.7868300593752113,0.741,0.7514058688729928

scripts/eval-mgtv-internlm.sh

@@ -20,8 +20,8 @@ pip install transformers==4.41.2
 
 export MODEL_NAME=internlm/internlm2_5-7b-chat-1m
 export LOGICAL_REASONING_DATA_PATH=datasets/mgtv
-export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full_r2.csv
-export ADAPTER_PATH_BASE=llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full_r2
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full_r3.csv
+export ADAPTER_PATH_BASE=llama-factory/saves/internlm2_5_7b/lora/sft_bf16_p2_full_r3
 export USING_LLAMA_FACTORY=true
 export START_EPOCH=4
 

scripts/tune-mgtv-internlm.sh

@@ -25,8 +25,12 @@ export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p1_full.yaml
 
 #export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full.csv
 #export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full.yaml
-export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full_r2.csv
-export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full_r2.yaml
+# export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full_r2.csv
+# export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full_r2.yaml
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full_r3.csv
+export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full_r3.yaml
 
 echo "Tuning with $CONFIG_FILE"
 $BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
+
+$BASEDIR/scripts/eval-mgtv-internlm.sh

scripts/tune-mgtv.sh

@@ -1 +1 @@
-tune-mgtv-llama3_8b.sh
+tune-mgtv-internlm.sh