internlm p2 r2

llama-factory/config/internlm2_5_7b_lora_sft_bf16_p2_full_r2.yaml

@@ -0,0 +1,46 @@
+### model
+model_name_or_path: internlm/internlm2_5-7b-chat-1m
+
+### method
+stage: sft
+do_train: true
+finetuning_type: lora
+lora_target: all
+# quantization_bit: 4                     # use 4-bit QLoRA
+loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
+# use_unsloth: true                       # use UnslothAI's LoRA optimization for 2x faster training
+upcast_layernorm: true
+
+### dataset
+dataset: alpaca_mgtv_p2
+template: intern2
+cutoff_len: 8192
+max_samples: 25000
+overwrite_cache: true
+preprocessing_num_workers: 16
+
+### output
+output_dir: saves/internlm2_5_7b/lora/sft_bf16_p2_full_r2
+logging_steps: 10
+save_steps: 175
+plot_loss: true
+# overwrite_output_dir: true
+
+### train
+per_device_train_batch_size: 16
+gradient_accumulation_steps: 8
+learning_rate: 1.0e-4
+num_train_epochs: 4.0
+lr_scheduler_type: cosine
+warmup_ratio: 0.1
+bf16: true
+ddp_timeout: 180000000
+
+### eval
+val_size: 0.1
+per_device_eval_batch_size: 1
+eval_strategy: epoch
+eval_steps: 1
+
+report_to: wandb
+run_name: internlm2_5_7b_p2_l40_r2 # optional

scripts/eval-mgtv-best.sh

@@ -14,9 +14,9 @@ lscpu
 grep MemTotal /proc/meminfo
 
 #pip install -r requirements.txt
-#cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes]
+#cd ../LLaMA-Factory && pip install -e .[torch,bitsandbytes] && cd $BASEDIR
+#pip install transformers==4.41.2
 
-pip install transformers==4.41.2
 export USING_LLAMA_FACTORY=true
 
 export MODEL_NAME=internlm/internlm2_5-7b-chat-1m

scripts/tune-mgtv-internlm.sh

@@ -23,7 +23,10 @@ export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p1_full.yaml
 #echo "Tuning with $CONFIG_FILE"
 #$BASEDIR/scripts/tune-lf.sh $CONFIG_FILE
 
-export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full.csv
-export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full.yaml
+#export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full.csv
+#export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full.yaml
+export LOGICAL_REASONING_RESULTS_PATH=results/mgtv-results_p2_full_r2.csv
+export CONFIG_FILE=config/internlm2_5_7b_lora_sft_bf16_p2_full_r2.yaml
+
 echo "Tuning with $CONFIG_FILE"
 $BASEDIR/scripts/tune-lf.sh $CONFIG_FILE

scripts/tune-mgtv.sh

@@ -1 +1 @@
-tune-mgtv-glm-4-9b.sh
+tune-mgtv-internlm.sh