| { | |
| "version": "0.2.2", | |
| "common_args": { | |
| "model_name_or_path": "./Qwen2.5-3B-Instruct", | |
| "adapter_name_or_path": "./model_output", //同时做为train_sft_args的output_dir | |
| "template": "qwen", | |
| "default_system": "请你扮演一名人类，不要说自己是人工智能", | |
| "finetuning_type": "lora", | |
| "trust_remote_code": true | |
| }, | |
| "cli_args": { | |
| "full_log": false | |
| }, | |
| "make_dataset_args": { | |
| //数据处理配置 | |
| "include_type": [ | |
| "文本" | |
| ], | |
| "blocked_words": [ // 禁用词 | |
| "例如 姓名", | |
| "例如 密码", | |
| "//....." | |
| ], | |
| "single_combine_strategy": "time_window", // 单人组成单句策略 | |
| "qa_match_strategy": "time_window", // 组成qa策略 | |
| "single_combine_time_window": 2, // 单人组成单句时间窗口（分钟）, | |
| "qa_match_time_window": 5, // 组成qa时间窗口（分钟）, | |
| "combine_msg_max_length": 256, // 组合后消息最大长度 配合cutoff_len 使用 | |
| "prompt_with_history": false, // 是否在prompt中包含历史对话 | |
| "clean_dataset": { | |
| "enable_clean": true, | |
| "clean_strategy": "llm", | |
| "llm": { | |
| "accept_score": 2, //可以接受的llm打分阈值,1分最差，5分最好,低于此分数的数据不会用于训练 | |
| } | |
| } | |
| }, | |
| "train_pt_args": { | |
| //预训练微调配置 | |
| "stage": "pt", | |
| "dataset": "wechat-pt", | |
| "dataset_dir": "./dataset/res_csv/pt", | |
| "lora_target": "q_proj,v_proj", | |
| "lora_rank": 2, | |
| "lora_dropout": 0.1, | |
| "output_dir": "model_output", | |
| "overwrite_cache": true, | |
| "per_device_train_batch_size": 1, | |
| "gradient_accumulation_steps": 1, | |
| "lr_scheduler_type": "cosine", | |
| "logging_steps": 10, | |
| "save_steps": 1000, | |
| "learning_rate": 0.001, | |
| "num_train_epochs": 30, | |
| "plot_loss": true, | |
| "fp16": true | |
| }, | |
| "train_sft_args": { | |
| //微调配置 | |
| "stage": "sft", | |
| "dataset": "wechat-sft", | |
| "dataset_dir": "./dataset/res_csv/sft", | |
| "use_fast_tokenizer": true, | |
| "lora_target": "q_proj,v_proj", | |
| "lora_rank": 4, | |
| "lora_dropout": 0.3, | |
| "weight_decay": 0.1, | |
| "overwrite_cache": true, | |
| "per_device_train_batch_size": 8, | |
| "gradient_accumulation_steps": 4, | |
| "lr_scheduler_type": "cosine", | |
| "cutoff_len": 256, | |
| "logging_steps": 5, | |
| "save_steps": 10, | |
| "learning_rate": 1e-4, | |
| "warmup_ratio": 0.1, | |
| "num_train_epochs": 1, | |
| "plot_loss": true, | |
| "fp16": true, | |
| "flash_attn": "fa2", | |
| // "deepspeed": "ds_config.json" //多卡训练 | |
| }, | |
| "infer_args": { | |
| "repetition_penalty": 1.2, | |
| "temperature": 0.5, | |
| "max_length": 50, | |
| "top_p": 0.65 | |
| } | |
| } |