Training in progress, epoch 1

README.md

@@ -32,13 +32,13 @@ More information needed
 ### Training hyperparameters
 
 The following hyperparameters were used during training:
-- learning_rate: 0.0005
-- train_batch_size: 8
-- eval_batch_size: 8
+- learning_rate: 0.005
+- train_batch_size: 20
+- eval_batch_size: 20
 - seed: 42
 - optimizer: Adam with betas=(0.9,0.95) and epsilon=1e-08
-- lr_scheduler_type: cosine
-- lr_scheduler_warmup_steps: 200
+- lr_scheduler_type: cosine_with_restarts
+- lr_scheduler_warmup_steps: 100
 - num_epochs: 1
 - mixed_precision_training: Native AMP
 

adapter_config.json

@@ -8,7 +8,7 @@
   "init_lora_weights": true,
   "layers_pattern": null,
   "layers_to_transform": null,
-  "lora_alpha": 16,
+  "lora_alpha": 8,
   "lora_dropout": 0.05,
   "modules_to_save": null,
   "peft_type": "LORA",
@@ -16,13 +16,13 @@
   "rank_pattern": {},
   "revision": null,
   "target_modules": [
-    "up_proj",
-    "o_proj",
-    "gate_proj",
     "k_proj",
-    "v_proj",
     "down_proj",
-    "q_proj"
+    "o_proj",
+    "v_proj",
+    "up_proj",
+    "q_proj",
+    "gate_proj"
   ],
   "task_type": "CAUSAL_LM"
 }

adapter_model.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9cd983ec4e34e31c66028a370aad768448bf4bf0f3f43f1b29b2cc62e32e9f82
+oid sha256:9bc7d5128353899de93f4052eef80c031c7565f07e8b64429aa3fdb2a0532d4e
 size 80013120

training_args.bin

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:add4c1b8058597a3a6d77d887369a731b4259a245b81f1b1da63f72d6edbb6e5
+oid sha256:e3c2dd014e2962d5de3c1662ec11941d18b77b61395cb797345ac65f2b7a68cf
 size 4600