Initial commit

README.md

@@ -0,0 +1,60 @@
+---
+license: other
+base_model: meta-llama/Llama-2-7b-chat-hf
+tags:
+- llama-factory
+- full
+- generated_from_trainer
+model-index:
+- name: train_2024-07-11-09-30-54_llama2_inst_truth
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# train_2024-07-11-09-30-54_llama2_inst_truth
+
+This model is a fine-tuned version of [meta-llama/Llama-2-7b-chat-hf](https://huggingface.co/meta-llama/Llama-2-7b-chat-hf) on the truth_train dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 5e-06
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- distributed_type: multi-GPU
+- num_devices: 8
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 256
+- total_eval_batch_size: 64
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_steps: 600
+- num_epochs: 5.0
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.42.3
+- Pytorch 2.3.0a0+ebedce2
+- Datasets 2.20.0
+- Tokenizers 0.19.1

all_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.951768488745981,
+    "num_input_tokens_seen": 6318560,
+    "total_flos": 2.5049338265940787e+17,
+    "train_loss": 0.9364173337708336,
+    "train_runtime": 4331.1035,
+    "train_samples_per_second": 22.95,
+    "train_steps_per_second": 0.089
+}

checkpoint-385/config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

checkpoint-385/generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.42.3"
+}

checkpoint-385/global_step385/bf16_zero_pp_rank_0_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a0a56a0147ee2f230de21a90617d17f29cfa3854d3ece1aa17c01452168b6ae2
+size 10107631180

checkpoint-385/global_step385/bf16_zero_pp_rank_1_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d1baec97d069910f03c2cd8f7ed0e8288e2f344b877c35c2341011602deea07e
+size 10107631372

checkpoint-385/global_step385/bf16_zero_pp_rank_2_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c027bd10605572bcb762c678165b894437661a8fc1d46b9b677c8d69acbfcd75
+size 10107631436

checkpoint-385/global_step385/bf16_zero_pp_rank_3_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1922032b61c90ca8f523a14a692f29ad291a5adb367d483e2d2f0327528a6e5a
+size 10107631372

checkpoint-385/global_step385/bf16_zero_pp_rank_4_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6cef6e0255720f677c3f9faacaac7222cbeefc2b370abfda81b88ed7c090d4c3
+size 10107631436

checkpoint-385/global_step385/bf16_zero_pp_rank_5_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:19a03f553f83951055cfebaa5c83e0990bc0f762716598f5f1af07372ed4886b
+size 10107631500

checkpoint-385/global_step385/bf16_zero_pp_rank_6_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:169c99232f4320585a6a3059870e92f8026ea021b27ce332df2f5ac60541a5cd
+size 10107631436

checkpoint-385/global_step385/bf16_zero_pp_rank_7_mp_rank_00_optim_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:61434bbc88c1bd1091aece024937a9aa1b9d735a4fc36afc8ed86194f3902e27
+size 10107630988

checkpoint-385/global_step385/mp_rank_00_model_states.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0cab62a9512819347dc18403d19d8c7d2b2ffc247243a19a0877a3c067692559
+size 13476919288

checkpoint-385/latest

@@ -0,0 +1 @@
+global_step385

checkpoint-385/model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05797c94c439de88c420a567e2bde6c29dad762705ba7588731295691e8b1173
+size 4938985352

checkpoint-385/model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eb6354f514ef3bd5ba958aca5d01b62d4199cd3baf18bd14d43954c90217bd5
+size 4947390880

checkpoint-385/model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03f1693b214e4e07203a5b47bb63f40ed75486d4b48e985781c1da8733a4c39
+size 3590488816

checkpoint-385/model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 13476831232
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

checkpoint-385/rng_state_0.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a0ef6f96a48e59aa52c4b471312c2a62378c19acc7ebbae839612b03a7d775a
+size 15984

checkpoint-385/rng_state_1.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab11d533c0fdad46ea8b8e295ba5fdb705e078eeb88cc28f37d82913508766e9
+size 15984

checkpoint-385/rng_state_2.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:615c168147e3465ce5bfab6da2ff4afc68566ce00ec0f0c6c9fc988038a58d0a
+size 15984

checkpoint-385/rng_state_3.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:79f71e8f8674ecaef9f8cdcbf7ac457a8b8ff15b12694ba2a2fffcb4b43f0f08
+size 15984

checkpoint-385/rng_state_4.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:88cf6d674dab5545c300a55135f08ca935730a3d35e2c419fb0b333f19482c19
+size 15984

checkpoint-385/rng_state_5.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2754f2cd8824702f027870d93748b3c0491b0ecd30f1e3d8e937116b2be6151f
+size 15984

checkpoint-385/rng_state_6.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1385124ac55604598f45ea6e2d141f29456647d3e7c10d12ca64ec93d312be8d
+size 15984

checkpoint-385/rng_state_7.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:416538efaec7391fa8fe782fb15146b83e5612d9e1961292c34c53e964806873
+size 15984

checkpoint-385/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5773c00222bef1db5ca810359f2616a1fbd633a740c194d6a3070d6b031a0320
+size 1064

checkpoint-385/special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

checkpoint-385/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

checkpoint-385/tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if loop.index0 == 0 and system_message is defined %}{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '<s>' + '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

checkpoint-385/trainer_state.json

@@ -0,0 +1,3113 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.951768488745981,
+  "eval_steps": 500,
+  "global_step": 385,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012861736334405145,
+      "grad_norm": 258.1509094238281,
+      "learning_rate": 8.333333333333335e-09,
+      "loss": 8.1083,
+      "num_input_tokens_seen": 16544,
+      "step": 1
+    },
+    {
+      "epoch": 0.02572347266881029,
+      "grad_norm": 266.15087890625,
+      "learning_rate": 1.666666666666667e-08,
+      "loss": 8.1052,
+      "num_input_tokens_seen": 33248,
+      "step": 2
+    },
+    {
+      "epoch": 0.03858520900321544,
+      "grad_norm": 276.69952392578125,
+      "learning_rate": 2.5000000000000002e-08,
+      "loss": 8.1366,
+      "num_input_tokens_seen": 49760,
+      "step": 3
+    },
+    {
+      "epoch": 0.05144694533762058,
+      "grad_norm": 268.43804931640625,
+      "learning_rate": 3.333333333333334e-08,
+      "loss": 8.0413,
+      "num_input_tokens_seen": 65824,
+      "step": 4
+    },
+    {
+      "epoch": 0.06430868167202572,
+      "grad_norm": 269.6364440917969,
+      "learning_rate": 4.166666666666667e-08,
+      "loss": 8.1055,
+      "num_input_tokens_seen": 82560,
+      "step": 5
+    },
+    {
+      "epoch": 0.07717041800643087,
+      "grad_norm": 274.33538818359375,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 7.9666,
+      "num_input_tokens_seen": 99808,
+      "step": 6
+    },
+    {
+      "epoch": 0.09003215434083602,
+      "grad_norm": 266.3240051269531,
+      "learning_rate": 5.833333333333334e-08,
+      "loss": 8.0182,
+      "num_input_tokens_seen": 115808,
+      "step": 7
+    },
+    {
+      "epoch": 0.10289389067524116,
+      "grad_norm": 274.253662109375,
+      "learning_rate": 6.666666666666668e-08,
+      "loss": 8.0363,
+      "num_input_tokens_seen": 131392,
+      "step": 8
+    },
+    {
+      "epoch": 0.1157556270096463,
+      "grad_norm": 263.321533203125,
+      "learning_rate": 7.500000000000001e-08,
+      "loss": 8.0918,
+      "num_input_tokens_seen": 147840,
+      "step": 9
+    },
+    {
+      "epoch": 0.12861736334405144,
+      "grad_norm": 266.48394775390625,
+      "learning_rate": 8.333333333333334e-08,
+      "loss": 8.0197,
+      "num_input_tokens_seen": 164064,
+      "step": 10
+    },
+    {
+      "epoch": 0.1414790996784566,
+      "grad_norm": 258.7409362792969,
+      "learning_rate": 9.166666666666668e-08,
+      "loss": 7.9714,
+      "num_input_tokens_seen": 180704,
+      "step": 11
+    },
+    {
+      "epoch": 0.15434083601286175,
+      "grad_norm": 271.2587585449219,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 8.0131,
+      "num_input_tokens_seen": 196064,
+      "step": 12
+    },
+    {
+      "epoch": 0.16720257234726688,
+      "grad_norm": 255.18431091308594,
+      "learning_rate": 1.0833333333333335e-07,
+      "loss": 7.9796,
+      "num_input_tokens_seen": 211424,
+      "step": 13
+    },
+    {
+      "epoch": 0.18006430868167203,
+      "grad_norm": 269.7242736816406,
+      "learning_rate": 1.1666666666666668e-07,
+      "loss": 7.9999,
+      "num_input_tokens_seen": 228096,
+      "step": 14
+    },
+    {
+      "epoch": 0.19292604501607716,
+      "grad_norm": 264.5142517089844,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 8.0168,
+      "num_input_tokens_seen": 244416,
+      "step": 15
+    },
+    {
+      "epoch": 0.2057877813504823,
+      "grad_norm": 267.4490661621094,
+      "learning_rate": 1.3333333333333336e-07,
+      "loss": 7.8423,
+      "num_input_tokens_seen": 261056,
+      "step": 16
+    },
+    {
+      "epoch": 0.21864951768488747,
+      "grad_norm": 269.4408874511719,
+      "learning_rate": 1.4166666666666668e-07,
+      "loss": 7.9208,
+      "num_input_tokens_seen": 276992,
+      "step": 17
+    },
+    {
+      "epoch": 0.2315112540192926,
+      "grad_norm": 256.512939453125,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 7.6898,
+      "num_input_tokens_seen": 293536,
+      "step": 18
+    },
+    {
+      "epoch": 0.24437299035369775,
+      "grad_norm": 265.34490966796875,
+      "learning_rate": 1.5833333333333336e-07,
+      "loss": 7.9285,
+      "num_input_tokens_seen": 309024,
+      "step": 19
+    },
+    {
+      "epoch": 0.2572347266881029,
+      "grad_norm": 268.9532470703125,
+      "learning_rate": 1.6666666666666668e-07,
+      "loss": 7.8924,
+      "num_input_tokens_seen": 325024,
+      "step": 20
+    },
+    {
+      "epoch": 0.27009646302250806,
+      "grad_norm": 268.1192321777344,
+      "learning_rate": 1.7500000000000002e-07,
+      "loss": 7.6173,
+      "num_input_tokens_seen": 341856,
+      "step": 21
+    },
+    {
+      "epoch": 0.2829581993569132,
+      "grad_norm": 271.13006591796875,
+      "learning_rate": 1.8333333333333336e-07,
+      "loss": 7.7661,
+      "num_input_tokens_seen": 358848,
+      "step": 22
+    },
+    {
+      "epoch": 0.2958199356913183,
+      "grad_norm": 260.5618591308594,
+      "learning_rate": 1.9166666666666668e-07,
+      "loss": 7.2936,
+      "num_input_tokens_seen": 375360,
+      "step": 23
+    },
+    {
+      "epoch": 0.3086816720257235,
+      "grad_norm": 249.86729431152344,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 7.3044,
+      "num_input_tokens_seen": 392608,
+      "step": 24
+    },
+    {
+      "epoch": 0.3215434083601286,
+      "grad_norm": 257.8570861816406,
+      "learning_rate": 2.0833333333333333e-07,
+      "loss": 7.35,
+      "num_input_tokens_seen": 408800,
+      "step": 25
+    },
+    {
+      "epoch": 0.33440514469453375,
+      "grad_norm": 246.40611267089844,
+      "learning_rate": 2.166666666666667e-07,
+      "loss": 7.1213,
+      "num_input_tokens_seen": 425632,
+      "step": 26
+    },
+    {
+      "epoch": 0.34726688102893893,
+      "grad_norm": 252.2755889892578,
+      "learning_rate": 2.2500000000000002e-07,
+      "loss": 7.1363,
+      "num_input_tokens_seen": 441952,
+      "step": 27
+    },
+    {
+      "epoch": 0.36012861736334406,
+      "grad_norm": 235.94859313964844,
+      "learning_rate": 2.3333333333333336e-07,
+      "loss": 6.9016,
+      "num_input_tokens_seen": 458080,
+      "step": 28
+    },
+    {
+      "epoch": 0.3729903536977492,
+      "grad_norm": 243.25863647460938,
+      "learning_rate": 2.416666666666667e-07,
+      "loss": 6.9764,
+      "num_input_tokens_seen": 473824,
+      "step": 29
+    },
+    {
+      "epoch": 0.3858520900321543,
+      "grad_norm": 250.39564514160156,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 6.9005,
+      "num_input_tokens_seen": 490336,
+      "step": 30
+    },
+    {
+      "epoch": 0.3987138263665595,
+      "grad_norm": 253.6007843017578,
+      "learning_rate": 2.5833333333333333e-07,
+      "loss": 6.9852,
+      "num_input_tokens_seen": 506976,
+      "step": 31
+    },
+    {
+      "epoch": 0.4115755627009646,
+      "grad_norm": 155.81201171875,
+      "learning_rate": 2.666666666666667e-07,
+      "loss": 5.834,
+      "num_input_tokens_seen": 523520,
+      "step": 32
+    },
+    {
+      "epoch": 0.42443729903536975,
+      "grad_norm": 124.55789947509766,
+      "learning_rate": 2.75e-07,
+      "loss": 5.5452,
+      "num_input_tokens_seen": 540672,
+      "step": 33
+    },
+    {
+      "epoch": 0.43729903536977494,
+      "grad_norm": 130.3427276611328,
+      "learning_rate": 2.8333333333333336e-07,
+      "loss": 5.5189,
+      "num_input_tokens_seen": 557696,
+      "step": 34
+    },
+    {
+      "epoch": 0.45016077170418006,
+      "grad_norm": 126.62003326416016,
+      "learning_rate": 2.916666666666667e-07,
+      "loss": 5.4654,
+      "num_input_tokens_seen": 574400,
+      "step": 35
+    },
+    {
+      "epoch": 0.4630225080385852,
+      "grad_norm": 120.25413513183594,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.3117,
+      "num_input_tokens_seen": 590112,
+      "step": 36
+    },
+    {
+      "epoch": 0.4758842443729904,
+      "grad_norm": 123.52056884765625,
+      "learning_rate": 3.083333333333334e-07,
+      "loss": 5.2744,
+      "num_input_tokens_seen": 605920,
+      "step": 37
+    },
+    {
+      "epoch": 0.4887459807073955,
+      "grad_norm": 124.25894927978516,
+      "learning_rate": 3.166666666666667e-07,
+      "loss": 5.2821,
+      "num_input_tokens_seen": 622208,
+      "step": 38
+    },
+    {
+      "epoch": 0.5016077170418006,
+      "grad_norm": 111.38367462158203,
+      "learning_rate": 3.25e-07,
+      "loss": 5.0967,
+      "num_input_tokens_seen": 638464,
+      "step": 39
+    },
+    {
+      "epoch": 0.5144694533762058,
+      "grad_norm": 119.48200225830078,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 5.2167,
+      "num_input_tokens_seen": 654656,
+      "step": 40
+    },
+    {
+      "epoch": 0.5273311897106109,
+      "grad_norm": 108.62528991699219,
+      "learning_rate": 3.416666666666667e-07,
+      "loss": 5.1517,
+      "num_input_tokens_seen": 671424,
+      "step": 41
+    },
+    {
+      "epoch": 0.5401929260450161,
+      "grad_norm": 109.66475677490234,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 4.6959,
+      "num_input_tokens_seen": 687968,
+      "step": 42
+    },
+    {
+      "epoch": 0.5530546623794212,
+      "grad_norm": 107.45171356201172,
+      "learning_rate": 3.583333333333334e-07,
+      "loss": 4.1458,
+      "num_input_tokens_seen": 704224,
+      "step": 43
+    },
+    {
+      "epoch": 0.5659163987138264,
+      "grad_norm": 102.30545806884766,
+      "learning_rate": 3.666666666666667e-07,
+      "loss": 3.6692,
+      "num_input_tokens_seen": 720704,
+      "step": 44
+    },
+    {
+      "epoch": 0.5787781350482315,
+      "grad_norm": 96.8830795288086,
+      "learning_rate": 3.75e-07,
+      "loss": 3.3322,
+      "num_input_tokens_seen": 737408,
+      "step": 45
+    },
+    {
+      "epoch": 0.5916398713826366,
+      "grad_norm": 102.91023254394531,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 3.2857,
+      "num_input_tokens_seen": 753728,
+      "step": 46
+    },
+    {
+      "epoch": 0.6045016077170418,
+      "grad_norm": 105.37537384033203,
+      "learning_rate": 3.9166666666666675e-07,
+      "loss": 3.0806,
+      "num_input_tokens_seen": 770080,
+      "step": 47
+    },
+    {
+      "epoch": 0.617363344051447,
+      "grad_norm": 98.45983123779297,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 2.8239,
+      "num_input_tokens_seen": 786912,
+      "step": 48
+    },
+    {
+      "epoch": 0.6302250803858521,
+      "grad_norm": 108.93148040771484,
+      "learning_rate": 4.083333333333334e-07,
+      "loss": 2.8695,
+      "num_input_tokens_seen": 804192,
+      "step": 49
+    },
+    {
+      "epoch": 0.6430868167202572,
+      "grad_norm": 100.31180572509766,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 2.6218,
+      "num_input_tokens_seen": 820928,
+      "step": 50
+    },
+    {
+      "epoch": 0.6559485530546624,
+      "grad_norm": 101.31784057617188,
+      "learning_rate": 4.2500000000000006e-07,
+      "loss": 2.3993,
+      "num_input_tokens_seen": 837152,
+      "step": 51
+    },
+    {
+      "epoch": 0.6688102893890675,
+      "grad_norm": 101.09466552734375,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 2.2828,
+      "num_input_tokens_seen": 853568,
+      "step": 52
+    },
+    {
+      "epoch": 0.6816720257234726,
+      "grad_norm": 99.55025482177734,
+      "learning_rate": 4.416666666666667e-07,
+      "loss": 2.2063,
+      "num_input_tokens_seen": 870592,
+      "step": 53
+    },
+    {
+      "epoch": 0.6945337620578779,
+      "grad_norm": 104.6786117553711,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 1.8878,
+      "num_input_tokens_seen": 886080,
+      "step": 54
+    },
+    {
+      "epoch": 0.707395498392283,
+      "grad_norm": 106.05968475341797,
+      "learning_rate": 4.583333333333333e-07,
+      "loss": 1.5971,
+      "num_input_tokens_seen": 903296,
+      "step": 55
+    },
+    {
+      "epoch": 0.7202572347266881,
+      "grad_norm": 93.43476867675781,
+      "learning_rate": 4.666666666666667e-07,
+      "loss": 0.9604,
+      "num_input_tokens_seen": 919296,
+      "step": 56
+    },
+    {
+      "epoch": 0.7331189710610932,
+      "grad_norm": 60.13694763183594,
+      "learning_rate": 4.7500000000000006e-07,
+      "loss": 0.6287,
+      "num_input_tokens_seen": 936096,
+      "step": 57
+    },
+    {
+      "epoch": 0.7459807073954984,
+      "grad_norm": 42.870086669921875,
+      "learning_rate": 4.833333333333334e-07,
+      "loss": 0.4844,
+      "num_input_tokens_seen": 953024,
+      "step": 58
+    },
+    {
+      "epoch": 0.7588424437299035,
+      "grad_norm": 45.984989166259766,
+      "learning_rate": 4.916666666666667e-07,
+      "loss": 0.3891,
+      "num_input_tokens_seen": 969024,
+      "step": 59
+    },
+    {
+      "epoch": 0.7717041800643086,
+      "grad_norm": 44.375484466552734,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.357,
+      "num_input_tokens_seen": 985600,
+      "step": 60
+    },
+    {
+      "epoch": 0.7845659163987139,
+      "grad_norm": 28.2038516998291,
+      "learning_rate": 5.083333333333334e-07,
+      "loss": 0.2854,
+      "num_input_tokens_seen": 1002336,
+      "step": 61
+    },
+    {
+      "epoch": 0.797427652733119,
+      "grad_norm": 13.842344284057617,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.2403,
+      "num_input_tokens_seen": 1018496,
+      "step": 62
+    },
+    {
+      "epoch": 0.8102893890675241,
+      "grad_norm": 21.351642608642578,
+      "learning_rate": 5.250000000000001e-07,
+      "loss": 0.2522,
+      "num_input_tokens_seen": 1035552,
+      "step": 63
+    },
+    {
+      "epoch": 0.8231511254019293,
+      "grad_norm": 21.38069725036621,
+      "learning_rate": 5.333333333333335e-07,
+      "loss": 0.2339,
+      "num_input_tokens_seen": 1051584,
+      "step": 64
+    },
+    {
+      "epoch": 0.8360128617363344,
+      "grad_norm": 9.553906440734863,
+      "learning_rate": 5.416666666666667e-07,
+      "loss": 0.2214,
+      "num_input_tokens_seen": 1068640,
+      "step": 65
+    },
+    {
+      "epoch": 0.8488745980707395,
+      "grad_norm": 6.100700855255127,
+      "learning_rate": 5.5e-07,
+      "loss": 0.1912,
+      "num_input_tokens_seen": 1085568,
+      "step": 66
+    },
+    {
+      "epoch": 0.8617363344051447,
+      "grad_norm": 18.959564208984375,
+      "learning_rate": 5.583333333333333e-07,
+      "loss": 0.2157,
+      "num_input_tokens_seen": 1102784,
+      "step": 67
+    },
+    {
+      "epoch": 0.8745980707395499,
+      "grad_norm": 16.303438186645508,
+      "learning_rate": 5.666666666666667e-07,
+      "loss": 0.2045,
+      "num_input_tokens_seen": 1118432,
+      "step": 68
+    },
+    {
+      "epoch": 0.887459807073955,
+      "grad_norm": 12.491047859191895,
+      "learning_rate": 5.750000000000001e-07,
+      "loss": 0.1964,
+      "num_input_tokens_seen": 1135232,
+      "step": 69
+    },
+    {
+      "epoch": 0.9003215434083601,
+      "grad_norm": 16.655309677124023,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.1901,
+      "num_input_tokens_seen": 1150496,
+      "step": 70
+    },
+    {
+      "epoch": 0.9131832797427653,
+      "grad_norm": 12.015859603881836,
+      "learning_rate": 5.916666666666667e-07,
+      "loss": 0.1891,
+      "num_input_tokens_seen": 1166848,
+      "step": 71
+    },
+    {
+      "epoch": 0.9260450160771704,
+      "grad_norm": 12.255152702331543,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.1841,
+      "num_input_tokens_seen": 1183552,
+      "step": 72
+    },
+    {
+      "epoch": 0.9389067524115756,
+      "grad_norm": 5.163099765777588,
+      "learning_rate": 6.083333333333334e-07,
+      "loss": 0.1803,
+      "num_input_tokens_seen": 1199968,
+      "step": 73
+    },
+    {
+      "epoch": 0.9517684887459807,
+      "grad_norm": 10.502095222473145,
+      "learning_rate": 6.166666666666668e-07,
+      "loss": 0.1657,
+      "num_input_tokens_seen": 1216256,
+      "step": 74
+    },
+    {
+      "epoch": 0.9646302250803859,
+      "grad_norm": 23.80043601989746,
+      "learning_rate": 6.25e-07,
+      "loss": 0.2157,
+      "num_input_tokens_seen": 1232736,
+      "step": 75
+    },
+    {
+      "epoch": 0.977491961414791,
+      "grad_norm": 19.7410831451416,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 0.2,
+      "num_input_tokens_seen": 1249248,
+      "step": 76
+    },
+    {
+      "epoch": 0.9903536977491961,
+      "grad_norm": 4.730998516082764,
+      "learning_rate": 6.416666666666667e-07,
+      "loss": 0.1519,
+      "num_input_tokens_seen": 1265152,
+      "step": 77
+    },
+    {
+      "epoch": 1.0032154340836013,
+      "grad_norm": 14.700318336486816,
+      "learning_rate": 6.5e-07,
+      "loss": 0.1765,
+      "num_input_tokens_seen": 1282336,
+      "step": 78
+    },
+    {
+      "epoch": 1.0160771704180065,
+      "grad_norm": 18.6468448638916,
+      "learning_rate": 6.583333333333333e-07,
+      "loss": 0.1907,
+      "num_input_tokens_seen": 1299168,
+      "step": 79
+    },
+    {
+      "epoch": 1.0289389067524115,
+      "grad_norm": 11.44071102142334,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.1593,
+      "num_input_tokens_seen": 1314816,
+      "step": 80
+    },
+    {
+      "epoch": 1.0418006430868167,
+      "grad_norm": 3.6438238620758057,
+      "learning_rate": 6.750000000000001e-07,
+      "loss": 0.1659,
+      "num_input_tokens_seen": 1331168,
+      "step": 81
+    },
+    {
+      "epoch": 1.0546623794212218,
+      "grad_norm": 3.6471991539001465,
+      "learning_rate": 6.833333333333334e-07,
+      "loss": 0.1619,
+      "num_input_tokens_seen": 1347968,
+      "step": 82
+    },
+    {
+      "epoch": 1.067524115755627,
+      "grad_norm": 4.136232376098633,
+      "learning_rate": 6.916666666666668e-07,
+      "loss": 0.168,
+      "num_input_tokens_seen": 1363872,
+      "step": 83
+    },
+    {
+      "epoch": 1.0803858520900322,
+      "grad_norm": 4.436996936798096,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.1627,
+      "num_input_tokens_seen": 1380896,
+      "step": 84
+    },
+    {
+      "epoch": 1.0932475884244373,
+      "grad_norm": 10.516170501708984,
+      "learning_rate": 7.083333333333334e-07,
+      "loss": 0.1421,
+      "num_input_tokens_seen": 1397792,
+      "step": 85
+    },
+    {
+      "epoch": 1.1061093247588425,
+      "grad_norm": 15.584237098693848,
+      "learning_rate": 7.166666666666668e-07,
+      "loss": 0.1517,
+      "num_input_tokens_seen": 1414272,
+      "step": 86
+    },
+    {
+      "epoch": 1.1189710610932475,
+      "grad_norm": 8.475227355957031,
+      "learning_rate": 7.25e-07,
+      "loss": 0.1519,
+      "num_input_tokens_seen": 1429696,
+      "step": 87
+    },
+    {
+      "epoch": 1.1318327974276527,
+      "grad_norm": 5.768808841705322,
+      "learning_rate": 7.333333333333334e-07,
+      "loss": 0.128,
+      "num_input_tokens_seen": 1446720,
+      "step": 88
+    },
+    {
+      "epoch": 1.144694533762058,
+      "grad_norm": 15.064901351928711,
+      "learning_rate": 7.416666666666668e-07,
+      "loss": 0.1481,
+      "num_input_tokens_seen": 1463008,
+      "step": 89
+    },
+    {
+      "epoch": 1.157556270096463,
+      "grad_norm": 17.70471954345703,
+      "learning_rate": 7.5e-07,
+      "loss": 0.1636,
+      "num_input_tokens_seen": 1480096,
+      "step": 90
+    },
+    {
+      "epoch": 1.1704180064308682,
+      "grad_norm": 5.037917613983154,
+      "learning_rate": 7.583333333333334e-07,
+      "loss": 0.1443,
+      "num_input_tokens_seen": 1497024,
+      "step": 91
+    },
+    {
+      "epoch": 1.1832797427652733,
+      "grad_norm": 17.397666931152344,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.1592,
+      "num_input_tokens_seen": 1513792,
+      "step": 92
+    },
+    {
+      "epoch": 1.1961414790996785,
+      "grad_norm": 18.808683395385742,
+      "learning_rate": 7.750000000000001e-07,
+      "loss": 0.1744,
+      "num_input_tokens_seen": 1529696,
+      "step": 93
+    },
+    {
+      "epoch": 1.2090032154340835,
+      "grad_norm": 15.173974990844727,
+      "learning_rate": 7.833333333333335e-07,
+      "loss": 0.1615,
+      "num_input_tokens_seen": 1546400,
+      "step": 94
+    },
+    {
+      "epoch": 1.2218649517684887,
+      "grad_norm": 3.6459403038024902,
+      "learning_rate": 7.916666666666667e-07,
+      "loss": 0.1466,
+      "num_input_tokens_seen": 1564448,
+      "step": 95
+    },
+    {
+      "epoch": 1.234726688102894,
+      "grad_norm": 4.910373210906982,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.1333,
+      "num_input_tokens_seen": 1580736,
+      "step": 96
+    },
+    {
+      "epoch": 1.247588424437299,
+      "grad_norm": 11.399758338928223,
+      "learning_rate": 8.083333333333334e-07,
+      "loss": 0.1475,
+      "num_input_tokens_seen": 1596928,
+      "step": 97
+    },
+    {
+      "epoch": 1.2604501607717042,
+      "grad_norm": 4.701439380645752,
+      "learning_rate": 8.166666666666668e-07,
+      "loss": 0.142,
+      "num_input_tokens_seen": 1613728,
+      "step": 98
+    },
+    {
+      "epoch": 1.2733118971061093,
+      "grad_norm": 7.250323295593262,
+      "learning_rate": 8.250000000000001e-07,
+      "loss": 0.148,
+      "num_input_tokens_seen": 1630112,
+      "step": 99
+    },
+    {
+      "epoch": 1.2861736334405145,
+      "grad_norm": 17.24268341064453,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.1539,
+      "num_input_tokens_seen": 1645248,
+      "step": 100
+    },
+    {
+      "epoch": 1.2990353697749195,
+      "grad_norm": 4.017528057098389,
+      "learning_rate": 8.416666666666667e-07,
+      "loss": 0.1496,
+      "num_input_tokens_seen": 1662400,
+      "step": 101
+    },
+    {
+      "epoch": 1.3118971061093248,
+      "grad_norm": 10.226727485656738,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 0.1563,
+      "num_input_tokens_seen": 1678528,
+      "step": 102
+    },
+    {
+      "epoch": 1.32475884244373,
+      "grad_norm": 4.085242748260498,
+      "learning_rate": 8.583333333333334e-07,
+      "loss": 0.1295,
+      "num_input_tokens_seen": 1694432,
+      "step": 103
+    },
+    {
+      "epoch": 1.337620578778135,
+      "grad_norm": 9.983370780944824,
+      "learning_rate": 8.666666666666668e-07,
+      "loss": 0.1353,
+      "num_input_tokens_seen": 1710624,
+      "step": 104
+    },
+    {
+      "epoch": 1.3504823151125402,
+      "grad_norm": 6.53292179107666,
+      "learning_rate": 8.75e-07,
+      "loss": 0.1376,
+      "num_input_tokens_seen": 1726560,
+      "step": 105
+    },
+    {
+      "epoch": 1.3633440514469453,
+      "grad_norm": 4.412125587463379,
+      "learning_rate": 8.833333333333334e-07,
+      "loss": 0.1313,
+      "num_input_tokens_seen": 1742912,
+      "step": 106
+    },
+    {
+      "epoch": 1.3762057877813505,
+      "grad_norm": 13.620604515075684,
+      "learning_rate": 8.916666666666668e-07,
+      "loss": 0.1367,
+      "num_input_tokens_seen": 1758688,
+      "step": 107
+    },
+    {
+      "epoch": 1.3890675241157555,
+      "grad_norm": 6.362453937530518,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.135,
+      "num_input_tokens_seen": 1775264,
+      "step": 108
+    },
+    {
+      "epoch": 1.4019292604501608,
+      "grad_norm": 3.8915514945983887,
+      "learning_rate": 9.083333333333335e-07,
+      "loss": 0.1212,
+      "num_input_tokens_seen": 1791904,
+      "step": 109
+    },
+    {
+      "epoch": 1.414790996784566,
+      "grad_norm": 9.137332916259766,
+      "learning_rate": 9.166666666666666e-07,
+      "loss": 0.1355,
+      "num_input_tokens_seen": 1808736,
+      "step": 110
+    },
+    {
+      "epoch": 1.427652733118971,
+      "grad_norm": 4.745510578155518,
+      "learning_rate": 9.25e-07,
+      "loss": 0.1439,
+      "num_input_tokens_seen": 1825504,
+      "step": 111
+    },
+    {
+      "epoch": 1.4405144694533762,
+      "grad_norm": 10.531625747680664,
+      "learning_rate": 9.333333333333334e-07,
+      "loss": 0.1323,
+      "num_input_tokens_seen": 1842080,
+      "step": 112
+    },
+    {
+      "epoch": 1.4533762057877815,
+      "grad_norm": 11.189078330993652,
+      "learning_rate": 9.416666666666667e-07,
+      "loss": 0.1376,
+      "num_input_tokens_seen": 1858432,
+      "step": 113
+    },
+    {
+      "epoch": 1.4662379421221865,
+      "grad_norm": 5.619433879852295,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 0.1191,
+      "num_input_tokens_seen": 1874304,
+      "step": 114
+    },
+    {
+      "epoch": 1.4790996784565915,
+      "grad_norm": 4.590907096862793,
+      "learning_rate": 9.583333333333334e-07,
+      "loss": 0.1045,
+      "num_input_tokens_seen": 1890144,
+      "step": 115
+    },
+    {
+      "epoch": 1.4919614147909968,
+      "grad_norm": 4.900210380554199,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 0.1467,
+      "num_input_tokens_seen": 1905664,
+      "step": 116
+    },
+    {
+      "epoch": 1.504823151125402,
+      "grad_norm": 3.0244414806365967,
+      "learning_rate": 9.750000000000002e-07,
+      "loss": 0.1142,
+      "num_input_tokens_seen": 1922240,
+      "step": 117
+    },
+    {
+      "epoch": 1.517684887459807,
+      "grad_norm": 2.886939287185669,
+      "learning_rate": 9.833333333333334e-07,
+      "loss": 0.1107,
+      "num_input_tokens_seen": 1938240,
+      "step": 118
+    },
+    {
+      "epoch": 1.5305466237942122,
+      "grad_norm": 6.869494438171387,
+      "learning_rate": 9.916666666666668e-07,
+      "loss": 0.146,
+      "num_input_tokens_seen": 1955008,
+      "step": 119
+    },
+    {
+      "epoch": 1.5434083601286175,
+      "grad_norm": 16.34566879272461,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1533,
+      "num_input_tokens_seen": 1971456,
+      "step": 120
+    },
+    {
+      "epoch": 1.5562700964630225,
+      "grad_norm": 15.347967147827148,
+      "learning_rate": 1.0083333333333333e-06,
+      "loss": 0.1315,
+      "num_input_tokens_seen": 1987552,
+      "step": 121
+    },
+    {
+      "epoch": 1.5691318327974275,
+      "grad_norm": 4.213266372680664,
+      "learning_rate": 1.0166666666666667e-06,
+      "loss": 0.1197,
+      "num_input_tokens_seen": 2004192,
+      "step": 122
+    },
+    {
+      "epoch": 1.5819935691318328,
+      "grad_norm": 14.208905220031738,
+      "learning_rate": 1.025e-06,
+      "loss": 0.1228,
+      "num_input_tokens_seen": 2020832,
+      "step": 123
+    },
+    {
+      "epoch": 1.594855305466238,
+      "grad_norm": 17.388845443725586,
+      "learning_rate": 1.0333333333333333e-06,
+      "loss": 0.1273,
+      "num_input_tokens_seen": 2037312,
+      "step": 124
+    },
+    {
+      "epoch": 1.607717041800643,
+      "grad_norm": 14.383207321166992,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 0.141,
+      "num_input_tokens_seen": 2054304,
+      "step": 125
+    },
+    {
+      "epoch": 1.6205787781350482,
+      "grad_norm": 6.2650370597839355,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 0.1315,
+      "num_input_tokens_seen": 2071136,
+      "step": 126
+    },
+    {
+      "epoch": 1.6334405144694535,
+      "grad_norm": 8.396490097045898,
+      "learning_rate": 1.0583333333333335e-06,
+      "loss": 0.1136,
+      "num_input_tokens_seen": 2088064,
+      "step": 127
+    },
+    {
+      "epoch": 1.6463022508038585,
+      "grad_norm": 4.598783016204834,
+      "learning_rate": 1.066666666666667e-06,
+      "loss": 0.1013,
+      "num_input_tokens_seen": 2105120,
+      "step": 128
+    },
+    {
+      "epoch": 1.6591639871382635,
+      "grad_norm": 9.912994384765625,
+      "learning_rate": 1.075e-06,
+      "loss": 0.1056,
+      "num_input_tokens_seen": 2120992,
+      "step": 129
+    },
+    {
+      "epoch": 1.6720257234726688,
+      "grad_norm": 7.70595645904541,
+      "learning_rate": 1.0833333333333335e-06,
+      "loss": 0.1071,
+      "num_input_tokens_seen": 2137984,
+      "step": 130
+    },
+    {
+      "epoch": 1.684887459807074,
+      "grad_norm": 6.183793544769287,
+      "learning_rate": 1.0916666666666667e-06,
+      "loss": 0.1357,
+      "num_input_tokens_seen": 2154240,
+      "step": 131
+    },
+    {
+      "epoch": 1.697749196141479,
+      "grad_norm": 6.863802909851074,
+      "learning_rate": 1.1e-06,
+      "loss": 0.1181,
+      "num_input_tokens_seen": 2170464,
+      "step": 132
+    },
+    {
+      "epoch": 1.7106109324758842,
+      "grad_norm": 2.7540361881256104,
+      "learning_rate": 1.1083333333333335e-06,
+      "loss": 0.0826,
+      "num_input_tokens_seen": 2187008,
+      "step": 133
+    },
+    {
+      "epoch": 1.7234726688102895,
+      "grad_norm": 12.97872257232666,
+      "learning_rate": 1.1166666666666666e-06,
+      "loss": 0.1221,
+      "num_input_tokens_seen": 2202880,
+      "step": 134
+    },
+    {
+      "epoch": 1.7363344051446945,
+      "grad_norm": 4.774652004241943,
+      "learning_rate": 1.125e-06,
+      "loss": 0.1021,
+      "num_input_tokens_seen": 2219744,
+      "step": 135
+    },
+    {
+      "epoch": 1.7491961414790995,
+      "grad_norm": 3.889441967010498,
+      "learning_rate": 1.1333333333333334e-06,
+      "loss": 0.098,
+      "num_input_tokens_seen": 2236064,
+      "step": 136
+    },
+    {
+      "epoch": 1.762057877813505,
+      "grad_norm": 5.095253944396973,
+      "learning_rate": 1.1416666666666668e-06,
+      "loss": 0.1085,
+      "num_input_tokens_seen": 2252832,
+      "step": 137
+    },
+    {
+      "epoch": 1.77491961414791,
+      "grad_norm": 5.206950664520264,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 0.1038,
+      "num_input_tokens_seen": 2268704,
+      "step": 138
+    },
+    {
+      "epoch": 1.787781350482315,
+      "grad_norm": 10.339069366455078,
+      "learning_rate": 1.1583333333333334e-06,
+      "loss": 0.1081,
+      "num_input_tokens_seen": 2285280,
+      "step": 139
+    },
+    {
+      "epoch": 1.8006430868167203,
+      "grad_norm": 8.227212905883789,
+      "learning_rate": 1.1666666666666668e-06,
+      "loss": 0.1287,
+      "num_input_tokens_seen": 2301600,
+      "step": 140
+    },
+    {
+      "epoch": 1.8135048231511255,
+      "grad_norm": 6.72472620010376,
+      "learning_rate": 1.175e-06,
+      "loss": 0.1068,
+      "num_input_tokens_seen": 2318048,
+      "step": 141
+    },
+    {
+      "epoch": 1.8263665594855305,
+      "grad_norm": 10.02261734008789,
+      "learning_rate": 1.1833333333333334e-06,
+      "loss": 0.1202,
+      "num_input_tokens_seen": 2333728,
+      "step": 142
+    },
+    {
+      "epoch": 1.8392282958199357,
+      "grad_norm": 10.770837783813477,
+      "learning_rate": 1.1916666666666668e-06,
+      "loss": 0.119,
+      "num_input_tokens_seen": 2349760,
+      "step": 143
+    },
+    {
+      "epoch": 1.852090032154341,
+      "grad_norm": 6.685535907745361,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.1273,
+      "num_input_tokens_seen": 2366336,
+      "step": 144
+    },
+    {
+      "epoch": 1.864951768488746,
+      "grad_norm": 4.437434673309326,
+      "learning_rate": 1.2083333333333333e-06,
+      "loss": 0.1024,
+      "num_input_tokens_seen": 2382016,
+      "step": 145
+    },
+    {
+      "epoch": 1.877813504823151,
+      "grad_norm": 11.942359924316406,
+      "learning_rate": 1.2166666666666667e-06,
+      "loss": 0.1143,
+      "num_input_tokens_seen": 2398176,
+      "step": 146
+    },
+    {
+      "epoch": 1.8906752411575563,
+      "grad_norm": 6.428237438201904,
+      "learning_rate": 1.2250000000000001e-06,
+      "loss": 0.1229,
+      "num_input_tokens_seen": 2414656,
+      "step": 147
+    },
+    {
+      "epoch": 1.9035369774919615,
+      "grad_norm": 7.896732807159424,
+      "learning_rate": 1.2333333333333335e-06,
+      "loss": 0.0964,
+      "num_input_tokens_seen": 2430400,
+      "step": 148
+    },
+    {
+      "epoch": 1.9163987138263665,
+      "grad_norm": 12.14167594909668,
+      "learning_rate": 1.2416666666666667e-06,
+      "loss": 0.1132,
+      "num_input_tokens_seen": 2447520,
+      "step": 149
+    },
+    {
+      "epoch": 1.9292604501607717,
+      "grad_norm": 3.0627174377441406,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0734,
+      "num_input_tokens_seen": 2463872,
+      "step": 150
+    },
+    {
+      "epoch": 1.942122186495177,
+      "grad_norm": 3.2333552837371826,
+      "learning_rate": 1.2583333333333333e-06,
+      "loss": 0.0939,
+      "num_input_tokens_seen": 2481344,
+      "step": 151
+    },
+    {
+      "epoch": 1.954983922829582,
+      "grad_norm": 14.160690307617188,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 0.1143,
+      "num_input_tokens_seen": 2497632,
+      "step": 152
+    },
+    {
+      "epoch": 1.967845659163987,
+      "grad_norm": 5.38826322555542,
+      "learning_rate": 1.275e-06,
+      "loss": 0.1114,
+      "num_input_tokens_seen": 2513024,
+      "step": 153
+    },
+    {
+      "epoch": 1.9807073954983923,
+      "grad_norm": 3.4523892402648926,
+      "learning_rate": 1.2833333333333335e-06,
+      "loss": 0.0948,
+      "num_input_tokens_seen": 2528800,
+      "step": 154
+    },
+    {
+      "epoch": 1.9935691318327975,
+      "grad_norm": 6.9574127197265625,
+      "learning_rate": 1.2916666666666669e-06,
+      "loss": 0.0805,
+      "num_input_tokens_seen": 2545696,
+      "step": 155
+    },
+    {
+      "epoch": 2.0064308681672025,
+      "grad_norm": 7.220541954040527,
+      "learning_rate": 1.3e-06,
+      "loss": 0.1001,
+      "num_input_tokens_seen": 2561728,
+      "step": 156
+    },
+    {
+      "epoch": 2.0192926045016075,
+      "grad_norm": 6.95176887512207,
+      "learning_rate": 1.3083333333333334e-06,
+      "loss": 0.1002,
+      "num_input_tokens_seen": 2577856,
+      "step": 157
+    },
+    {
+      "epoch": 2.032154340836013,
+      "grad_norm": 10.752556800842285,
+      "learning_rate": 1.3166666666666666e-06,
+      "loss": 0.0847,
+      "num_input_tokens_seen": 2593984,
+      "step": 158
+    },
+    {
+      "epoch": 2.045016077170418,
+      "grad_norm": 6.510659217834473,
+      "learning_rate": 1.3250000000000002e-06,
+      "loss": 0.071,
+      "num_input_tokens_seen": 2610720,
+      "step": 159
+    },
+    {
+      "epoch": 2.057877813504823,
+      "grad_norm": 3.897312641143799,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0791,
+      "num_input_tokens_seen": 2626400,
+      "step": 160
+    },
+    {
+      "epoch": 2.0707395498392285,
+      "grad_norm": 2.8594729900360107,
+      "learning_rate": 1.3416666666666666e-06,
+      "loss": 0.0769,
+      "num_input_tokens_seen": 2642496,
+      "step": 161
+    },
+    {
+      "epoch": 2.0836012861736335,
+      "grad_norm": 8.970246315002441,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 0.0916,
+      "num_input_tokens_seen": 2658144,
+      "step": 162
+    },
+    {
+      "epoch": 2.0964630225080385,
+      "grad_norm": 2.965878963470459,
+      "learning_rate": 1.3583333333333334e-06,
+      "loss": 0.0584,
+      "num_input_tokens_seen": 2674560,
+      "step": 163
+    },
+    {
+      "epoch": 2.1093247588424435,
+      "grad_norm": 6.79748010635376,
+      "learning_rate": 1.3666666666666668e-06,
+      "loss": 0.0811,
+      "num_input_tokens_seen": 2690688,
+      "step": 164
+    },
+    {
+      "epoch": 2.122186495176849,
+      "grad_norm": 10.366084098815918,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.0663,
+      "num_input_tokens_seen": 2706720,
+      "step": 165
+    },
+    {
+      "epoch": 2.135048231511254,
+      "grad_norm": 8.038703918457031,
+      "learning_rate": 1.3833333333333336e-06,
+      "loss": 0.0802,
+      "num_input_tokens_seen": 2723008,
+      "step": 166
+    },
+    {
+      "epoch": 2.147909967845659,
+      "grad_norm": 6.01867151260376,
+      "learning_rate": 1.3916666666666668e-06,
+      "loss": 0.0587,
+      "num_input_tokens_seen": 2738784,
+      "step": 167
+    },
+    {
+      "epoch": 2.1607717041800645,
+      "grad_norm": 5.053437232971191,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0953,
+      "num_input_tokens_seen": 2755072,
+      "step": 168
+    },
+    {
+      "epoch": 2.1736334405144695,
+      "grad_norm": 7.119932174682617,
+      "learning_rate": 1.4083333333333335e-06,
+      "loss": 0.0795,
+      "num_input_tokens_seen": 2771296,
+      "step": 169
+    },
+    {
+      "epoch": 2.1864951768488745,
+      "grad_norm": 4.602327346801758,
+      "learning_rate": 1.4166666666666667e-06,
+      "loss": 0.1015,
+      "num_input_tokens_seen": 2787712,
+      "step": 170
+    },
+    {
+      "epoch": 2.19935691318328,
+      "grad_norm": 2.7282161712646484,
+      "learning_rate": 1.425e-06,
+      "loss": 0.0614,
+      "num_input_tokens_seen": 2803552,
+      "step": 171
+    },
+    {
+      "epoch": 2.212218649517685,
+      "grad_norm": 4.694391250610352,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 0.0753,
+      "num_input_tokens_seen": 2820352,
+      "step": 172
+    },
+    {
+      "epoch": 2.22508038585209,
+      "grad_norm": 3.231022357940674,
+      "learning_rate": 1.4416666666666667e-06,
+      "loss": 0.08,
+      "num_input_tokens_seen": 2837696,
+      "step": 173
+    },
+    {
+      "epoch": 2.237942122186495,
+      "grad_norm": 4.522435188293457,
+      "learning_rate": 1.45e-06,
+      "loss": 0.0603,
+      "num_input_tokens_seen": 2854272,
+      "step": 174
+    },
+    {
+      "epoch": 2.2508038585209005,
+      "grad_norm": 8.292889595031738,
+      "learning_rate": 1.4583333333333335e-06,
+      "loss": 0.0874,
+      "num_input_tokens_seen": 2870464,
+      "step": 175
+    },
+    {
+      "epoch": 2.2636655948553055,
+      "grad_norm": 3.635068655014038,
+      "learning_rate": 1.4666666666666669e-06,
+      "loss": 0.0833,
+      "num_input_tokens_seen": 2886880,
+      "step": 176
+    },
+    {
+      "epoch": 2.2765273311897105,
+      "grad_norm": 3.8000781536102295,
+      "learning_rate": 1.475e-06,
+      "loss": 0.0676,
+      "num_input_tokens_seen": 2902880,
+      "step": 177
+    },
+    {
+      "epoch": 2.289389067524116,
+      "grad_norm": 9.920318603515625,
+      "learning_rate": 1.4833333333333337e-06,
+      "loss": 0.086,
+      "num_input_tokens_seen": 2919584,
+      "step": 178
+    },
+    {
+      "epoch": 2.302250803858521,
+      "grad_norm": 3.8999791145324707,
+      "learning_rate": 1.4916666666666669e-06,
+      "loss": 0.0662,
+      "num_input_tokens_seen": 2935136,
+      "step": 179
+    },
+    {
+      "epoch": 2.315112540192926,
+      "grad_norm": 6.011571407318115,
+      "learning_rate": 1.5e-06,
+      "loss": 0.1028,
+      "num_input_tokens_seen": 2951680,
+      "step": 180
+    },
+    {
+      "epoch": 2.327974276527331,
+      "grad_norm": 5.94368839263916,
+      "learning_rate": 1.5083333333333336e-06,
+      "loss": 0.0882,
+      "num_input_tokens_seen": 2967712,
+      "step": 181
+    },
+    {
+      "epoch": 2.3408360128617365,
+      "grad_norm": 3.9432876110076904,
+      "learning_rate": 1.5166666666666668e-06,
+      "loss": 0.0806,
+      "num_input_tokens_seen": 2984224,
+      "step": 182
+    },
+    {
+      "epoch": 2.3536977491961415,
+      "grad_norm": 18.06218147277832,
+      "learning_rate": 1.525e-06,
+      "loss": 0.1108,
+      "num_input_tokens_seen": 2999904,
+      "step": 183
+    },
+    {
+      "epoch": 2.3665594855305465,
+      "grad_norm": 5.799235820770264,
+      "learning_rate": 1.5333333333333334e-06,
+      "loss": 0.0687,
+      "num_input_tokens_seen": 3017376,
+      "step": 184
+    },
+    {
+      "epoch": 2.379421221864952,
+      "grad_norm": 4.252455711364746,
+      "learning_rate": 1.5416666666666668e-06,
+      "loss": 0.0736,
+      "num_input_tokens_seen": 3033728,
+      "step": 185
+    },
+    {
+      "epoch": 2.392282958199357,
+      "grad_norm": 6.050833702087402,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 0.0779,
+      "num_input_tokens_seen": 3050176,
+      "step": 186
+    },
+    {
+      "epoch": 2.405144694533762,
+      "grad_norm": 8.076692581176758,
+      "learning_rate": 1.5583333333333334e-06,
+      "loss": 0.0709,
+      "num_input_tokens_seen": 3066400,
+      "step": 187
+    },
+    {
+      "epoch": 2.418006430868167,
+      "grad_norm": 7.372990608215332,
+      "learning_rate": 1.566666666666667e-06,
+      "loss": 0.0652,
+      "num_input_tokens_seen": 3083104,
+      "step": 188
+    },
+    {
+      "epoch": 2.4308681672025725,
+      "grad_norm": 14.172348022460938,
+      "learning_rate": 1.5750000000000002e-06,
+      "loss": 0.1095,
+      "num_input_tokens_seen": 3100000,
+      "step": 189
+    },
+    {
+      "epoch": 2.4437299035369775,
+      "grad_norm": 4.726587772369385,
+      "learning_rate": 1.5833333333333333e-06,
+      "loss": 0.0618,
+      "num_input_tokens_seen": 3116800,
+      "step": 190
+    },
+    {
+      "epoch": 2.4565916398713825,
+      "grad_norm": 10.408024787902832,
+      "learning_rate": 1.591666666666667e-06,
+      "loss": 0.0666,
+      "num_input_tokens_seen": 3133568,
+      "step": 191
+    },
+    {
+      "epoch": 2.469453376205788,
+      "grad_norm": 7.295907497406006,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0573,
+      "num_input_tokens_seen": 3150048,
+      "step": 192
+    },
+    {
+      "epoch": 2.482315112540193,
+      "grad_norm": 6.490230083465576,
+      "learning_rate": 1.6083333333333333e-06,
+      "loss": 0.0577,
+      "num_input_tokens_seen": 3166688,
+      "step": 193
+    },
+    {
+      "epoch": 2.495176848874598,
+      "grad_norm": 6.940796375274658,
+      "learning_rate": 1.6166666666666667e-06,
+      "loss": 0.0813,
+      "num_input_tokens_seen": 3182752,
+      "step": 194
+    },
+    {
+      "epoch": 2.508038585209003,
+      "grad_norm": 8.307454109191895,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.066,
+      "num_input_tokens_seen": 3198944,
+      "step": 195
+    },
+    {
+      "epoch": 2.5209003215434085,
+      "grad_norm": 4.472862243652344,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 0.0622,
+      "num_input_tokens_seen": 3215552,
+      "step": 196
+    },
+    {
+      "epoch": 2.5337620578778135,
+      "grad_norm": 3.5001590251922607,
+      "learning_rate": 1.6416666666666667e-06,
+      "loss": 0.0616,
+      "num_input_tokens_seen": 3231488,
+      "step": 197
+    },
+    {
+      "epoch": 2.5466237942122185,
+      "grad_norm": 13.425641059875488,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 0.0993,
+      "num_input_tokens_seen": 3247840,
+      "step": 198
+    },
+    {
+      "epoch": 2.559485530546624,
+      "grad_norm": 12.016642570495605,
+      "learning_rate": 1.6583333333333335e-06,
+      "loss": 0.0702,
+      "num_input_tokens_seen": 3264128,
+      "step": 199
+    },
+    {
+      "epoch": 2.572347266881029,
+      "grad_norm": 10.653433799743652,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0743,
+      "num_input_tokens_seen": 3280192,
+      "step": 200
+    },
+    {
+      "epoch": 2.585209003215434,
+      "grad_norm": 3.59945011138916,
+      "learning_rate": 1.6750000000000003e-06,
+      "loss": 0.0647,
+      "num_input_tokens_seen": 3296480,
+      "step": 201
+    },
+    {
+      "epoch": 2.598070739549839,
+      "grad_norm": 11.130800247192383,
+      "learning_rate": 1.6833333333333335e-06,
+      "loss": 0.0814,
+      "num_input_tokens_seen": 3313120,
+      "step": 202
+    },
+    {
+      "epoch": 2.6109324758842445,
+      "grad_norm": 11.733818054199219,
+      "learning_rate": 1.6916666666666666e-06,
+      "loss": 0.0861,
+      "num_input_tokens_seen": 3329184,
+      "step": 203
+    },
+    {
+      "epoch": 2.6237942122186495,
+      "grad_norm": 6.696333885192871,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0769,
+      "num_input_tokens_seen": 3346208,
+      "step": 204
+    },
+    {
+      "epoch": 2.6366559485530545,
+      "grad_norm": 8.523835182189941,
+      "learning_rate": 1.7083333333333334e-06,
+      "loss": 0.0888,
+      "num_input_tokens_seen": 3363008,
+      "step": 205
+    },
+    {
+      "epoch": 2.64951768488746,
+      "grad_norm": 12.503429412841797,
+      "learning_rate": 1.7166666666666668e-06,
+      "loss": 0.1017,
+      "num_input_tokens_seen": 3379488,
+      "step": 206
+    },
+    {
+      "epoch": 2.662379421221865,
+      "grad_norm": 7.644631385803223,
+      "learning_rate": 1.725e-06,
+      "loss": 0.0677,
+      "num_input_tokens_seen": 3395648,
+      "step": 207
+    },
+    {
+      "epoch": 2.67524115755627,
+      "grad_norm": 7.251969814300537,
+      "learning_rate": 1.7333333333333336e-06,
+      "loss": 0.0861,
+      "num_input_tokens_seen": 3412736,
+      "step": 208
+    },
+    {
+      "epoch": 2.688102893890675,
+      "grad_norm": 3.763118267059326,
+      "learning_rate": 1.7416666666666668e-06,
+      "loss": 0.0562,
+      "num_input_tokens_seen": 3429472,
+      "step": 209
+    },
+    {
+      "epoch": 2.7009646302250805,
+      "grad_norm": 8.583102226257324,
+      "learning_rate": 1.75e-06,
+      "loss": 0.0641,
+      "num_input_tokens_seen": 3445632,
+      "step": 210
+    },
+    {
+      "epoch": 2.7138263665594855,
+      "grad_norm": 9.621906280517578,
+      "learning_rate": 1.7583333333333336e-06,
+      "loss": 0.0829,
+      "num_input_tokens_seen": 3462720,
+      "step": 211
+    },
+    {
+      "epoch": 2.7266881028938905,
+      "grad_norm": 6.067561626434326,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 0.0632,
+      "num_input_tokens_seen": 3478720,
+      "step": 212
+    },
+    {
+      "epoch": 2.739549839228296,
+      "grad_norm": 4.912470817565918,
+      "learning_rate": 1.7750000000000002e-06,
+      "loss": 0.062,
+      "num_input_tokens_seen": 3494304,
+      "step": 213
+    },
+    {
+      "epoch": 2.752411575562701,
+      "grad_norm": 8.044519424438477,
+      "learning_rate": 1.7833333333333336e-06,
+      "loss": 0.0816,
+      "num_input_tokens_seen": 3510176,
+      "step": 214
+    },
+    {
+      "epoch": 2.765273311897106,
+      "grad_norm": 8.589887619018555,
+      "learning_rate": 1.7916666666666667e-06,
+      "loss": 0.096,
+      "num_input_tokens_seen": 3526848,
+      "step": 215
+    },
+    {
+      "epoch": 2.778135048231511,
+      "grad_norm": 3.3357672691345215,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0639,
+      "num_input_tokens_seen": 3543136,
+      "step": 216
+    },
+    {
+      "epoch": 2.7909967845659165,
+      "grad_norm": 5.992812156677246,
+      "learning_rate": 1.8083333333333335e-06,
+      "loss": 0.0915,
+      "num_input_tokens_seen": 3558240,
+      "step": 217
+    },
+    {
+      "epoch": 2.8038585209003215,
+      "grad_norm": 4.8690924644470215,
+      "learning_rate": 1.816666666666667e-06,
+      "loss": 0.061,
+      "num_input_tokens_seen": 3574240,
+      "step": 218
+    },
+    {
+      "epoch": 2.816720257234727,
+      "grad_norm": 2.928898811340332,
+      "learning_rate": 1.825e-06,
+      "loss": 0.0572,
+      "num_input_tokens_seen": 3590240,
+      "step": 219
+    },
+    {
+      "epoch": 2.829581993569132,
+      "grad_norm": 2.8405849933624268,
+      "learning_rate": 1.8333333333333333e-06,
+      "loss": 0.0497,
+      "num_input_tokens_seen": 3606912,
+      "step": 220
+    },
+    {
+      "epoch": 2.842443729903537,
+      "grad_norm": 5.940427303314209,
+      "learning_rate": 1.8416666666666669e-06,
+      "loss": 0.0672,
+      "num_input_tokens_seen": 3623008,
+      "step": 221
+    },
+    {
+      "epoch": 2.855305466237942,
+      "grad_norm": 4.370597839355469,
+      "learning_rate": 1.85e-06,
+      "loss": 0.0563,
+      "num_input_tokens_seen": 3639456,
+      "step": 222
+    },
+    {
+      "epoch": 2.868167202572347,
+      "grad_norm": 3.9959638118743896,
+      "learning_rate": 1.8583333333333335e-06,
+      "loss": 0.069,
+      "num_input_tokens_seen": 3656288,
+      "step": 223
+    },
+    {
+      "epoch": 2.8810289389067525,
+      "grad_norm": 4.756357192993164,
+      "learning_rate": 1.8666666666666669e-06,
+      "loss": 0.0824,
+      "num_input_tokens_seen": 3672896,
+      "step": 224
+    },
+    {
+      "epoch": 2.8938906752411575,
+      "grad_norm": 3.312553644180298,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.057,
+      "num_input_tokens_seen": 3689632,
+      "step": 225
+    },
+    {
+      "epoch": 2.906752411575563,
+      "grad_norm": 3.71880841255188,
+      "learning_rate": 1.8833333333333334e-06,
+      "loss": 0.0549,
+      "num_input_tokens_seen": 3706944,
+      "step": 226
+    },
+    {
+      "epoch": 2.919614147909968,
+      "grad_norm": 4.111685276031494,
+      "learning_rate": 1.8916666666666668e-06,
+      "loss": 0.0652,
+      "num_input_tokens_seen": 3723072,
+      "step": 227
+    },
+    {
+      "epoch": 2.932475884244373,
+      "grad_norm": 4.362407684326172,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.0743,
+      "num_input_tokens_seen": 3740064,
+      "step": 228
+    },
+    {
+      "epoch": 2.945337620578778,
+      "grad_norm": 2.441969156265259,
+      "learning_rate": 1.9083333333333334e-06,
+      "loss": 0.0416,
+      "num_input_tokens_seen": 3756864,
+      "step": 229
+    },
+    {
+      "epoch": 2.958199356913183,
+      "grad_norm": 3.4988579750061035,
+      "learning_rate": 1.916666666666667e-06,
+      "loss": 0.0668,
+      "num_input_tokens_seen": 3774336,
+      "step": 230
+    },
+    {
+      "epoch": 2.9710610932475885,
+      "grad_norm": 3.7807693481445312,
+      "learning_rate": 1.925e-06,
+      "loss": 0.0603,
+      "num_input_tokens_seen": 3790336,
+      "step": 231
+    },
+    {
+      "epoch": 2.9839228295819935,
+      "grad_norm": 3.471942901611328,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 0.066,
+      "num_input_tokens_seen": 3807520,
+      "step": 232
+    },
+    {
+      "epoch": 2.996784565916399,
+      "grad_norm": 3.4720284938812256,
+      "learning_rate": 1.9416666666666666e-06,
+      "loss": 0.0692,
+      "num_input_tokens_seen": 3823872,
+      "step": 233
+    },
+    {
+      "epoch": 3.009646302250804,
+      "grad_norm": 1.9317958354949951,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 0.0323,
+      "num_input_tokens_seen": 3840320,
+      "step": 234
+    },
+    {
+      "epoch": 3.022508038585209,
+      "grad_norm": 4.849405288696289,
+      "learning_rate": 1.9583333333333334e-06,
+      "loss": 0.0403,
+      "num_input_tokens_seen": 3856256,
+      "step": 235
+    },
+    {
+      "epoch": 3.035369774919614,
+      "grad_norm": 2.371988296508789,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 0.0477,
+      "num_input_tokens_seen": 3872512,
+      "step": 236
+    },
+    {
+      "epoch": 3.0482315112540195,
+      "grad_norm": 3.362964153289795,
+      "learning_rate": 1.975e-06,
+      "loss": 0.0356,
+      "num_input_tokens_seen": 3888256,
+      "step": 237
+    },
+    {
+      "epoch": 3.0610932475884245,
+      "grad_norm": 2.618176221847534,
+      "learning_rate": 1.9833333333333335e-06,
+      "loss": 0.0324,
+      "num_input_tokens_seen": 3905024,
+      "step": 238
+    },
+    {
+      "epoch": 3.0739549839228295,
+      "grad_norm": 2.5595703125,
+      "learning_rate": 1.991666666666667e-06,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 3922016,
+      "step": 239
+    },
+    {
+      "epoch": 3.0868167202572345,
+      "grad_norm": 2.788818597793579,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0315,
+      "num_input_tokens_seen": 3938400,
+      "step": 240
+    },
+    {
+      "epoch": 3.09967845659164,
+      "grad_norm": 3.9531607627868652,
+      "learning_rate": 2.0083333333333337e-06,
+      "loss": 0.0366,
+      "num_input_tokens_seen": 3954912,
+      "step": 241
+    },
+    {
+      "epoch": 3.112540192926045,
+      "grad_norm": 3.2901012897491455,
+      "learning_rate": 2.0166666666666667e-06,
+      "loss": 0.0307,
+      "num_input_tokens_seen": 3970400,
+      "step": 242
+    },
+    {
+      "epoch": 3.12540192926045,
+      "grad_norm": 4.283529281616211,
+      "learning_rate": 2.025e-06,
+      "loss": 0.0338,
+      "num_input_tokens_seen": 3986944,
+      "step": 243
+    },
+    {
+      "epoch": 3.1382636655948555,
+      "grad_norm": 4.467242240905762,
+      "learning_rate": 2.0333333333333335e-06,
+      "loss": 0.0348,
+      "num_input_tokens_seen": 4003648,
+      "step": 244
+    },
+    {
+      "epoch": 3.1511254019292605,
+      "grad_norm": 3.3191885948181152,
+      "learning_rate": 2.041666666666667e-06,
+      "loss": 0.0355,
+      "num_input_tokens_seen": 4020384,
+      "step": 245
+    },
+    {
+      "epoch": 3.1639871382636655,
+      "grad_norm": 4.061459064483643,
+      "learning_rate": 2.05e-06,
+      "loss": 0.0272,
+      "num_input_tokens_seen": 4037088,
+      "step": 246
+    },
+    {
+      "epoch": 3.176848874598071,
+      "grad_norm": 3.0776522159576416,
+      "learning_rate": 2.0583333333333337e-06,
+      "loss": 0.0464,
+      "num_input_tokens_seen": 4053472,
+      "step": 247
+    },
+    {
+      "epoch": 3.189710610932476,
+      "grad_norm": 2.13104510307312,
+      "learning_rate": 2.0666666666666666e-06,
+      "loss": 0.0268,
+      "num_input_tokens_seen": 4069248,
+      "step": 248
+    },
+    {
+      "epoch": 3.202572347266881,
+      "grad_norm": 2.1540310382843018,
+      "learning_rate": 2.075e-06,
+      "loss": 0.0229,
+      "num_input_tokens_seen": 4085024,
+      "step": 249
+    },
+    {
+      "epoch": 3.215434083601286,
+      "grad_norm": 3.068967342376709,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0373,
+      "num_input_tokens_seen": 4102432,
+      "step": 250
+    },
+    {
+      "epoch": 3.2282958199356915,
+      "grad_norm": 2.7403666973114014,
+      "learning_rate": 2.091666666666667e-06,
+      "loss": 0.0361,
+      "num_input_tokens_seen": 4118368,
+      "step": 251
+    },
+    {
+      "epoch": 3.2411575562700965,
+      "grad_norm": 4.994419574737549,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0401,
+      "num_input_tokens_seen": 4134784,
+      "step": 252
+    },
+    {
+      "epoch": 3.2540192926045015,
+      "grad_norm": 2.287346839904785,
+      "learning_rate": 2.1083333333333336e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4150528,
+      "step": 253
+    },
+    {
+      "epoch": 3.266881028938907,
+      "grad_norm": 3.649418354034424,
+      "learning_rate": 2.116666666666667e-06,
+      "loss": 0.0481,
+      "num_input_tokens_seen": 4166816,
+      "step": 254
+    },
+    {
+      "epoch": 3.279742765273312,
+      "grad_norm": 2.9041836261749268,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 4182880,
+      "step": 255
+    },
+    {
+      "epoch": 3.292604501607717,
+      "grad_norm": 3.602240800857544,
+      "learning_rate": 2.133333333333334e-06,
+      "loss": 0.0457,
+      "num_input_tokens_seen": 4200128,
+      "step": 256
+    },
+    {
+      "epoch": 3.305466237942122,
+      "grad_norm": 1.6382755041122437,
+      "learning_rate": 2.1416666666666668e-06,
+      "loss": 0.0175,
+      "num_input_tokens_seen": 4215680,
+      "step": 257
+    },
+    {
+      "epoch": 3.3183279742765275,
+      "grad_norm": 2.4267561435699463,
+      "learning_rate": 2.15e-06,
+      "loss": 0.0371,
+      "num_input_tokens_seen": 4232736,
+      "step": 258
+    },
+    {
+      "epoch": 3.3311897106109325,
+      "grad_norm": 2.553440570831299,
+      "learning_rate": 2.1583333333333336e-06,
+      "loss": 0.0268,
+      "num_input_tokens_seen": 4249664,
+      "step": 259
+    },
+    {
+      "epoch": 3.3440514469453375,
+      "grad_norm": 2.951179027557373,
+      "learning_rate": 2.166666666666667e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 4266016,
+      "step": 260
+    },
+    {
+      "epoch": 3.356913183279743,
+      "grad_norm": 2.710747003555298,
+      "learning_rate": 2.1750000000000004e-06,
+      "loss": 0.0264,
+      "num_input_tokens_seen": 4281664,
+      "step": 261
+    },
+    {
+      "epoch": 3.369774919614148,
+      "grad_norm": 3.465902090072632,
+      "learning_rate": 2.1833333333333333e-06,
+      "loss": 0.0336,
+      "num_input_tokens_seen": 4298368,
+      "step": 262
+    },
+    {
+      "epoch": 3.382636655948553,
+      "grad_norm": 4.009627819061279,
+      "learning_rate": 2.191666666666667e-06,
+      "loss": 0.0434,
+      "num_input_tokens_seen": 4314336,
+      "step": 263
+    },
+    {
+      "epoch": 3.395498392282958,
+      "grad_norm": 4.071226596832275,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0389,
+      "num_input_tokens_seen": 4330144,
+      "step": 264
+    },
+    {
+      "epoch": 3.4083601286173635,
+      "grad_norm": 4.341132164001465,
+      "learning_rate": 2.2083333333333335e-06,
+      "loss": 0.045,
+      "num_input_tokens_seen": 4345856,
+      "step": 265
+    },
+    {
+      "epoch": 3.4212218649517685,
+      "grad_norm": 4.663236141204834,
+      "learning_rate": 2.216666666666667e-06,
+      "loss": 0.0331,
+      "num_input_tokens_seen": 4361696,
+      "step": 266
+    },
+    {
+      "epoch": 3.4340836012861735,
+      "grad_norm": 2.17475962638855,
+      "learning_rate": 2.2250000000000003e-06,
+      "loss": 0.0228,
+      "num_input_tokens_seen": 4378304,
+      "step": 267
+    },
+    {
+      "epoch": 3.446945337620579,
+      "grad_norm": 5.099360942840576,
+      "learning_rate": 2.2333333333333333e-06,
+      "loss": 0.0307,
+      "num_input_tokens_seen": 4395232,
+      "step": 268
+    },
+    {
+      "epoch": 3.459807073954984,
+      "grad_norm": 4.4752020835876465,
+      "learning_rate": 2.2416666666666667e-06,
+      "loss": 0.0332,
+      "num_input_tokens_seen": 4411232,
+      "step": 269
+    },
+    {
+      "epoch": 3.472668810289389,
+      "grad_norm": 4.047648906707764,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0662,
+      "num_input_tokens_seen": 4427520,
+      "step": 270
+    },
+    {
+      "epoch": 3.485530546623794,
+      "grad_norm": 5.564074516296387,
+      "learning_rate": 2.2583333333333335e-06,
+      "loss": 0.0431,
+      "num_input_tokens_seen": 4443904,
+      "step": 271
+    },
+    {
+      "epoch": 3.4983922829581995,
+      "grad_norm": 3.9845523834228516,
+      "learning_rate": 2.266666666666667e-06,
+      "loss": 0.0423,
+      "num_input_tokens_seen": 4460608,
+      "step": 272
+    },
+    {
+      "epoch": 3.5112540192926045,
+      "grad_norm": 3.6314871311187744,
+      "learning_rate": 2.2750000000000002e-06,
+      "loss": 0.0447,
+      "num_input_tokens_seen": 4477120,
+      "step": 273
+    },
+    {
+      "epoch": 3.5241157556270095,
+      "grad_norm": 3.1981313228607178,
+      "learning_rate": 2.2833333333333336e-06,
+      "loss": 0.0337,
+      "num_input_tokens_seen": 4493152,
+      "step": 274
+    },
+    {
+      "epoch": 3.536977491961415,
+      "grad_norm": 2.1383423805236816,
+      "learning_rate": 2.2916666666666666e-06,
+      "loss": 0.0319,
+      "num_input_tokens_seen": 4509120,
+      "step": 275
+    },
+    {
+      "epoch": 3.54983922829582,
+      "grad_norm": 2.778956890106201,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 4525984,
+      "step": 276
+    },
+    {
+      "epoch": 3.562700964630225,
+      "grad_norm": 2.518002510070801,
+      "learning_rate": 2.3083333333333334e-06,
+      "loss": 0.0244,
+      "num_input_tokens_seen": 4542816,
+      "step": 277
+    },
+    {
+      "epoch": 3.57556270096463,
+      "grad_norm": 3.8519890308380127,
+      "learning_rate": 2.316666666666667e-06,
+      "loss": 0.0377,
+      "num_input_tokens_seen": 4559872,
+      "step": 278
+    },
+    {
+      "epoch": 3.5884244372990355,
+      "grad_norm": 5.541743278503418,
+      "learning_rate": 2.325e-06,
+      "loss": 0.0477,
+      "num_input_tokens_seen": 4576288,
+      "step": 279
+    },
+    {
+      "epoch": 3.6012861736334405,
+      "grad_norm": 4.099382400512695,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0332,
+      "num_input_tokens_seen": 4591456,
+      "step": 280
+    },
+    {
+      "epoch": 3.6141479099678455,
+      "grad_norm": 2.509037733078003,
+      "learning_rate": 2.341666666666667e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 4608608,
+      "step": 281
+    },
+    {
+      "epoch": 3.627009646302251,
+      "grad_norm": 6.764764308929443,
+      "learning_rate": 2.35e-06,
+      "loss": 0.0468,
+      "num_input_tokens_seen": 4625120,
+      "step": 282
+    },
+    {
+      "epoch": 3.639871382636656,
+      "grad_norm": 6.219926834106445,
+      "learning_rate": 2.3583333333333338e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 4641952,
+      "step": 283
+    },
+    {
+      "epoch": 3.652733118971061,
+      "grad_norm": 3.869555950164795,
+      "learning_rate": 2.3666666666666667e-06,
+      "loss": 0.041,
+      "num_input_tokens_seen": 4657920,
+      "step": 284
+    },
+    {
+      "epoch": 3.665594855305466,
+      "grad_norm": 4.1329264640808105,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0461,
+      "num_input_tokens_seen": 4674688,
+      "step": 285
+    },
+    {
+      "epoch": 3.6784565916398715,
+      "grad_norm": 6.7732648849487305,
+      "learning_rate": 2.3833333333333335e-06,
+      "loss": 0.0594,
+      "num_input_tokens_seen": 4690592,
+      "step": 286
+    },
+    {
+      "epoch": 3.6913183279742765,
+      "grad_norm": 8.550586700439453,
+      "learning_rate": 2.391666666666667e-06,
+      "loss": 0.0728,
+      "num_input_tokens_seen": 4707360,
+      "step": 287
+    },
+    {
+      "epoch": 3.7041800643086815,
+      "grad_norm": 4.587222099304199,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.029,
+      "num_input_tokens_seen": 4724192,
+      "step": 288
+    },
+    {
+      "epoch": 3.717041800643087,
+      "grad_norm": 3.3474302291870117,
+      "learning_rate": 2.4083333333333337e-06,
+      "loss": 0.0412,
+      "num_input_tokens_seen": 4741344,
+      "step": 289
+    },
+    {
+      "epoch": 3.729903536977492,
+      "grad_norm": 4.359026908874512,
+      "learning_rate": 2.4166666666666667e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4756768,
+      "step": 290
+    },
+    {
+      "epoch": 3.742765273311897,
+      "grad_norm": 8.029643058776855,
+      "learning_rate": 2.425e-06,
+      "loss": 0.0796,
+      "num_input_tokens_seen": 4772608,
+      "step": 291
+    },
+    {
+      "epoch": 3.755627009646302,
+      "grad_norm": 3.9188246726989746,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 0.0447,
+      "num_input_tokens_seen": 4788928,
+      "step": 292
+    },
+    {
+      "epoch": 3.7684887459807075,
+      "grad_norm": 2.2623226642608643,
+      "learning_rate": 2.441666666666667e-06,
+      "loss": 0.0252,
+      "num_input_tokens_seen": 4805408,
+      "step": 293
+    },
+    {
+      "epoch": 3.7813504823151125,
+      "grad_norm": 5.41758394241333,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 0.0458,
+      "num_input_tokens_seen": 4821248,
+      "step": 294
+    },
+    {
+      "epoch": 3.7942122186495175,
+      "grad_norm": 3.402024984359741,
+      "learning_rate": 2.4583333333333332e-06,
+      "loss": 0.0431,
+      "num_input_tokens_seen": 4837024,
+      "step": 295
+    },
+    {
+      "epoch": 3.807073954983923,
+      "grad_norm": 2.2912893295288086,
+      "learning_rate": 2.466666666666667e-06,
+      "loss": 0.0422,
+      "num_input_tokens_seen": 4853760,
+      "step": 296
+    },
+    {
+      "epoch": 3.819935691318328,
+      "grad_norm": 2.5446107387542725,
+      "learning_rate": 2.475e-06,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 4870560,
+      "step": 297
+    },
+    {
+      "epoch": 3.832797427652733,
+      "grad_norm": 4.001276969909668,
+      "learning_rate": 2.4833333333333334e-06,
+      "loss": 0.0473,
+      "num_input_tokens_seen": 4887040,
+      "step": 298
+    },
+    {
+      "epoch": 3.845659163987138,
+      "grad_norm": 2.1214165687561035,
+      "learning_rate": 2.491666666666667e-06,
+      "loss": 0.0223,
+      "num_input_tokens_seen": 4903232,
+      "step": 299
+    },
+    {
+      "epoch": 3.8585209003215435,
+      "grad_norm": 2.0410876274108887,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0274,
+      "num_input_tokens_seen": 4919392,
+      "step": 300
+    },
+    {
+      "epoch": 3.8713826366559485,
+      "grad_norm": 4.287325382232666,
+      "learning_rate": 2.5083333333333336e-06,
+      "loss": 0.0454,
+      "num_input_tokens_seen": 4935648,
+      "step": 301
+    },
+    {
+      "epoch": 3.884244372990354,
+      "grad_norm": 4.437230110168457,
+      "learning_rate": 2.5166666666666666e-06,
+      "loss": 0.022,
+      "num_input_tokens_seen": 4952768,
+      "step": 302
+    },
+    {
+      "epoch": 3.897106109324759,
+      "grad_norm": 3.8610730171203613,
+      "learning_rate": 2.5250000000000004e-06,
+      "loss": 0.0357,
+      "num_input_tokens_seen": 4969408,
+      "step": 303
+    },
+    {
+      "epoch": 3.909967845659164,
+      "grad_norm": 2.636066436767578,
+      "learning_rate": 2.5333333333333338e-06,
+      "loss": 0.0458,
+      "num_input_tokens_seen": 4986592,
+      "step": 304
+    },
+    {
+      "epoch": 3.922829581993569,
+      "grad_norm": 4.873857021331787,
+      "learning_rate": 2.5416666666666668e-06,
+      "loss": 0.0454,
+      "num_input_tokens_seen": 5002720,
+      "step": 305
+    },
+    {
+      "epoch": 3.935691318327974,
+      "grad_norm": 2.460498571395874,
+      "learning_rate": 2.55e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 5018848,
+      "step": 306
+    },
+    {
+      "epoch": 3.9485530546623795,
+      "grad_norm": 1.9787613153457642,
+      "learning_rate": 2.558333333333334e-06,
+      "loss": 0.0234,
+      "num_input_tokens_seen": 5034528,
+      "step": 307
+    },
+    {
+      "epoch": 3.9614147909967845,
+      "grad_norm": 2.9524741172790527,
+      "learning_rate": 2.566666666666667e-06,
+      "loss": 0.0366,
+      "num_input_tokens_seen": 5051328,
+      "step": 308
+    },
+    {
+      "epoch": 3.97427652733119,
+      "grad_norm": 2.858250379562378,
+      "learning_rate": 2.5750000000000003e-06,
+      "loss": 0.0236,
+      "num_input_tokens_seen": 5067712,
+      "step": 309
+    },
+    {
+      "epoch": 3.987138263665595,
+      "grad_norm": 2.985665798187256,
+      "learning_rate": 2.5833333333333337e-06,
+      "loss": 0.0403,
+      "num_input_tokens_seen": 5085088,
+      "step": 310
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 2.72078013420105,
+      "learning_rate": 2.5916666666666667e-06,
+      "loss": 0.0395,
+      "num_input_tokens_seen": 5101536,
+      "step": 311
+    },
+    {
+      "epoch": 4.012861736334405,
+      "grad_norm": 1.2907602787017822,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0163,
+      "num_input_tokens_seen": 5117696,
+      "step": 312
+    },
+    {
+      "epoch": 4.02572347266881,
+      "grad_norm": 1.910504937171936,
+      "learning_rate": 2.608333333333333e-06,
+      "loss": 0.0181,
+      "num_input_tokens_seen": 5133408,
+      "step": 313
+    },
+    {
+      "epoch": 4.038585209003215,
+      "grad_norm": 1.1473162174224854,
+      "learning_rate": 2.616666666666667e-06,
+      "loss": 0.013,
+      "num_input_tokens_seen": 5149664,
+      "step": 314
+    },
+    {
+      "epoch": 4.051446945337621,
+      "grad_norm": 2.9885878562927246,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0178,
+      "num_input_tokens_seen": 5165344,
+      "step": 315
+    },
+    {
+      "epoch": 4.064308681672026,
+      "grad_norm": 2.308147430419922,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 0.0153,
+      "num_input_tokens_seen": 5183264,
+      "step": 316
+    },
+    {
+      "epoch": 4.077170418006431,
+      "grad_norm": 3.1830930709838867,
+      "learning_rate": 2.6416666666666666e-06,
+      "loss": 0.0205,
+      "num_input_tokens_seen": 5199296,
+      "step": 317
+    },
+    {
+      "epoch": 4.090032154340836,
+      "grad_norm": 0.674608051776886,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 0.0021,
+      "num_input_tokens_seen": 5216032,
+      "step": 318
+    },
+    {
+      "epoch": 4.102893890675241,
+      "grad_norm": 3.075059175491333,
+      "learning_rate": 2.6583333333333334e-06,
+      "loss": 0.022,
+      "num_input_tokens_seen": 5232544,
+      "step": 319
+    },
+    {
+      "epoch": 4.115755627009646,
+      "grad_norm": 1.1589558124542236,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0134,
+      "num_input_tokens_seen": 5248704,
+      "step": 320
+    },
+    {
+      "epoch": 4.128617363344051,
+      "grad_norm": 4.190515518188477,
+      "learning_rate": 2.6750000000000002e-06,
+      "loss": 0.0175,
+      "num_input_tokens_seen": 5264384,
+      "step": 321
+    },
+    {
+      "epoch": 4.141479099678457,
+      "grad_norm": 3.435070276260376,
+      "learning_rate": 2.683333333333333e-06,
+      "loss": 0.023,
+      "num_input_tokens_seen": 5280928,
+      "step": 322
+    },
+    {
+      "epoch": 4.154340836012862,
+      "grad_norm": 3.3521199226379395,
+      "learning_rate": 2.691666666666667e-06,
+      "loss": 0.0303,
+      "num_input_tokens_seen": 5296512,
+      "step": 323
+    },
+    {
+      "epoch": 4.167202572347267,
+      "grad_norm": 2.0815391540527344,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.0187,
+      "num_input_tokens_seen": 5313440,
+      "step": 324
+    },
+    {
+      "epoch": 4.180064308681672,
+      "grad_norm": 1.968308687210083,
+      "learning_rate": 2.7083333333333334e-06,
+      "loss": 0.0126,
+      "num_input_tokens_seen": 5329088,
+      "step": 325
+    },
+    {
+      "epoch": 4.192926045016077,
+      "grad_norm": 2.134216070175171,
+      "learning_rate": 2.7166666666666668e-06,
+      "loss": 0.0203,
+      "num_input_tokens_seen": 5345376,
+      "step": 326
+    },
+    {
+      "epoch": 4.205787781350482,
+      "grad_norm": 1.2245476245880127,
+      "learning_rate": 2.7250000000000006e-06,
+      "loss": 0.0078,
+      "num_input_tokens_seen": 5361568,
+      "step": 327
+    },
+    {
+      "epoch": 4.218649517684887,
+      "grad_norm": 1.908057451248169,
+      "learning_rate": 2.7333333333333336e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 5377536,
+      "step": 328
+    },
+    {
+      "epoch": 4.231511254019293,
+      "grad_norm": 1.6413862705230713,
+      "learning_rate": 2.741666666666667e-06,
+      "loss": 0.0113,
+      "num_input_tokens_seen": 5395040,
+      "step": 329
+    },
+    {
+      "epoch": 4.244372990353698,
+      "grad_norm": 1.442226529121399,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.0058,
+      "num_input_tokens_seen": 5411488,
+      "step": 330
+    },
+    {
+      "epoch": 4.257234726688103,
+      "grad_norm": 0.9877079129219055,
+      "learning_rate": 2.7583333333333333e-06,
+      "loss": 0.007,
+      "num_input_tokens_seen": 5427328,
+      "step": 331
+    },
+    {
+      "epoch": 4.270096463022508,
+      "grad_norm": 2.9424357414245605,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 5444352,
+      "step": 332
+    },
+    {
+      "epoch": 4.282958199356913,
+      "grad_norm": 3.5407063961029053,
+      "learning_rate": 2.7750000000000005e-06,
+      "loss": 0.0276,
+      "num_input_tokens_seen": 5459840,
+      "step": 333
+    },
+    {
+      "epoch": 4.295819935691318,
+      "grad_norm": 4.147865295410156,
+      "learning_rate": 2.7833333333333335e-06,
+      "loss": 0.0367,
+      "num_input_tokens_seen": 5476000,
+      "step": 334
+    },
+    {
+      "epoch": 4.308681672025724,
+      "grad_norm": 2.4164767265319824,
+      "learning_rate": 2.791666666666667e-06,
+      "loss": 0.0161,
+      "num_input_tokens_seen": 5491968,
+      "step": 335
+    },
+    {
+      "epoch": 4.321543408360129,
+      "grad_norm": 2.553860902786255,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.018,
+      "num_input_tokens_seen": 5508864,
+      "step": 336
+    },
+    {
+      "epoch": 4.334405144694534,
+      "grad_norm": 1.050309419631958,
+      "learning_rate": 2.8083333333333333e-06,
+      "loss": 0.0044,
+      "num_input_tokens_seen": 5525536,
+      "step": 337
+    },
+    {
+      "epoch": 4.347266881028939,
+      "grad_norm": 2.375540018081665,
+      "learning_rate": 2.816666666666667e-06,
+      "loss": 0.0109,
+      "num_input_tokens_seen": 5542368,
+      "step": 338
+    },
+    {
+      "epoch": 4.360128617363344,
+      "grad_norm": 3.041912794113159,
+      "learning_rate": 2.825e-06,
+      "loss": 0.0173,
+      "num_input_tokens_seen": 5558400,
+      "step": 339
+    },
+    {
+      "epoch": 4.372990353697749,
+      "grad_norm": 2.464390754699707,
+      "learning_rate": 2.8333333333333335e-06,
+      "loss": 0.0107,
+      "num_input_tokens_seen": 5575008,
+      "step": 340
+    },
+    {
+      "epoch": 4.385852090032154,
+      "grad_norm": 1.3086371421813965,
+      "learning_rate": 2.841666666666667e-06,
+      "loss": 0.0116,
+      "num_input_tokens_seen": 5590784,
+      "step": 341
+    },
+    {
+      "epoch": 4.39871382636656,
+      "grad_norm": 3.057859420776367,
+      "learning_rate": 2.85e-06,
+      "loss": 0.0281,
+      "num_input_tokens_seen": 5608032,
+      "step": 342
+    },
+    {
+      "epoch": 4.411575562700965,
+      "grad_norm": 2.7371633052825928,
+      "learning_rate": 2.8583333333333336e-06,
+      "loss": 0.0246,
+      "num_input_tokens_seen": 5624704,
+      "step": 343
+    },
+    {
+      "epoch": 4.42443729903537,
+      "grad_norm": 1.7932931184768677,
+      "learning_rate": 2.866666666666667e-06,
+      "loss": 0.0146,
+      "num_input_tokens_seen": 5641056,
+      "step": 344
+    },
+    {
+      "epoch": 4.437299035369775,
+      "grad_norm": 3.9446775913238525,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0439,
+      "num_input_tokens_seen": 5656608,
+      "step": 345
+    },
+    {
+      "epoch": 4.45016077170418,
+      "grad_norm": 3.7706024646759033,
+      "learning_rate": 2.8833333333333334e-06,
+      "loss": 0.0279,
+      "num_input_tokens_seen": 5672928,
+      "step": 346
+    },
+    {
+      "epoch": 4.463022508038585,
+      "grad_norm": 2.38531756401062,
+      "learning_rate": 2.8916666666666672e-06,
+      "loss": 0.0276,
+      "num_input_tokens_seen": 5689728,
+      "step": 347
+    },
+    {
+      "epoch": 4.47588424437299,
+      "grad_norm": 2.534675121307373,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0167,
+      "num_input_tokens_seen": 5705824,
+      "step": 348
+    },
+    {
+      "epoch": 4.488745980707396,
+      "grad_norm": 3.28116774559021,
+      "learning_rate": 2.9083333333333336e-06,
+      "loss": 0.0258,
+      "num_input_tokens_seen": 5723008,
+      "step": 349
+    },
+    {
+      "epoch": 4.501607717041801,
+      "grad_norm": 3.5873191356658936,
+      "learning_rate": 2.916666666666667e-06,
+      "loss": 0.0306,
+      "num_input_tokens_seen": 5739680,
+      "step": 350
+    },
+    {
+      "epoch": 4.514469453376206,
+      "grad_norm": 4.889535427093506,
+      "learning_rate": 2.925e-06,
+      "loss": 0.0395,
+      "num_input_tokens_seen": 5756480,
+      "step": 351
+    },
+    {
+      "epoch": 4.527331189710611,
+      "grad_norm": 1.8129762411117554,
+      "learning_rate": 2.9333333333333338e-06,
+      "loss": 0.0203,
+      "num_input_tokens_seen": 5773088,
+      "step": 352
+    },
+    {
+      "epoch": 4.540192926045016,
+      "grad_norm": 1.5045830011367798,
+      "learning_rate": 2.941666666666667e-06,
+      "loss": 0.0241,
+      "num_input_tokens_seen": 5789312,
+      "step": 353
+    },
+    {
+      "epoch": 4.553054662379421,
+      "grad_norm": 1.4582304954528809,
+      "learning_rate": 2.95e-06,
+      "loss": 0.0151,
+      "num_input_tokens_seen": 5804352,
+      "step": 354
+    },
+    {
+      "epoch": 4.565916398713826,
+      "grad_norm": 1.6875511407852173,
+      "learning_rate": 2.9583333333333335e-06,
+      "loss": 0.0355,
+      "num_input_tokens_seen": 5820384,
+      "step": 355
+    },
+    {
+      "epoch": 4.578778135048232,
+      "grad_norm": 1.0483993291854858,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 0.0093,
+      "num_input_tokens_seen": 5837280,
+      "step": 356
+    },
+    {
+      "epoch": 4.591639871382637,
+      "grad_norm": 2.731086254119873,
+      "learning_rate": 2.9750000000000003e-06,
+      "loss": 0.0314,
+      "num_input_tokens_seen": 5853760,
+      "step": 357
+    },
+    {
+      "epoch": 4.604501607717042,
+      "grad_norm": 1.5985758304595947,
+      "learning_rate": 2.9833333333333337e-06,
+      "loss": 0.0155,
+      "num_input_tokens_seen": 5870240,
+      "step": 358
+    },
+    {
+      "epoch": 4.617363344051447,
+      "grad_norm": 1.3119585514068604,
+      "learning_rate": 2.991666666666667e-06,
+      "loss": 0.0138,
+      "num_input_tokens_seen": 5887296,
+      "step": 359
+    },
+    {
+      "epoch": 4.630225080385852,
+      "grad_norm": 2.3107335567474365,
+      "learning_rate": 3e-06,
+      "loss": 0.0212,
+      "num_input_tokens_seen": 5904384,
+      "step": 360
+    },
+    {
+      "epoch": 4.643086816720257,
+      "grad_norm": 1.8473104238510132,
+      "learning_rate": 3.0083333333333335e-06,
+      "loss": 0.0255,
+      "num_input_tokens_seen": 5920832,
+      "step": 361
+    },
+    {
+      "epoch": 4.655948553054662,
+      "grad_norm": 1.6255360841751099,
+      "learning_rate": 3.0166666666666673e-06,
+      "loss": 0.0231,
+      "num_input_tokens_seen": 5936736,
+      "step": 362
+    },
+    {
+      "epoch": 4.668810289389068,
+      "grad_norm": 1.843116044998169,
+      "learning_rate": 3.0250000000000003e-06,
+      "loss": 0.0083,
+      "num_input_tokens_seen": 5953120,
+      "step": 363
+    },
+    {
+      "epoch": 4.681672025723473,
+      "grad_norm": 1.7974934577941895,
+      "learning_rate": 3.0333333333333337e-06,
+      "loss": 0.0122,
+      "num_input_tokens_seen": 5970336,
+      "step": 364
+    },
+    {
+      "epoch": 4.694533762057878,
+      "grad_norm": 1.799026370048523,
+      "learning_rate": 3.0416666666666666e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 5987488,
+      "step": 365
+    },
+    {
+      "epoch": 4.707395498392283,
+      "grad_norm": 2.38661789894104,
+      "learning_rate": 3.05e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 6003296,
+      "step": 366
+    },
+    {
+      "epoch": 4.720257234726688,
+      "grad_norm": 2.2720234394073486,
+      "learning_rate": 3.058333333333334e-06,
+      "loss": 0.0292,
+      "num_input_tokens_seen": 6020768,
+      "step": 367
+    },
+    {
+      "epoch": 4.733118971061093,
+      "grad_norm": 3.6864068508148193,
+      "learning_rate": 3.066666666666667e-06,
+      "loss": 0.032,
+      "num_input_tokens_seen": 6036992,
+      "step": 368
+    },
+    {
+      "epoch": 4.745980707395498,
+      "grad_norm": 2.1829123497009277,
+      "learning_rate": 3.075e-06,
+      "loss": 0.0189,
+      "num_input_tokens_seen": 6053568,
+      "step": 369
+    },
+    {
+      "epoch": 4.758842443729904,
+      "grad_norm": 2.033625602722168,
+      "learning_rate": 3.0833333333333336e-06,
+      "loss": 0.022,
+      "num_input_tokens_seen": 6070336,
+      "step": 370
+    },
+    {
+      "epoch": 4.771704180064309,
+      "grad_norm": 2.665611982345581,
+      "learning_rate": 3.0916666666666666e-06,
+      "loss": 0.0242,
+      "num_input_tokens_seen": 6087200,
+      "step": 371
+    },
+    {
+      "epoch": 4.784565916398714,
+      "grad_norm": 1.981733798980713,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.015,
+      "num_input_tokens_seen": 6104224,
+      "step": 372
+    },
+    {
+      "epoch": 4.797427652733119,
+      "grad_norm": 1.30072820186615,
+      "learning_rate": 3.1083333333333338e-06,
+      "loss": 0.0154,
+      "num_input_tokens_seen": 6120576,
+      "step": 373
+    },
+    {
+      "epoch": 4.810289389067524,
+      "grad_norm": 2.196270704269409,
+      "learning_rate": 3.1166666666666668e-06,
+      "loss": 0.0195,
+      "num_input_tokens_seen": 6137440,
+      "step": 374
+    },
+    {
+      "epoch": 4.823151125401929,
+      "grad_norm": 2.597658157348633,
+      "learning_rate": 3.125e-06,
+      "loss": 0.031,
+      "num_input_tokens_seen": 6154976,
+      "step": 375
+    },
+    {
+      "epoch": 4.836012861736334,
+      "grad_norm": 3.6647465229034424,
+      "learning_rate": 3.133333333333334e-06,
+      "loss": 0.0376,
+      "num_input_tokens_seen": 6170912,
+      "step": 376
+    },
+    {
+      "epoch": 4.84887459807074,
+      "grad_norm": 1.159162998199463,
+      "learning_rate": 3.141666666666667e-06,
+      "loss": 0.0093,
+      "num_input_tokens_seen": 6187968,
+      "step": 377
+    },
+    {
+      "epoch": 4.861736334405145,
+      "grad_norm": 1.5810602903366089,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 6204320,
+      "step": 378
+    },
+    {
+      "epoch": 4.87459807073955,
+      "grad_norm": 2.0108237266540527,
+      "learning_rate": 3.1583333333333337e-06,
+      "loss": 0.0191,
+      "num_input_tokens_seen": 6220512,
+      "step": 379
+    },
+    {
+      "epoch": 4.887459807073955,
+      "grad_norm": 1.5167394876480103,
+      "learning_rate": 3.1666666666666667e-06,
+      "loss": 0.0132,
+      "num_input_tokens_seen": 6236512,
+      "step": 380
+    },
+    {
+      "epoch": 4.90032154340836,
+      "grad_norm": 2.1819732189178467,
+      "learning_rate": 3.175e-06,
+      "loss": 0.0111,
+      "num_input_tokens_seen": 6252416,
+      "step": 381
+    },
+    {
+      "epoch": 4.913183279742765,
+      "grad_norm": 1.414007306098938,
+      "learning_rate": 3.183333333333334e-06,
+      "loss": 0.0145,
+      "num_input_tokens_seen": 6268992,
+      "step": 382
+    },
+    {
+      "epoch": 4.92604501607717,
+      "grad_norm": 1.6091843843460083,
+      "learning_rate": 3.191666666666667e-06,
+      "loss": 0.0082,
+      "num_input_tokens_seen": 6285312,
+      "step": 383
+    },
+    {
+      "epoch": 4.938906752411576,
+      "grad_norm": 1.0037862062454224,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 6300992,
+      "step": 384
+    },
+    {
+      "epoch": 4.951768488745981,
+      "grad_norm": 2.4763152599334717,
+      "learning_rate": 3.2083333333333337e-06,
+      "loss": 0.0243,
+      "num_input_tokens_seen": 6318560,
+      "step": 385
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 385,
+  "num_input_tokens_seen": 6318560,
+  "num_train_epochs": 5,
+  "save_steps": 980,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5049338265940787e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-385/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76f84697cc5f35a19aa098445f2ef3e5650b3d161fb9f59aeb9c53bd327abfd6
+size 6584

checkpoint-385/zero_to_fp32.py

@@ -0,0 +1,604 @@
+#!/usr/bin/env python
+
+# Copyright (c) Microsoft Corporation.
+# SPDX-License-Identifier: Apache-2.0
+
+# DeepSpeed Team
+
+# This script extracts fp32 consolidated weights from a zero 1, 2 and 3 DeepSpeed checkpoints. It gets
+# copied into the top level checkpoint dir, so the user can easily do the conversion at any point in
+# the future. Once extracted, the weights don't require DeepSpeed and can be used in any
+# application.
+#
+# example: python zero_to_fp32.py . pytorch_model.bin
+
+import argparse
+import torch
+import glob
+import math
+import os
+import re
+from collections import OrderedDict
+from dataclasses import dataclass
+
+# while this script doesn't use deepspeed to recover data, since the checkpoints are pickled with
+# DeepSpeed data structures it has to be available in the current python environment.
+from deepspeed.utils import logger
+from deepspeed.checkpoint.constants import (DS_VERSION, OPTIMIZER_STATE_DICT, SINGLE_PARTITION_OF_FP32_GROUPS,
+                                            FP32_FLAT_GROUPS, ZERO_STAGE, PARTITION_COUNT, PARAM_SHAPES, BUFFER_NAMES,
+                                            FROZEN_PARAM_SHAPES, FROZEN_PARAM_FRAGMENTS)
+
+
+@dataclass
+class zero_model_state:
+    buffers: dict()
+    param_shapes: dict()
+    shared_params: list
+    ds_version: int
+    frozen_param_shapes: dict()
+    frozen_param_fragments: dict()
+
+
+debug = 0
+
+# load to cpu
+device = torch.device('cpu')
+
+
+def atoi(text):
+    return int(text) if text.isdigit() else text
+
+
+def natural_keys(text):
+    '''
+    alist.sort(key=natural_keys) sorts in human order
+    http://nedbatchelder.com/blog/200712/human_sorting.html
+    (See Toothy's implementation in the comments)
+    '''
+    return [atoi(c) for c in re.split(r'(\d+)', text)]
+
+
+def get_model_state_file(checkpoint_dir, zero_stage):
+    if not os.path.isdir(checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{checkpoint_dir}' doesn't exist")
+
+    # there should be only one file
+    if zero_stage <= 2:
+        file = os.path.join(checkpoint_dir, "mp_rank_00_model_states.pt")
+    elif zero_stage == 3:
+        file = os.path.join(checkpoint_dir, "zero_pp_rank_0_mp_rank_00_model_states.pt")
+
+    if not os.path.exists(file):
+        raise FileNotFoundError(f"can't find model states file at '{file}'")
+
+    return file
+
+
+def get_checkpoint_files(checkpoint_dir, glob_pattern):
+    # XXX: need to test that this simple glob rule works for multi-node setup too
+    ckpt_files = sorted(glob.glob(os.path.join(checkpoint_dir, glob_pattern)), key=natural_keys)
+
+    if len(ckpt_files) == 0:
+        raise FileNotFoundError(f"can't find {glob_pattern} files in directory '{checkpoint_dir}'")
+
+    return ckpt_files
+
+
+def get_optim_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_optim_states.pt")
+
+
+def get_model_state_files(checkpoint_dir):
+    return get_checkpoint_files(checkpoint_dir, "*_model_states.pt")
+
+
+def parse_model_states(files):
+    zero_model_states = []
+    for file in files:
+        state_dict = torch.load(file, map_location=device)
+
+        if BUFFER_NAMES not in state_dict:
+            raise ValueError(f"{file} is not a model state checkpoint")
+        buffer_names = state_dict[BUFFER_NAMES]
+        if debug:
+            print("Found buffers:", buffer_names)
+
+        # recover just the buffers while restoring them to fp32 if they were saved in fp16
+        buffers = {k: v.float() for k, v in state_dict["module"].items() if k in buffer_names}
+        param_shapes = state_dict[PARAM_SHAPES]
+
+        # collect parameters that are included in param_shapes
+        param_names = []
+        for s in param_shapes:
+            for name in s.keys():
+                param_names.append(name)
+
+        # update with frozen parameters
+        frozen_param_shapes = state_dict.get(FROZEN_PARAM_SHAPES, None)
+        if frozen_param_shapes is not None:
+            if debug:
+                print(f"Found frozen_param_shapes: {frozen_param_shapes}")
+            param_names += list(frozen_param_shapes.keys())
+
+        # handle shared params
+        shared_params = [[k, v] for k, v in state_dict["shared_params"].items()]
+
+        ds_version = state_dict.get(DS_VERSION, None)
+
+        frozen_param_fragments = state_dict.get(FROZEN_PARAM_FRAGMENTS, None)
+
+        z_model_state = zero_model_state(buffers=buffers,
+                                         param_shapes=param_shapes,
+                                         shared_params=shared_params,
+                                         ds_version=ds_version,
+                                         frozen_param_shapes=frozen_param_shapes,
+                                         frozen_param_fragments=frozen_param_fragments)
+        zero_model_states.append(z_model_state)
+
+    return zero_model_states
+
+
+def parse_optim_states(files, ds_checkpoint_dir):
+
+    total_files = len(files)
+    state_dicts = []
+    for f in files:
+        state_dict = torch.load(f, map_location=device)
+        # immediately discard the potentially huge 2 optimizer states as we only care for fp32 master weights
+        # and also handle the case where it was already removed by another helper script
+        state_dict["optimizer_state_dict"].pop("optimizer_state_dict", None)
+        state_dicts.append(state_dict)
+
+    if not ZERO_STAGE in state_dicts[0][OPTIMIZER_STATE_DICT]:
+        raise ValueError(f"{files[0]} is not a zero checkpoint")
+    zero_stage = state_dicts[0][OPTIMIZER_STATE_DICT][ZERO_STAGE]
+    world_size = state_dicts[0][OPTIMIZER_STATE_DICT][PARTITION_COUNT]
+
+    # For ZeRO-2 each param group can have different partition_count as data parallelism for expert
+    # parameters can be different from data parallelism for non-expert parameters. So we can just
+    # use the max of the partition_count to get the dp world_size.
+
+    if type(world_size) is list:
+        world_size = max(world_size)
+
+    if world_size != total_files:
+        raise ValueError(
+            f"Expected {world_size} of '*_optim_states.pt' under '{ds_checkpoint_dir}' but found {total_files} files. "
+            "Possibly due to an overwrite of an old checkpoint, or a checkpoint didn't get saved by one or more processes."
+        )
+
+    # the groups are named differently in each stage
+    if zero_stage <= 2:
+        fp32_groups_key = SINGLE_PARTITION_OF_FP32_GROUPS
+    elif zero_stage == 3:
+        fp32_groups_key = FP32_FLAT_GROUPS
+    else:
+        raise ValueError(f"unknown zero stage {zero_stage}")
+
+    if zero_stage <= 2:
+        fp32_flat_groups = [state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key] for i in range(len(state_dicts))]
+    elif zero_stage == 3:
+        # if there is more than one param group, there will be multiple flattened tensors - one
+        # flattened tensor per group - for simplicity merge them into a single tensor
+        #
+        # XXX: could make the script more memory efficient for when there are multiple groups - it
+        # will require matching the sub-lists of param_shapes for each param group flattened tensor
+
+        fp32_flat_groups = [
+            torch.cat(state_dicts[i][OPTIMIZER_STATE_DICT][fp32_groups_key], 0) for i in range(len(state_dicts))
+        ]
+
+    return zero_stage, world_size, fp32_flat_groups
+
+
+def _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters):
+    """
+    Returns fp32 state_dict reconstructed from ds checkpoint
+
+    Args:
+        - ``ds_checkpoint_dir``: path to the deepspeed checkpoint folder (where the optimizer files are)
+
+    """
+    print(f"Processing zero checkpoint '{ds_checkpoint_dir}'")
+
+    optim_files = get_optim_files(ds_checkpoint_dir)
+    zero_stage, world_size, fp32_flat_groups = parse_optim_states(optim_files, ds_checkpoint_dir)
+    print(f"Detected checkpoint of type zero stage {zero_stage}, world_size: {world_size}")
+
+    model_files = get_model_state_files(ds_checkpoint_dir)
+
+    zero_model_states = parse_model_states(model_files)
+    print(f'Parsing checkpoint created by deepspeed=={zero_model_states[0].ds_version}')
+
+    if zero_stage <= 2:
+        return _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+    elif zero_stage == 3:
+        return _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                                          exclude_frozen_parameters)
+
+
+def _zero2_merge_frozen_params(state_dict, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+    frozen_param_fragments = zero_model_states[0].frozen_param_fragments
+
+    if debug:
+        num_elem = sum(s.numel() for s in frozen_param_shapes.values())
+        print(f'rank 0: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in frozen_param_fragments.values()])
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        state_dict[name] = frozen_param_fragments[name]
+
+        if debug:
+            print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _has_callable(obj, fn):
+    attr = getattr(obj, fn, None)
+    return callable(attr)
+
+
+def _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+
+    # Reconstruction protocol:
+    #
+    # XXX: document this
+
+    if debug:
+        for i in range(world_size):
+            for j in range(len(fp32_flat_groups[0])):
+                print(f"{FP32_FLAT_GROUPS}[{i}][{j}].shape={fp32_flat_groups[i][j].shape}")
+
+    # XXX: memory usage doubles here (zero2)
+    num_param_groups = len(fp32_flat_groups[0])
+    merged_single_partition_of_fp32_groups = []
+    for i in range(num_param_groups):
+        merged_partitions = [sd[i] for sd in fp32_flat_groups]
+        full_single_fp32_vector = torch.cat(merged_partitions, 0)
+        merged_single_partition_of_fp32_groups.append(full_single_fp32_vector)
+    avail_numel = sum(
+        [full_single_fp32_vector.numel() for full_single_fp32_vector in merged_single_partition_of_fp32_groups])
+
+    if debug:
+        wanted_params = sum([len(shapes) for shapes in param_shapes])
+        wanted_numel = sum([sum(shape.numel() for shape in shapes.values()) for shapes in param_shapes])
+        # not asserting if there is a mismatch due to possible padding
+        print(f"Have {avail_numel} numels to process.")
+        print(f"Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    total_numel = 0
+    total_params = 0
+    for shapes, full_single_fp32_vector in zip(param_shapes, merged_single_partition_of_fp32_groups):
+        offset = 0
+        avail_numel = full_single_fp32_vector.numel()
+        for name, shape in shapes.items():
+
+            unpartitioned_numel = shape.numel() if _has_callable(shape, 'numel') else math.prod(shape)
+            total_numel += unpartitioned_numel
+            total_params += 1
+
+            if debug:
+                print(f"{name} full shape: {shape} unpartitioned numel {unpartitioned_numel} ")
+            state_dict[name] = full_single_fp32_vector.narrow(0, offset, unpartitioned_numel).view(shape)
+            offset += unpartitioned_numel
+
+        # Z2 started to align to 2*world_size to improve nccl performance. Therefore both offset and
+        # avail_numel can differ by anywhere between 0..2*world_size. Due to two unrelated complex
+        # paddings performed in the code it's almost impossible to predict the exact numbers w/o the
+        # live optimizer object, so we are checking that the numbers are within the right range
+        align_to = 2 * world_size
+
+        def zero2_align(x):
+            return align_to * math.ceil(x / align_to)
+
+        if debug:
+            print(f"original offset={offset}, avail_numel={avail_numel}")
+
+        offset = zero2_align(offset)
+        avail_numel = zero2_align(avail_numel)
+
+        if debug:
+            print(f"aligned  offset={offset}, avail_numel={avail_numel}")
+
+        # Sanity check
+        if offset != avail_numel:
+            raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero2_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero2_merge_frozen_params(state_dict, zero_model_states)
+
+    _zero2_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def zero3_partitioned_param_info(unpartitioned_numel, world_size):
+    remainder = unpartitioned_numel % world_size
+    padding_numel = (world_size - remainder) if remainder else 0
+    partitioned_numel = math.ceil(unpartitioned_numel / world_size)
+    return partitioned_numel, padding_numel
+
+
+def _zero3_merge_frozen_params(state_dict, world_size, zero_model_states):
+    if zero_model_states[0].frozen_param_shapes is None or len(zero_model_states[0].frozen_param_shapes) == 0:
+        return
+
+    if debug:
+        for i in range(world_size):
+            num_elem = sum(s.numel() for s in zero_model_states[i].frozen_param_fragments.values())
+            print(f'rank {i}: {FROZEN_PARAM_SHAPES}.numel = {num_elem}')
+
+        frozen_param_shapes = zero_model_states[0].frozen_param_shapes
+        wanted_params = len(frozen_param_shapes)
+        wanted_numel = sum(s.numel() for s in frozen_param_shapes.values())
+        avail_numel = sum([p.numel() for p in zero_model_states[0].frozen_param_fragments.values()]) * world_size
+        print(f'Frozen params: Have {avail_numel} numels to process.')
+        print(f'Frozen params: Need {wanted_numel} numels in {wanted_params} params')
+
+    total_params = 0
+    total_numel = 0
+    for name, shape in zero_model_states[0].frozen_param_shapes.items():
+        total_params += 1
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+
+        param_frags = tuple(model_state.frozen_param_fragments[name] for model_state in zero_model_states)
+        state_dict[name] = torch.cat(param_frags, 0).narrow(0, 0, unpartitioned_numel).view(shape)
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Frozen params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+    print(f"Reconstructed Frozen fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states):
+    param_shapes = zero_model_states[0].param_shapes
+    avail_numel = fp32_flat_groups[0].numel() * world_size
+    # Reconstruction protocol: For zero3 we need to zip the partitions together at boundary of each
+    # param, re-consolidating each param, while dealing with padding if any
+
+    # merge list of dicts, preserving order
+    param_shapes = {k: v for d in param_shapes for k, v in d.items()}
+
+    if debug:
+        for i in range(world_size):
+            print(f"{FP32_FLAT_GROUPS}[{i}].shape={fp32_flat_groups[i].shape}")
+
+        wanted_params = len(param_shapes)
+        wanted_numel = sum(shape.numel() for shape in param_shapes.values())
+        # not asserting if there is a mismatch due to possible padding
+        avail_numel = fp32_flat_groups[0].numel() * world_size
+        print(f"Trainable params: Have {avail_numel} numels to process.")
+        print(f"Trainable params: Need {wanted_numel} numels in {wanted_params} params.")
+
+    # params
+    # XXX: for huge models that can't fit into the host's RAM we will have to recode this to support
+    # out-of-core computing solution
+    offset = 0
+    total_numel = 0
+    total_params = 0
+    for name, shape in param_shapes.items():
+
+        unpartitioned_numel = shape.numel()
+        total_numel += unpartitioned_numel
+        total_params += 1
+
+        partitioned_numel, partitioned_padding_numel = zero3_partitioned_param_info(unpartitioned_numel, world_size)
+
+        if debug:
+            print(
+                f"Trainable params: {total_params} {name} full shape: {shape} partition0 numel={partitioned_numel} partitioned_padding_numel={partitioned_padding_numel}"
+            )
+
+        # XXX: memory usage doubles here
+        state_dict[name] = torch.cat(
+            tuple(fp32_flat_groups[i].narrow(0, offset, partitioned_numel) for i in range(world_size)),
+            0).narrow(0, 0, unpartitioned_numel).view(shape)
+        offset += partitioned_numel
+
+    offset *= world_size
+
+    # Sanity check
+    if offset != avail_numel:
+        raise ValueError(f"consumed {offset} numels out of {avail_numel} - something is wrong")
+
+    print(f"Reconstructed Trainable fp32 state dict with {total_params} params {total_numel} elements")
+
+
+def _get_fp32_state_dict_from_zero3_checkpoint(world_size, fp32_flat_groups, zero_model_states,
+                                               exclude_frozen_parameters):
+    state_dict = OrderedDict()
+
+    # buffers
+    buffers = zero_model_states[0].buffers
+    state_dict.update(buffers)
+    if debug:
+        print(f"added {len(buffers)} buffers")
+
+    if not exclude_frozen_parameters:
+        _zero3_merge_frozen_params(state_dict, world_size, zero_model_states)
+
+    _zero3_merge_trainable_params(state_dict, world_size, fp32_flat_groups, zero_model_states)
+
+    # recover shared parameters
+    for pair in zero_model_states[0].shared_params:
+        if pair[1] in state_dict:
+            state_dict[pair[0]] = state_dict[pair[1]]
+
+    return state_dict
+
+
+def get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated state_dict that can be loaded with
+    ``load_state_dict()`` and used for training without DeepSpeed or shared with others, for example
+    via a model hub.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in 'latest' file. e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+
+    Returns:
+        - pytorch ``state_dict``
+
+    Note: this approach may not work if your application doesn't have sufficient free CPU memory and
+    you may need to use the offline approach using the ``zero_to_fp32.py`` script that is saved with
+    the checkpoint.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import get_fp32_state_dict_from_zero_checkpoint
+        # do the training and checkpoint saving
+        state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir) # already on cpu
+        model = model.cpu() # move to cpu
+        model.load_state_dict(state_dict)
+        # submit to model hub or save the model to share with others
+
+    In this example the ``model`` will no longer be usable in the deepspeed context of the same
+    application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    If you want it all done for you, use ``load_state_dict_from_zero_checkpoint`` instead.
+
+    """
+    if tag is None:
+        latest_path = os.path.join(checkpoint_dir, 'latest')
+        if os.path.isfile(latest_path):
+            with open(latest_path, 'r') as fd:
+                tag = fd.read().strip()
+        else:
+            raise ValueError(f"Unable to find 'latest' file at {latest_path}")
+
+    ds_checkpoint_dir = os.path.join(checkpoint_dir, tag)
+
+    if not os.path.isdir(ds_checkpoint_dir):
+        raise FileNotFoundError(f"Directory '{ds_checkpoint_dir}' doesn't exist")
+
+    return _get_fp32_state_dict_from_zero_checkpoint(ds_checkpoint_dir, exclude_frozen_parameters)
+
+
+def convert_zero_checkpoint_to_fp32_state_dict(checkpoint_dir, output_file, tag=None, exclude_frozen_parameters=False):
+    """
+    Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict`` file that can be
+    loaded with ``torch.load(file)`` + ``load_state_dict()`` and used for training without DeepSpeed.
+
+    Args:
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``output_file``: path to the pytorch fp32 state_dict output file (e.g. path/pytorch_model.bin)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+        - ``exclude_frozen_parameters``: exclude frozen parameters
+    """
+
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag, exclude_frozen_parameters)
+    print(f"Saving fp32 state dict to {output_file}")
+    torch.save(state_dict, output_file)
+
+
+def load_state_dict_from_zero_checkpoint(model, checkpoint_dir, tag=None):
+    """
+    1. Put the provided model to cpu
+    2. Convert ZeRO 2 or 3 checkpoint into a single fp32 consolidated ``state_dict``
+    3. Load it into the provided model
+
+    Args:
+        - ``model``: the model object to update
+        - ``checkpoint_dir``: path to the desired checkpoint folder. (one that contains the tag-folder, like ``global_step14``)
+        - ``tag``: checkpoint tag used as a unique identifier for checkpoint. If not provided will attempt to load tag in the file named ``latest`` in the checkpoint folder, e.g., ``global_step14``
+
+    Returns:
+        - ``model`: modified model
+
+    Make sure you have plenty of CPU memory available before you call this function. If you don't
+    have enough use the ``zero_to_fp32.py`` utility to do the conversion. You will find it
+    conveniently placed for you in the checkpoint folder.
+
+    A typical usage might be ::
+
+        from deepspeed.utils.zero_to_fp32 import load_state_dict_from_zero_checkpoint
+        model = load_state_dict_from_zero_checkpoint(trainer.model, checkpoint_dir)
+        # submit to model hub or save the model to share with others
+
+    Note, that once this was run, the ``model`` will no longer be usable in the deepspeed context
+    of the same application. i.e. you will need to re-initialize the deepspeed engine, since
+    ``model.load_state_dict(state_dict)`` will remove all the deepspeed magic from it.
+
+    """
+    logger.info(f"Extracting fp32 weights")
+    state_dict = get_fp32_state_dict_from_zero_checkpoint(checkpoint_dir, tag)
+
+    logger.info(f"Overwriting model with fp32 weights")
+    model = model.cpu()
+    model.load_state_dict(state_dict, strict=False)
+
+    return model
+
+
+if __name__ == "__main__":
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("checkpoint_dir",
+                        type=str,
+                        help="path to the desired checkpoint folder, e.g., path/checkpoint-12")
+    parser.add_argument(
+        "output_file",
+        type=str,
+        help="path to the pytorch fp32 state_dict output file (e.g. path/checkpoint-12/pytorch_model.bin)")
+    parser.add_argument("-t",
+                        "--tag",
+                        type=str,
+                        default=None,
+                        help="checkpoint tag used as a unique identifier for checkpoint. e.g., global_step1")
+    parser.add_argument("--exclude_frozen_parameters", action='store_true', help="exclude frozen parameters")
+    parser.add_argument("-d", "--debug", action='store_true', help="enable debug")
+    args = parser.parse_args()
+
+    debug = args.debug
+
+    convert_zero_checkpoint_to_fp32_state_dict(args.checkpoint_dir,
+                                               args.output_file,
+                                               tag=args.tag,
+                                               exclude_frozen_parameters=args.exclude_frozen_parameters)

config.json

@@ -0,0 +1,29 @@
+{
+  "_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.42.3",
+  "use_cache": false,
+  "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,10 @@
+{
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9,
+  "transformers_version": "4.42.3"
+}

llamaboard_config.yaml

@@ -0,0 +1,65 @@
+top.booster: auto
+top.checkpoint_path: null
+top.finetuning_type: full
+top.model_name: LLaMA2-7B-Chat
+top.quantization_bit: none
+top.quantization_method: bitsandbytes
+top.rope_scaling: none
+top.template: llama2
+top.visual_inputs: false
+train.additional_target: ''
+train.badam_mode: layer
+train.badam_switch_interval: 50
+train.badam_switch_mode: ascending
+train.badam_update_ratio: 0.05
+train.batch_size: 4
+train.compute_type: bf16
+train.create_new_adapter: false
+train.cutoff_len: 1024
+train.dataset:
+- truth_train
+train.dataset_dir: data
+train.ds_offload: false
+train.ds_stage: '2'
+train.freeze_extra_modules: ''
+train.freeze_trainable_layers: 2
+train.freeze_trainable_modules: all
+train.galore_rank: 16
+train.galore_scale: 0.25
+train.galore_target: all
+train.galore_update_interval: 200
+train.gradient_accumulation_steps: 8
+train.learning_rate: 5e-6
+train.logging_steps: 1
+train.lora_alpha: 16
+train.lora_dropout: 0
+train.lora_rank: 8
+train.lora_target: ''
+train.loraplus_lr_ratio: 0
+train.lr_scheduler_type: cosine
+train.max_grad_norm: '1.0'
+train.max_samples: '100000'
+train.neat_packing: false
+train.neftune_alpha: 0
+train.num_train_epochs: '5.0'
+train.optim: adamw_torch
+train.packing: false
+train.ppo_score_norm: false
+train.ppo_whiten_rewards: false
+train.pref_beta: 0.1
+train.pref_ftx: 0
+train.pref_loss: sigmoid
+train.report_to: false
+train.resize_vocab: false
+train.reward_model: null
+train.save_steps: 980
+train.shift_attn: false
+train.training_stage: Supervised Fine-Tuning
+train.use_badam: false
+train.use_dora: false
+train.use_galore: false
+train.use_llama_pro: false
+train.use_pissa: false
+train.use_rslora: false
+train.val_size: 0
+train.warmup_steps: 600

model-00001-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:05797c94c439de88c420a567e2bde6c29dad762705ba7588731295691e8b1173
+size 4938985352

model-00002-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0eb6354f514ef3bd5ba958aca5d01b62d4199cd3baf18bd14d43954c90217bd5
+size 4947390880

model-00003-of-00003.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b03f1693b214e4e07203a5b47bb63f40ed75486d4b48e985781c1da8733a4c39
+size 3590488816

model.safetensors.index.json

@@ -0,0 +1,298 @@
+{
+  "metadata": {
+    "total_size": 13476831232
+  },
+  "weight_map": {
+    "lm_head.weight": "model-00003-of-00003.safetensors",
+    "model.embed_tokens.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.0.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.1.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.10.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.11.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.11.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.12.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.12.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.13.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.14.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.15.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.16.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.17.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.18.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.19.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.2.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.2.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.20.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.20.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.21.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.input_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.down_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.post_attention_layernorm.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.22.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.mlp.gate_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.mlp.up_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.23.self_attn.k_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.o_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.q_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.23.self_attn.v_proj.weight": "model-00002-of-00003.safetensors",
+    "model.layers.24.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.24.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.25.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.26.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.27.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.28.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.29.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.3.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.3.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.30.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.30.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.input_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.down_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.gate_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.mlp.up_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.post_attention_layernorm.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.k_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.o_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.q_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.31.self_attn.v_proj.weight": "model-00003-of-00003.safetensors",
+    "model.layers.4.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.4.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.5.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.6.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.7.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.8.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.input_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.down_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.gate_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.mlp.up_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.post_attention_layernorm.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.k_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.o_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.q_proj.weight": "model-00001-of-00003.safetensors",
+    "model.layers.9.self_attn.v_proj.weight": "model-00001-of-00003.safetensors",
+    "model.norm.weight": "model-00003-of-00003.safetensors"
+  }
+}

running_log.txt

@@ -0,0 +1,1025 @@
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 2, device: cuda:2, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 4, device: cuda:4, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 7, device: cuda:7, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 5, device: cuda:5, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 6, device: cuda:6, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 1, device: cuda:1, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+07/11/2024 09:33:21 - INFO - llamafactory.hparams.parser - Process rank: 3, device: cuda:3, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+[INFO|parser.py:325] 2024-07-11 09:33:21,701 >> Process rank: 0, device: cuda:0, n_gpu: 1, distributed training: True, compute dtype: torch.bfloat16
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 09:33:24,201 >> loading file tokenizer.model from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/tokenizer.model
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 09:33:24,201 >> loading file tokenizer.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/tokenizer.json
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 09:33:24,201 >> loading file added_tokens.json from cache at None
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 09:33:24,201 >> loading file special_tokens_map.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/special_tokens_map.json
+
+[INFO|tokenization_utils_base.py:2161] 2024-07-11 09:33:24,202 >> loading file tokenizer_config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/tokenizer_config.json
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+[INFO|template.py:372] 2024-07-11 09:33:24,309 >> Add pad token: </s>
+
+[INFO|loader.py:50] 2024-07-11 09:33:24,310 >> Loading dataset train_output.json...
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+07/11/2024 09:33:24 - INFO - llamafactory.data.template - Add pad token: </s>
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+07/11/2024 09:33:25 - INFO - llamafactory.data.loader - Loading dataset train_output.json...
+
+[INFO|configuration_utils.py:733] 2024-07-11 09:33:27,545 >> loading configuration file config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/config.json
+
+[INFO|configuration_utils.py:800] 2024-07-11 09:33:27,546 >> Model config LlamaConfig {
+  "_name_or_path": "meta-llama/Llama-2-7b-chat-hf",
+  "architectures": [
+    "LlamaForCausalLM"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "hidden_act": "silu",
+  "hidden_size": 4096,
+  "initializer_range": 0.02,
+  "intermediate_size": 11008,
+  "max_position_embeddings": 4096,
+  "mlp_bias": false,
+  "model_type": "llama",
+  "num_attention_heads": 32,
+  "num_hidden_layers": 32,
+  "num_key_value_heads": 32,
+  "pretraining_tp": 1,
+  "rms_norm_eps": 1e-05,
+  "rope_scaling": null,
+  "rope_theta": 10000.0,
+  "tie_word_embeddings": false,
+  "torch_dtype": "float16",
+  "transformers_version": "4.42.3",
+  "use_cache": true,
+  "vocab_size": 32000
+}
+
+
+[INFO|modeling_utils.py:3556] 2024-07-11 09:33:28,355 >> loading weights file model.safetensors from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/model.safetensors.index.json
+
+[INFO|modeling_utils.py:1531] 2024-07-11 09:35:30,794 >> Instantiating LlamaForCausalLM model under default dtype torch.bfloat16.
+
+[INFO|configuration_utils.py:1000] 2024-07-11 09:35:30,799 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "eos_token_id": 2
+}
+
+
+[INFO|modeling_utils.py:4364] 2024-07-11 09:35:47,952 >> All model checkpoint weights were used when initializing LlamaForCausalLM.
+
+
+[INFO|modeling_utils.py:4372] 2024-07-11 09:35:47,952 >> All the weights of LlamaForCausalLM were initialized from the model checkpoint at meta-llama/Llama-2-7b-chat-hf.
+If your task is similar to the task the model of the checkpoint was trained on, you can already use LlamaForCausalLM for predictions without further training.
+
+[INFO|configuration_utils.py:955] 2024-07-11 09:35:48,301 >> loading configuration file generation_config.json from cache at /root/.cache/huggingface/hub/models--meta-llama--Llama-2-7b-chat-hf/snapshots/f5db02db724555f92da89c216ac04704f23d4590/generation_config.json
+
+[INFO|configuration_utils.py:1000] 2024-07-11 09:35:48,301 >> Generate config GenerationConfig {
+  "bos_token_id": 1,
+  "do_sample": true,
+  "eos_token_id": 2,
+  "max_length": 4096,
+  "pad_token_id": 0,
+  "temperature": 0.6,
+  "top_p": 0.9
+}
+
+
+[INFO|checkpointing.py:103] 2024-07-11 09:35:48,308 >> Gradient checkpointing enabled.
+
+[INFO|attention.py:80] 2024-07-11 09:35:48,308 >> Using torch SDPA for faster training and inference.
+
+[INFO|adapter.py:302] 2024-07-11 09:35:48,309 >> Upcasting trainable params to float32.
+
+[INFO|adapter.py:48] 2024-07-11 09:35:48,309 >> Fine-tuning method: Full
+
+[INFO|loader.py:196] 2024-07-11 09:35:48,360 >> trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+[INFO|trainer.py:642] 2024-07-11 09:35:48,366 >> Using auto half precision backend
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.checkpointing - Gradient checkpointing enabled.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.model_utils.attention - Using torch SDPA for faster training and inference.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Upcasting trainable params to float32.
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.adapter - Fine-tuning method: Full
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+07/11/2024 09:35:49 - INFO - llamafactory.model.loader - trainable params: 6,738,415,616 || all params: 6,738,415,616 || trainable%: 100.0000
+
+[INFO|trainer.py:2128] 2024-07-11 09:36:08,514 >> ***** Running training *****
+
+[INFO|trainer.py:2129] 2024-07-11 09:36:08,514 >>   Num examples = 19,880
+
+[INFO|trainer.py:2130] 2024-07-11 09:36:08,514 >>   Num Epochs = 5
+
+[INFO|trainer.py:2131] 2024-07-11 09:36:08,514 >>   Instantaneous batch size per device = 4
+
+[INFO|trainer.py:2134] 2024-07-11 09:36:08,514 >>   Total train batch size (w. parallel, distributed & accumulation) = 256
+
+[INFO|trainer.py:2135] 2024-07-11 09:36:08,514 >>   Gradient Accumulation steps = 8
+
+[INFO|trainer.py:2136] 2024-07-11 09:36:08,514 >>   Total optimization steps = 385
+
+[INFO|trainer.py:2137] 2024-07-11 09:36:08,516 >>   Number of trainable parameters = 6,738,415,616
+
+[INFO|callbacks.py:310] 2024-07-11 09:36:20,236 >> {'loss': 8.1083, 'learning_rate': 8.3333e-09, 'epoch': 0.01, 'throughput': 1411.72}
+
+[INFO|callbacks.py:310] 2024-07-11 09:36:31,361 >> {'loss': 8.1052, 'learning_rate': 1.6667e-08, 'epoch': 0.03, 'throughput': 1455.41}
+
+[INFO|callbacks.py:310] 2024-07-11 09:36:42,439 >> {'loss': 8.1366, 'learning_rate': 2.5000e-08, 'epoch': 0.04, 'throughput': 1466.88}
+
+[INFO|callbacks.py:310] 2024-07-11 09:36:53,528 >> {'loss': 8.0413, 'learning_rate': 3.3333e-08, 'epoch': 0.05, 'throughput': 1462.41}
+
+[INFO|callbacks.py:310] 2024-07-11 09:37:04,628 >> {'loss': 8.1055, 'learning_rate': 4.1667e-08, 'epoch': 0.06, 'throughput': 1471.37}
+
+[INFO|callbacks.py:310] 2024-07-11 09:37:15,764 >> {'loss': 7.9666, 'learning_rate': 5.0000e-08, 'epoch': 0.08, 'throughput': 1484.19}
+
+[INFO|callbacks.py:310] 2024-07-11 09:37:26,881 >> {'loss': 8.0182, 'learning_rate': 5.8333e-08, 'epoch': 0.09, 'throughput': 1477.83}
+
+[INFO|callbacks.py:310] 2024-07-11 09:37:38,008 >> {'loss': 8.0363, 'learning_rate': 6.6667e-08, 'epoch': 0.10, 'throughput': 1468.21}
+
+[INFO|callbacks.py:310] 2024-07-11 09:37:49,153 >> {'loss': 8.0918, 'learning_rate': 7.5000e-08, 'epoch': 0.12, 'throughput': 1469.05}
+
+[INFO|callbacks.py:310] 2024-07-11 09:38:00,281 >> {'loss': 8.0197, 'learning_rate': 8.3333e-08, 'epoch': 0.13, 'throughput': 1467.95}
+
+[INFO|callbacks.py:310] 2024-07-11 09:38:11,405 >> {'loss': 7.9714, 'learning_rate': 9.1667e-08, 'epoch': 0.14, 'throughput': 1470.48}
+
+[INFO|callbacks.py:310] 2024-07-11 09:38:22,461 >> {'loss': 8.0131, 'learning_rate': 1.0000e-07, 'epoch': 0.15, 'throughput': 1463.77}
+
+[INFO|callbacks.py:310] 2024-07-11 09:38:33,540 >> {'loss': 7.9796, 'learning_rate': 1.0833e-07, 'epoch': 0.17, 'throughput': 1457.86}
+
+[INFO|callbacks.py:310] 2024-07-11 09:38:44,657 >> {'loss': 7.9999, 'learning_rate': 1.1667e-07, 'epoch': 0.18, 'throughput': 1460.84}
+
+[INFO|callbacks.py:310] 2024-07-11 09:38:55,796 >> {'loss': 8.0168, 'learning_rate': 1.2500e-07, 'epoch': 0.19, 'throughput': 1461.12}
+
+[INFO|callbacks.py:310] 2024-07-11 09:39:06,924 >> {'loss': 7.8423, 'learning_rate': 1.3333e-07, 'epoch': 0.21, 'throughput': 1463.26}
+
+[INFO|callbacks.py:310] 2024-07-11 09:39:18,061 >> {'loss': 7.9208, 'learning_rate': 1.4167e-07, 'epoch': 0.22, 'throughput': 1461.36}
+
+[INFO|callbacks.py:310] 2024-07-11 09:39:29,196 >> {'loss': 7.6898, 'learning_rate': 1.5000e-07, 'epoch': 0.23, 'throughput': 1462.71}
+
+[INFO|callbacks.py:310] 2024-07-11 09:39:40,310 >> {'loss': 7.9285, 'learning_rate': 1.5833e-07, 'epoch': 0.24, 'throughput': 1459.08}
+
+[INFO|callbacks.py:310] 2024-07-11 09:39:51,418 >> {'loss': 7.8924, 'learning_rate': 1.6667e-07, 'epoch': 0.26, 'throughput': 1458.15}
+
+[INFO|callbacks.py:310] 2024-07-11 09:40:02,525 >> {'loss': 7.6173, 'learning_rate': 1.7500e-07, 'epoch': 0.27, 'throughput': 1460.88}
+
+[INFO|callbacks.py:310] 2024-07-11 09:40:13,636 >> {'loss': 7.7661, 'learning_rate': 1.8333e-07, 'epoch': 0.28, 'throughput': 1463.97}
+
+[INFO|callbacks.py:310] 2024-07-11 09:40:24,739 >> {'loss': 7.2936, 'learning_rate': 1.9167e-07, 'epoch': 0.30, 'throughput': 1464.98}
+
+[INFO|callbacks.py:310] 2024-07-11 09:40:35,863 >> {'loss': 7.3044, 'learning_rate': 2.0000e-07, 'epoch': 0.31, 'throughput': 1468.54}
+
+[INFO|callbacks.py:310] 2024-07-11 09:40:46,970 >> {'loss': 7.3500, 'learning_rate': 2.0833e-07, 'epoch': 0.32, 'throughput': 1468.11}
+
+[INFO|callbacks.py:310] 2024-07-11 09:40:58,094 >> {'loss': 7.1213, 'learning_rate': 2.1667e-07, 'epoch': 0.33, 'throughput': 1469.84}
+
+[INFO|callbacks.py:310] 2024-07-11 09:41:09,205 >> {'loss': 7.1363, 'learning_rate': 2.2500e-07, 'epoch': 0.35, 'throughput': 1469.80}
+
+[INFO|callbacks.py:310] 2024-07-11 09:41:20,376 >> {'loss': 6.9016, 'learning_rate': 2.3333e-07, 'epoch': 0.36, 'throughput': 1468.87}
+
+[INFO|callbacks.py:310] 2024-07-11 09:41:31,471 >> {'loss': 6.9764, 'learning_rate': 2.4167e-07, 'epoch': 0.37, 'throughput': 1467.16}
+
+[INFO|callbacks.py:310] 2024-07-11 09:41:42,574 >> {'loss': 6.9005, 'learning_rate': 2.5000e-07, 'epoch': 0.39, 'throughput': 1467.82}
+
+[INFO|callbacks.py:310] 2024-07-11 09:41:53,697 >> {'loss': 6.9852, 'learning_rate': 2.5833e-07, 'epoch': 0.40, 'throughput': 1468.73}
+
+[INFO|callbacks.py:310] 2024-07-11 09:42:04,833 >> {'loss': 5.8340, 'learning_rate': 2.6667e-07, 'epoch': 0.41, 'throughput': 1469.26}
+
+[INFO|callbacks.py:310] 2024-07-11 09:42:15,965 >> {'loss': 5.5452, 'learning_rate': 2.7500e-07, 'epoch': 0.42, 'throughput': 1471.43}
+
+[INFO|callbacks.py:310] 2024-07-11 09:42:27,099 >> {'loss': 5.5189, 'learning_rate': 2.8333e-07, 'epoch': 0.44, 'throughput': 1473.12}
+
+[INFO|callbacks.py:310] 2024-07-11 09:42:38,222 >> {'loss': 5.4654, 'learning_rate': 2.9167e-07, 'epoch': 0.45, 'throughput': 1473.94}
+
+[INFO|callbacks.py:310] 2024-07-11 09:42:49,345 >> {'loss': 5.3117, 'learning_rate': 3.0000e-07, 'epoch': 0.46, 'throughput': 1472.23}
+
+[INFO|callbacks.py:310] 2024-07-11 09:43:00,464 >> {'loss': 5.2744, 'learning_rate': 3.0833e-07, 'epoch': 0.48, 'throughput': 1470.87}
+
+[INFO|callbacks.py:310] 2024-07-11 09:43:11,566 >> {'loss': 5.2821, 'learning_rate': 3.1667e-07, 'epoch': 0.49, 'throughput': 1470.77}
+
+[INFO|callbacks.py:310] 2024-07-11 09:43:22,656 >> {'loss': 5.0967, 'learning_rate': 3.2500e-07, 'epoch': 0.50, 'throughput': 1470.64}
+
+[INFO|callbacks.py:310] 2024-07-11 09:43:33,736 >> {'loss': 5.2167, 'learning_rate': 3.3333e-07, 'epoch': 0.51, 'throughput': 1470.41}
+
+[INFO|callbacks.py:310] 2024-07-11 09:43:44,884 >> {'loss': 5.1517, 'learning_rate': 3.4167e-07, 'epoch': 0.53, 'throughput': 1471.24}
+
+[INFO|callbacks.py:310] 2024-07-11 09:43:56,018 >> {'loss': 4.6959, 'learning_rate': 3.5000e-07, 'epoch': 0.54, 'throughput': 1471.59}
+
+[INFO|callbacks.py:310] 2024-07-11 09:44:07,144 >> {'loss': 4.1458, 'learning_rate': 3.5833e-07, 'epoch': 0.55, 'throughput': 1471.34}
+
+[INFO|callbacks.py:310] 2024-07-11 09:44:18,276 >> {'loss': 3.6692, 'learning_rate': 3.6667e-07, 'epoch': 0.57, 'throughput': 1471.55}
+
+[INFO|callbacks.py:310] 2024-07-11 09:44:29,413 >> {'loss': 3.3322, 'learning_rate': 3.7500e-07, 'epoch': 0.58, 'throughput': 1472.18}
+
+[INFO|callbacks.py:310] 2024-07-11 09:44:40,545 >> {'loss': 3.2857, 'learning_rate': 3.8333e-07, 'epoch': 0.59, 'throughput': 1472.04}
+
+[INFO|callbacks.py:310] 2024-07-11 09:44:51,649 >> {'loss': 3.0806, 'learning_rate': 3.9167e-07, 'epoch': 0.60, 'throughput': 1472.06}
+
+[INFO|callbacks.py:310] 2024-07-11 09:45:02,778 >> {'loss': 2.8239, 'learning_rate': 4.0000e-07, 'epoch': 0.62, 'throughput': 1472.90}
+
+[INFO|callbacks.py:310] 2024-07-11 09:45:13,907 >> {'loss': 2.8695, 'learning_rate': 4.0833e-07, 'epoch': 0.63, 'throughput': 1474.53}
+
+[INFO|callbacks.py:310] 2024-07-11 09:45:25,026 >> {'loss': 2.6218, 'learning_rate': 4.1667e-07, 'epoch': 0.64, 'throughput': 1475.14}
+
+[INFO|callbacks.py:310] 2024-07-11 09:45:36,130 >> {'loss': 2.3993, 'learning_rate': 4.2500e-07, 'epoch': 0.66, 'throughput': 1474.86}
+
+[INFO|callbacks.py:310] 2024-07-11 09:45:47,281 >> {'loss': 2.2828, 'learning_rate': 4.3333e-07, 'epoch': 0.67, 'throughput': 1474.81}
+
+[INFO|callbacks.py:310] 2024-07-11 09:45:58,412 >> {'loss': 2.2063, 'learning_rate': 4.4167e-07, 'epoch': 0.68, 'throughput': 1475.84}
+
+[INFO|callbacks.py:310] 2024-07-11 09:46:09,535 >> {'loss': 1.8878, 'learning_rate': 4.5000e-07, 'epoch': 0.69, 'throughput': 1474.30}
+
+[INFO|callbacks.py:310] 2024-07-11 09:46:20,652 >> {'loss': 1.5971, 'learning_rate': 4.5833e-07, 'epoch': 0.71, 'throughput': 1475.65}
+
+[INFO|callbacks.py:310] 2024-07-11 09:46:31,747 >> {'loss': 0.9604, 'learning_rate': 4.6667e-07, 'epoch': 0.72, 'throughput': 1475.05}
+
+[INFO|callbacks.py:310] 2024-07-11 09:46:42,858 >> {'loss': 0.6287, 'learning_rate': 4.7500e-07, 'epoch': 0.73, 'throughput': 1475.70}
+
+[INFO|callbacks.py:310] 2024-07-11 09:46:53,999 >> {'loss': 0.4844, 'learning_rate': 4.8333e-07, 'epoch': 0.75, 'throughput': 1476.45}
+
+[INFO|callbacks.py:310] 2024-07-11 09:47:05,114 >> {'loss': 0.3891, 'learning_rate': 4.9167e-07, 'epoch': 0.76, 'throughput': 1475.83}
+
+[INFO|callbacks.py:310] 2024-07-11 09:47:16,243 >> {'loss': 0.3570, 'learning_rate': 5.0000e-07, 'epoch': 0.77, 'throughput': 1476.05}
+
+[INFO|callbacks.py:310] 2024-07-11 09:47:27,365 >> {'loss': 0.2854, 'learning_rate': 5.0833e-07, 'epoch': 0.78, 'throughput': 1476.53}
+
+[INFO|callbacks.py:310] 2024-07-11 09:47:38,471 >> {'loss': 0.2403, 'learning_rate': 5.1667e-07, 'epoch': 0.80, 'throughput': 1476.18}
+
+[INFO|callbacks.py:310] 2024-07-11 09:47:49,613 >> {'loss': 0.2522, 'learning_rate': 5.2500e-07, 'epoch': 0.81, 'throughput': 1477.05}
+
+[INFO|callbacks.py:310] 2024-07-11 09:48:00,728 >> {'loss': 0.2339, 'learning_rate': 5.3333e-07, 'epoch': 0.82, 'throughput': 1476.51}
+
+[INFO|callbacks.py:310] 2024-07-11 09:48:11,827 >> {'loss': 0.2214, 'learning_rate': 5.4167e-07, 'epoch': 0.84, 'throughput': 1477.43}
+
+[INFO|callbacks.py:310] 2024-07-11 09:48:22,937 >> {'loss': 0.1912, 'learning_rate': 5.5000e-07, 'epoch': 0.85, 'throughput': 1478.13}
+
+[INFO|callbacks.py:310] 2024-07-11 09:48:34,066 >> {'loss': 0.2157, 'learning_rate': 5.5833e-07, 'epoch': 0.86, 'throughput': 1479.16}
+
+[INFO|callbacks.py:310] 2024-07-11 09:48:45,169 >> {'loss': 0.2045, 'learning_rate': 5.6667e-07, 'epoch': 0.87, 'throughput': 1478.13}
+
+[INFO|callbacks.py:310] 2024-07-11 09:48:56,325 >> {'loss': 0.1964, 'learning_rate': 5.7500e-07, 'epoch': 0.89, 'throughput': 1478.54}
+
+[INFO|callbacks.py:310] 2024-07-11 09:49:07,421 >> {'loss': 0.1901, 'learning_rate': 5.8333e-07, 'epoch': 0.90, 'throughput': 1477.07}
+
+[INFO|callbacks.py:310] 2024-07-11 09:49:18,532 >> {'loss': 0.1891, 'learning_rate': 5.9167e-07, 'epoch': 0.91, 'throughput': 1477.00}
+
+[INFO|callbacks.py:310] 2024-07-11 09:49:29,697 >> {'loss': 0.1841, 'learning_rate': 6.0000e-07, 'epoch': 0.93, 'throughput': 1477.26}
+
+[INFO|callbacks.py:310] 2024-07-11 09:49:40,800 >> {'loss': 0.1803, 'learning_rate': 6.0833e-07, 'epoch': 0.94, 'throughput': 1477.28}
+
+[INFO|callbacks.py:310] 2024-07-11 09:49:51,887 >> {'loss': 0.1657, 'learning_rate': 6.1667e-07, 'epoch': 0.95, 'throughput': 1477.17}
+
+[INFO|callbacks.py:310] 2024-07-11 09:50:02,989 >> {'loss': 0.2157, 'learning_rate': 6.2500e-07, 'epoch': 0.96, 'throughput': 1477.26}
+
+[INFO|callbacks.py:310] 2024-07-11 09:50:14,088 >> {'loss': 0.2000, 'learning_rate': 6.3333e-07, 'epoch': 0.98, 'throughput': 1477.40}
+
+[INFO|callbacks.py:310] 2024-07-11 09:50:25,168 >> {'loss': 0.1519, 'learning_rate': 6.4167e-07, 'epoch': 0.99, 'throughput': 1476.86}
+
+[INFO|callbacks.py:310] 2024-07-11 09:50:36,302 >> {'loss': 0.1765, 'learning_rate': 6.5000e-07, 'epoch': 1.00, 'throughput': 1477.71}
+
+[INFO|callbacks.py:310] 2024-07-11 09:50:47,461 >> {'loss': 0.1907, 'learning_rate': 6.5833e-07, 'epoch': 1.02, 'throughput': 1478.10}
+
+[INFO|callbacks.py:310] 2024-07-11 09:50:58,581 >> {'loss': 0.1593, 'learning_rate': 6.6667e-07, 'epoch': 1.03, 'throughput': 1477.21}
+
+[INFO|callbacks.py:310] 2024-07-11 09:51:09,696 >> {'loss': 0.1659, 'learning_rate': 6.7500e-07, 'epoch': 1.04, 'throughput': 1477.14}
+
+[INFO|callbacks.py:310] 2024-07-11 09:51:20,799 >> {'loss': 0.1619, 'learning_rate': 6.8333e-07, 'epoch': 1.05, 'throughput': 1477.58}
+
+[INFO|callbacks.py:310] 2024-07-11 09:51:31,873 >> {'loss': 0.1680, 'learning_rate': 6.9167e-07, 'epoch': 1.07, 'throughput': 1477.08}
+
+[INFO|callbacks.py:310] 2024-07-11 09:51:43,028 >> {'loss': 0.1627, 'learning_rate': 7.0000e-07, 'epoch': 1.08, 'throughput': 1477.67}
+
+[INFO|callbacks.py:310] 2024-07-11 09:51:54,163 >> {'loss': 0.1421, 'learning_rate': 7.0833e-07, 'epoch': 1.09, 'throughput': 1478.13}
+
+[INFO|callbacks.py:310] 2024-07-11 09:52:05,268 >> {'loss': 0.1517, 'learning_rate': 7.1667e-07, 'epoch': 1.11, 'throughput': 1478.20}
+
+[INFO|callbacks.py:310] 2024-07-11 09:52:16,378 >> {'loss': 0.1519, 'learning_rate': 7.2500e-07, 'epoch': 1.12, 'throughput': 1477.17}
+
+[INFO|callbacks.py:310] 2024-07-11 09:52:27,517 >> {'loss': 0.1280, 'learning_rate': 7.3333e-07, 'epoch': 1.13, 'throughput': 1477.75}
+
+[INFO|callbacks.py:310] 2024-07-11 09:52:38,648 >> {'loss': 0.1481, 'learning_rate': 7.4167e-07, 'epoch': 1.14, 'throughput': 1477.59}
+
+[INFO|callbacks.py:310] 2024-07-11 09:52:49,791 >> {'loss': 0.1636, 'learning_rate': 7.5000e-07, 'epoch': 1.16, 'throughput': 1478.21}
+
+[INFO|callbacks.py:310] 2024-07-11 09:53:00,912 >> {'loss': 0.1443, 'learning_rate': 7.5833e-07, 'epoch': 1.17, 'throughput': 1478.70}
+
+[INFO|callbacks.py:310] 2024-07-11 09:53:12,019 >> {'loss': 0.1592, 'learning_rate': 7.6667e-07, 'epoch': 1.18, 'throughput': 1479.03}
+
+[INFO|callbacks.py:310] 2024-07-11 09:53:23,069 >> {'loss': 0.1744, 'learning_rate': 7.7500e-07, 'epoch': 1.20, 'throughput': 1478.61}
+
+[INFO|callbacks.py:310] 2024-07-11 09:53:34,191 >> {'loss': 0.1615, 'learning_rate': 7.8333e-07, 'epoch': 1.21, 'throughput': 1478.86}
+
+[INFO|callbacks.py:310] 2024-07-11 09:53:45,333 >> {'loss': 0.1466, 'learning_rate': 7.9167e-07, 'epoch': 1.22, 'throughput': 1480.34}
+
+[INFO|callbacks.py:310] 2024-07-11 09:53:56,455 >> {'loss': 0.1333, 'learning_rate': 8.0000e-07, 'epoch': 1.23, 'throughput': 1480.18}
+
+[INFO|callbacks.py:310] 2024-07-11 09:54:07,585 >> {'loss': 0.1475, 'learning_rate': 8.0833e-07, 'epoch': 1.25, 'throughput': 1479.91}
+
+[INFO|callbacks.py:310] 2024-07-11 09:54:18,772 >> {'loss': 0.1420, 'learning_rate': 8.1667e-07, 'epoch': 1.26, 'throughput': 1480.14}
+
+[INFO|callbacks.py:310] 2024-07-11 09:54:29,887 >> {'loss': 0.1480, 'learning_rate': 8.2500e-07, 'epoch': 1.27, 'throughput': 1480.08}
+
+[INFO|callbacks.py:310] 2024-07-11 09:54:40,939 >> {'loss': 0.1539, 'learning_rate': 8.3333e-07, 'epoch': 1.29, 'throughput': 1478.98}
+
+[INFO|callbacks.py:310] 2024-07-11 09:54:52,040 >> {'loss': 0.1496, 'learning_rate': 8.4167e-07, 'epoch': 1.30, 'throughput': 1479.63}
+
+[INFO|callbacks.py:310] 2024-07-11 09:55:03,133 >> {'loss': 0.1563, 'learning_rate': 8.5000e-07, 'epoch': 1.31, 'throughput': 1479.38}
+
+[INFO|callbacks.py:310] 2024-07-11 09:55:14,217 >> {'loss': 0.1295, 'learning_rate': 8.5833e-07, 'epoch': 1.32, 'throughput': 1478.95}
+
+[INFO|callbacks.py:310] 2024-07-11 09:55:25,352 >> {'loss': 0.1353, 'learning_rate': 8.6667e-07, 'epoch': 1.34, 'throughput': 1478.71}
+
+[INFO|callbacks.py:310] 2024-07-11 09:55:36,494 >> {'loss': 0.1376, 'learning_rate': 8.7500e-07, 'epoch': 1.35, 'throughput': 1478.25}
+
+[INFO|callbacks.py:310] 2024-07-11 09:55:47,650 >> {'loss': 0.1313, 'learning_rate': 8.8333e-07, 'epoch': 1.36, 'throughput': 1478.13}
+
+[INFO|callbacks.py:310] 2024-07-11 09:55:58,764 >> {'loss': 0.1367, 'learning_rate': 8.9167e-07, 'epoch': 1.38, 'throughput': 1477.58}
+
+[INFO|callbacks.py:310] 2024-07-11 09:56:09,911 >> {'loss': 0.1350, 'learning_rate': 9.0000e-07, 'epoch': 1.39, 'throughput': 1477.67}
+
+[INFO|callbacks.py:310] 2024-07-11 09:56:20,982 >> {'loss': 0.1212, 'learning_rate': 9.0833e-07, 'epoch': 1.40, 'throughput': 1477.90}
+
+[INFO|callbacks.py:310] 2024-07-11 09:56:32,078 >> {'loss': 0.1355, 'learning_rate': 9.1667e-07, 'epoch': 1.41, 'throughput': 1478.26}
+
+[INFO|callbacks.py:310] 2024-07-11 09:56:43,192 >> {'loss': 0.1439, 'learning_rate': 9.2500e-07, 'epoch': 1.43, 'throughput': 1478.53}
+
+[INFO|callbacks.py:310] 2024-07-11 09:56:54,346 >> {'loss': 0.1323, 'learning_rate': 9.3333e-07, 'epoch': 1.44, 'throughput': 1478.60}
+
+[INFO|callbacks.py:310] 2024-07-11 09:57:05,472 >> {'loss': 0.1376, 'learning_rate': 9.4167e-07, 'epoch': 1.45, 'throughput': 1478.52}
+
+[INFO|callbacks.py:310] 2024-07-11 09:57:16,583 >> {'loss': 0.1191, 'learning_rate': 9.5000e-07, 'epoch': 1.47, 'throughput': 1478.08}
+
+[INFO|callbacks.py:310] 2024-07-11 09:57:27,706 >> {'loss': 0.1045, 'learning_rate': 9.5833e-07, 'epoch': 1.48, 'throughput': 1477.61}
+
+[INFO|callbacks.py:310] 2024-07-11 09:57:38,811 >> {'loss': 0.1467, 'learning_rate': 9.6667e-07, 'epoch': 1.49, 'throughput': 1476.92}
+
+[INFO|callbacks.py:310] 2024-07-11 09:57:49,937 >> {'loss': 0.1142, 'learning_rate': 9.7500e-07, 'epoch': 1.50, 'throughput': 1477.03}
+
+[INFO|callbacks.py:310] 2024-07-11 09:58:01,037 >> {'loss': 0.1107, 'learning_rate': 9.8333e-07, 'epoch': 1.52, 'throughput': 1476.73}
+
+[INFO|callbacks.py:310] 2024-07-11 09:58:12,150 >> {'loss': 0.1460, 'learning_rate': 9.9167e-07, 'epoch': 1.53, 'throughput': 1477.00}
+
+[INFO|callbacks.py:310] 2024-07-11 09:58:23,267 >> {'loss': 0.1533, 'learning_rate': 1.0000e-06, 'epoch': 1.54, 'throughput': 1477.02}
+
+[INFO|callbacks.py:310] 2024-07-11 09:58:34,358 >> {'loss': 0.1315, 'learning_rate': 1.0083e-06, 'epoch': 1.56, 'throughput': 1476.81}
+
+[INFO|callbacks.py:310] 2024-07-11 09:58:45,523 >> {'loss': 0.1197, 'learning_rate': 1.0167e-06, 'epoch': 1.57, 'throughput': 1476.92}
+
+[INFO|callbacks.py:310] 2024-07-11 09:58:56,666 >> {'loss': 0.1228, 'learning_rate': 1.0250e-06, 'epoch': 1.58, 'throughput': 1477.06}
+
+[INFO|callbacks.py:310] 2024-07-11 09:59:07,814 >> {'loss': 0.1273, 'learning_rate': 1.0333e-06, 'epoch': 1.59, 'throughput': 1477.06}
+
+[INFO|callbacks.py:310] 2024-07-11 09:59:18,959 >> {'loss': 0.1410, 'learning_rate': 1.0417e-06, 'epoch': 1.61, 'throughput': 1477.45}
+
+[INFO|callbacks.py:310] 2024-07-11 09:59:30,083 >> {'loss': 0.1315, 'learning_rate': 1.0500e-06, 'epoch': 1.62, 'throughput': 1477.73}
+
+[INFO|callbacks.py:310] 2024-07-11 09:59:41,190 >> {'loss': 0.1136, 'learning_rate': 1.0583e-06, 'epoch': 1.63, 'throughput': 1478.09}
+
+[INFO|callbacks.py:310] 2024-07-11 09:59:52,299 >> {'loss': 0.1013, 'learning_rate': 1.0667e-06, 'epoch': 1.65, 'throughput': 1478.54}
+
+[INFO|callbacks.py:310] 2024-07-11 10:00:03,422 >> {'loss': 0.1056, 'learning_rate': 1.0750e-06, 'epoch': 1.66, 'throughput': 1478.14}
+
+[INFO|callbacks.py:310] 2024-07-11 10:00:14,531 >> {'loss': 0.1071, 'learning_rate': 1.0833e-06, 'epoch': 1.67, 'throughput': 1478.54}
+
+[INFO|callbacks.py:310] 2024-07-11 10:00:25,649 >> {'loss': 0.1357, 'learning_rate': 1.0917e-06, 'epoch': 1.68, 'throughput': 1478.41}
+
+[INFO|callbacks.py:310] 2024-07-11 10:00:36,774 >> {'loss': 0.1181, 'learning_rate': 1.1000e-06, 'epoch': 1.70, 'throughput': 1478.26}
+
+[INFO|callbacks.py:310] 2024-07-11 10:00:47,927 >> {'loss': 0.0826, 'learning_rate': 1.1083e-06, 'epoch': 1.71, 'throughput': 1478.30}
+
+[INFO|callbacks.py:310] 2024-07-11 10:00:59,041 >> {'loss': 0.1221, 'learning_rate': 1.1167e-06, 'epoch': 1.72, 'throughput': 1477.92}
+
+[INFO|callbacks.py:310] 2024-07-11 10:01:10,170 >> {'loss': 0.1021, 'learning_rate': 1.1250e-06, 'epoch': 1.74, 'throughput': 1478.20}
+
+[INFO|callbacks.py:310] 2024-07-11 10:01:21,237 >> {'loss': 0.0980, 'learning_rate': 1.1333e-06, 'epoch': 1.75, 'throughput': 1478.17}
+
+[INFO|callbacks.py:310] 2024-07-11 10:01:32,344 >> {'loss': 0.1085, 'learning_rate': 1.1417e-06, 'epoch': 1.76, 'throughput': 1478.40}
+
+[INFO|callbacks.py:310] 2024-07-11 10:01:43,423 >> {'loss': 0.1038, 'learning_rate': 1.1500e-06, 'epoch': 1.77, 'throughput': 1478.07}
+
+[INFO|callbacks.py:310] 2024-07-11 10:01:54,521 >> {'loss': 0.1081, 'learning_rate': 1.1583e-06, 'epoch': 1.79, 'throughput': 1478.19}
+
+[INFO|callbacks.py:310] 2024-07-11 10:02:05,652 >> {'loss': 0.1287, 'learning_rate': 1.1667e-06, 'epoch': 1.80, 'throughput': 1478.10}
+
+[INFO|callbacks.py:310] 2024-07-11 10:02:16,763 >> {'loss': 0.1068, 'learning_rate': 1.1750e-06, 'epoch': 1.81, 'throughput': 1478.11}
+
+[INFO|callbacks.py:310] 2024-07-11 10:02:27,850 >> {'loss': 0.1202, 'learning_rate': 1.1833e-06, 'epoch': 1.83, 'throughput': 1477.67}
+
+[INFO|callbacks.py:310] 2024-07-11 10:02:38,969 >> {'loss': 0.1190, 'learning_rate': 1.1917e-06, 'epoch': 1.84, 'throughput': 1477.42}
+
+[INFO|callbacks.py:310] 2024-07-11 10:02:50,085 >> {'loss': 0.1273, 'learning_rate': 1.2000e-06, 'epoch': 1.85, 'throughput': 1477.51}
+
+[INFO|callbacks.py:310] 2024-07-11 10:03:01,139 >> {'loss': 0.1024, 'learning_rate': 1.2083e-06, 'epoch': 1.86, 'throughput': 1477.11}
+
+[INFO|callbacks.py:310] 2024-07-11 10:03:12,232 >> {'loss': 0.1143, 'learning_rate': 1.2167e-06, 'epoch': 1.88, 'throughput': 1476.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:03:23,344 >> {'loss': 0.1229, 'learning_rate': 1.2250e-06, 'epoch': 1.89, 'throughput': 1477.01}
+
+[INFO|callbacks.py:310] 2024-07-11 10:03:34,449 >> {'loss': 0.0964, 'learning_rate': 1.2333e-06, 'epoch': 1.90, 'throughput': 1476.61}
+
+[INFO|callbacks.py:310] 2024-07-11 10:03:45,575 >> {'loss': 0.1132, 'learning_rate': 1.2417e-06, 'epoch': 1.92, 'throughput': 1477.03}
+
+[INFO|callbacks.py:310] 2024-07-11 10:03:56,704 >> {'loss': 0.0734, 'learning_rate': 1.2500e-06, 'epoch': 1.93, 'throughput': 1476.98}
+
+[INFO|callbacks.py:310] 2024-07-11 10:04:07,855 >> {'loss': 0.0939, 'learning_rate': 1.2583e-06, 'epoch': 1.94, 'throughput': 1477.57}
+
+[INFO|callbacks.py:310] 2024-07-11 10:04:18,984 >> {'loss': 0.1143, 'learning_rate': 1.2667e-06, 'epoch': 1.95, 'throughput': 1477.48}
+
+[INFO|callbacks.py:310] 2024-07-11 10:04:30,044 >> {'loss': 0.1114, 'learning_rate': 1.2750e-06, 'epoch': 1.97, 'throughput': 1476.92}
+
+[INFO|callbacks.py:310] 2024-07-11 10:04:41,125 >> {'loss': 0.0948, 'learning_rate': 1.2833e-06, 'epoch': 1.98, 'throughput': 1476.58}
+
+[INFO|callbacks.py:310] 2024-07-11 10:04:52,252 >> {'loss': 0.0805, 'learning_rate': 1.2917e-06, 'epoch': 1.99, 'throughput': 1476.85}
+
+[INFO|callbacks.py:310] 2024-07-11 10:05:03,374 >> {'loss': 0.1001, 'learning_rate': 1.3000e-06, 'epoch': 2.01, 'throughput': 1476.62}
+
+[INFO|callbacks.py:310] 2024-07-11 10:05:14,515 >> {'loss': 0.1002, 'learning_rate': 1.3083e-06, 'epoch': 2.02, 'throughput': 1476.44}
+
+[INFO|callbacks.py:310] 2024-07-11 10:05:25,648 >> {'loss': 0.0847, 'learning_rate': 1.3167e-06, 'epoch': 2.03, 'throughput': 1476.26}
+
+[INFO|callbacks.py:310] 2024-07-11 10:05:36,802 >> {'loss': 0.0710, 'learning_rate': 1.3250e-06, 'epoch': 2.05, 'throughput': 1476.41}
+
+[INFO|callbacks.py:310] 2024-07-11 10:05:47,915 >> {'loss': 0.0791, 'learning_rate': 1.3333e-06, 'epoch': 2.06, 'throughput': 1476.00}
+
+[INFO|callbacks.py:310] 2024-07-11 10:05:59,027 >> {'loss': 0.0769, 'learning_rate': 1.3417e-06, 'epoch': 2.07, 'throughput': 1475.83}
+
+[INFO|callbacks.py:310] 2024-07-11 10:06:10,091 >> {'loss': 0.0916, 'learning_rate': 1.3500e-06, 'epoch': 2.08, 'throughput': 1475.46}
+
+[INFO|callbacks.py:310] 2024-07-11 10:06:21,185 >> {'loss': 0.0584, 'learning_rate': 1.3583e-06, 'epoch': 2.10, 'throughput': 1475.48}
+
+[INFO|callbacks.py:310] 2024-07-11 10:06:32,281 >> {'loss': 0.0811, 'learning_rate': 1.3667e-06, 'epoch': 2.11, 'throughput': 1475.35}
+
+[INFO|callbacks.py:310] 2024-07-11 10:06:43,373 >> {'loss': 0.0663, 'learning_rate': 1.3750e-06, 'epoch': 2.12, 'throughput': 1475.17}
+
+[INFO|callbacks.py:310] 2024-07-11 10:06:54,462 >> {'loss': 0.0802, 'learning_rate': 1.3833e-06, 'epoch': 2.14, 'throughput': 1475.13}
+
+[INFO|callbacks.py:310] 2024-07-11 10:07:05,548 >> {'loss': 0.0587, 'learning_rate': 1.3917e-06, 'epoch': 2.15, 'throughput': 1474.82}
+
+[INFO|callbacks.py:310] 2024-07-11 10:07:16,701 >> {'loss': 0.0953, 'learning_rate': 1.4000e-06, 'epoch': 2.16, 'throughput': 1474.73}
+
+[INFO|callbacks.py:310] 2024-07-11 10:07:27,833 >> {'loss': 0.0795, 'learning_rate': 1.4083e-06, 'epoch': 2.17, 'throughput': 1474.63}
+
+[INFO|callbacks.py:310] 2024-07-11 10:07:38,959 >> {'loss': 0.1015, 'learning_rate': 1.4167e-06, 'epoch': 2.19, 'throughput': 1474.63}
+
+[INFO|callbacks.py:310] 2024-07-11 10:07:50,039 >> {'loss': 0.0614, 'learning_rate': 1.4250e-06, 'epoch': 2.20, 'throughput': 1474.37}
+
+[INFO|callbacks.py:310] 2024-07-11 10:08:01,150 >> {'loss': 0.0753, 'learning_rate': 1.4333e-06, 'epoch': 2.21, 'throughput': 1474.59}
+
+[INFO|callbacks.py:310] 2024-07-11 10:08:12,309 >> {'loss': 0.0800, 'learning_rate': 1.4417e-06, 'epoch': 2.23, 'throughput': 1475.05}
+
+[INFO|callbacks.py:310] 2024-07-11 10:08:23,444 >> {'loss': 0.0603, 'learning_rate': 1.4500e-06, 'epoch': 2.24, 'throughput': 1475.13}
+
+[INFO|callbacks.py:310] 2024-07-11 10:08:34,582 >> {'loss': 0.0874, 'learning_rate': 1.4583e-06, 'epoch': 2.25, 'throughput': 1475.01}
+
+[INFO|callbacks.py:310] 2024-07-11 10:08:45,695 >> {'loss': 0.0833, 'learning_rate': 1.4667e-06, 'epoch': 2.26, 'throughput': 1475.02}
+
+[INFO|callbacks.py:310] 2024-07-11 10:08:56,832 >> {'loss': 0.0676, 'learning_rate': 1.4750e-06, 'epoch': 2.28, 'throughput': 1474.80}
+
+[INFO|callbacks.py:310] 2024-07-11 10:09:07,950 >> {'loss': 0.0860, 'learning_rate': 1.4833e-06, 'epoch': 2.29, 'throughput': 1474.96}
+
+[INFO|callbacks.py:310] 2024-07-11 10:09:19,042 >> {'loss': 0.0662, 'learning_rate': 1.4917e-06, 'epoch': 2.30, 'throughput': 1474.55}
+
+[INFO|callbacks.py:310] 2024-07-11 10:09:30,161 >> {'loss': 0.1028, 'learning_rate': 1.5000e-06, 'epoch': 2.32, 'throughput': 1474.63}
+
+[INFO|callbacks.py:310] 2024-07-11 10:09:41,272 >> {'loss': 0.0882, 'learning_rate': 1.5083e-06, 'epoch': 2.33, 'throughput': 1474.45}
+
+[INFO|callbacks.py:310] 2024-07-11 10:09:52,377 >> {'loss': 0.0806, 'learning_rate': 1.5167e-06, 'epoch': 2.34, 'throughput': 1474.52}
+
+[INFO|callbacks.py:310] 2024-07-11 10:10:03,473 >> {'loss': 0.1108, 'learning_rate': 1.5250e-06, 'epoch': 2.35, 'throughput': 1474.19}
+
+[INFO|callbacks.py:310] 2024-07-11 10:10:14,639 >> {'loss': 0.0687, 'learning_rate': 1.5333e-06, 'epoch': 2.37, 'throughput': 1474.68}
+
+[INFO|callbacks.py:310] 2024-07-11 10:10:25,742 >> {'loss': 0.0736, 'learning_rate': 1.5417e-06, 'epoch': 2.38, 'throughput': 1474.67}
+
+[INFO|callbacks.py:310] 2024-07-11 10:10:36,853 >> {'loss': 0.0779, 'learning_rate': 1.5500e-06, 'epoch': 2.39, 'throughput': 1474.70}
+
+[INFO|callbacks.py:310] 2024-07-11 10:10:47,981 >> {'loss': 0.0709, 'learning_rate': 1.5583e-06, 'epoch': 2.41, 'throughput': 1474.61}
+
+[INFO|callbacks.py:310] 2024-07-11 10:10:59,101 >> {'loss': 0.0652, 'learning_rate': 1.5667e-06, 'epoch': 2.42, 'throughput': 1474.76}
+
+[INFO|callbacks.py:310] 2024-07-11 10:11:10,209 >> {'loss': 0.1095, 'learning_rate': 1.5750e-06, 'epoch': 2.43, 'throughput': 1475.00}
+
+[INFO|callbacks.py:310] 2024-07-11 10:11:21,338 >> {'loss': 0.0618, 'learning_rate': 1.5833e-06, 'epoch': 2.44, 'throughput': 1475.18}
+
+[INFO|callbacks.py:310] 2024-07-11 10:11:32,451 >> {'loss': 0.0666, 'learning_rate': 1.5917e-06, 'epoch': 2.46, 'throughput': 1475.36}
+
+[INFO|callbacks.py:310] 2024-07-11 10:11:43,569 >> {'loss': 0.0573, 'learning_rate': 1.6000e-06, 'epoch': 2.47, 'throughput': 1475.40}
+
+[INFO|callbacks.py:310] 2024-07-11 10:11:54,697 >> {'loss': 0.0577, 'learning_rate': 1.6083e-06, 'epoch': 2.48, 'throughput': 1475.50}
+
+[INFO|callbacks.py:310] 2024-07-11 10:12:05,820 >> {'loss': 0.0813, 'learning_rate': 1.6167e-06, 'epoch': 2.50, 'throughput': 1475.34}
+
+[INFO|callbacks.py:310] 2024-07-11 10:12:16,940 >> {'loss': 0.0660, 'learning_rate': 1.6250e-06, 'epoch': 2.51, 'throughput': 1475.24}
+
+[INFO|callbacks.py:310] 2024-07-11 10:12:28,083 >> {'loss': 0.0622, 'learning_rate': 1.6333e-06, 'epoch': 2.52, 'throughput': 1475.32}
+
+[INFO|callbacks.py:310] 2024-07-11 10:12:39,175 >> {'loss': 0.0616, 'learning_rate': 1.6417e-06, 'epoch': 2.53, 'throughput': 1475.12}
+
+[INFO|callbacks.py:310] 2024-07-11 10:12:50,265 >> {'loss': 0.0993, 'learning_rate': 1.6500e-06, 'epoch': 2.55, 'throughput': 1475.12}
+
+[INFO|callbacks.py:310] 2024-07-11 10:13:01,337 >> {'loss': 0.0702, 'learning_rate': 1.6583e-06, 'epoch': 2.56, 'throughput': 1475.10}
+
+[INFO|callbacks.py:310] 2024-07-11 10:13:12,425 >> {'loss': 0.0743, 'learning_rate': 1.6667e-06, 'epoch': 2.57, 'throughput': 1474.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:13:23,525 >> {'loss': 0.0647, 'learning_rate': 1.6750e-06, 'epoch': 2.59, 'throughput': 1474.93}
+
+[INFO|callbacks.py:310] 2024-07-11 10:13:34,651 >> {'loss': 0.0814, 'learning_rate': 1.6833e-06, 'epoch': 2.60, 'throughput': 1475.03}
+
+[INFO|callbacks.py:310] 2024-07-11 10:13:45,757 >> {'loss': 0.0861, 'learning_rate': 1.6917e-06, 'epoch': 2.61, 'throughput': 1474.89}
+
+[INFO|callbacks.py:310] 2024-07-11 10:13:56,905 >> {'loss': 0.0769, 'learning_rate': 1.7000e-06, 'epoch': 2.62, 'throughput': 1475.15}
+
+[INFO|callbacks.py:310] 2024-07-11 10:14:08,051 >> {'loss': 0.0888, 'learning_rate': 1.7083e-06, 'epoch': 2.64, 'throughput': 1475.31}
+
+[INFO|callbacks.py:310] 2024-07-11 10:14:19,164 >> {'loss': 0.1017, 'learning_rate': 1.7167e-06, 'epoch': 2.65, 'throughput': 1475.34}
+
+[INFO|callbacks.py:310] 2024-07-11 10:14:30,261 >> {'loss': 0.0677, 'learning_rate': 1.7250e-06, 'epoch': 2.66, 'throughput': 1475.25}
+
+[INFO|callbacks.py:310] 2024-07-11 10:14:41,375 >> {'loss': 0.0861, 'learning_rate': 1.7333e-06, 'epoch': 2.68, 'throughput': 1475.55}
+
+[INFO|callbacks.py:310] 2024-07-11 10:14:52,497 >> {'loss': 0.0562, 'learning_rate': 1.7417e-06, 'epoch': 2.69, 'throughput': 1475.69}
+
+[INFO|callbacks.py:310] 2024-07-11 10:15:03,612 >> {'loss': 0.0641, 'learning_rate': 1.7500e-06, 'epoch': 2.70, 'throughput': 1475.59}
+
+[INFO|callbacks.py:310] 2024-07-11 10:15:14,730 >> {'loss': 0.0829, 'learning_rate': 1.7583e-06, 'epoch': 2.71, 'throughput': 1475.88}
+
+[INFO|callbacks.py:310] 2024-07-11 10:15:25,844 >> {'loss': 0.0632, 'learning_rate': 1.7667e-06, 'epoch': 2.73, 'throughput': 1475.70}
+
+[INFO|callbacks.py:310] 2024-07-11 10:15:36,953 >> {'loss': 0.0620, 'learning_rate': 1.7750e-06, 'epoch': 2.74, 'throughput': 1475.36}
+
+[INFO|callbacks.py:310] 2024-07-11 10:15:48,070 >> {'loss': 0.0816, 'learning_rate': 1.7833e-06, 'epoch': 2.75, 'throughput': 1475.14}
+
+[INFO|callbacks.py:310] 2024-07-11 10:15:59,182 >> {'loss': 0.0960, 'learning_rate': 1.7917e-06, 'epoch': 2.77, 'throughput': 1475.26}
+
+[INFO|callbacks.py:310] 2024-07-11 10:16:10,296 >> {'loss': 0.0639, 'learning_rate': 1.8000e-06, 'epoch': 2.78, 'throughput': 1475.21}
+
+[INFO|callbacks.py:310] 2024-07-11 10:16:21,386 >> {'loss': 0.0915, 'learning_rate': 1.8083e-06, 'epoch': 2.79, 'throughput': 1474.69}
+
+[INFO|callbacks.py:310] 2024-07-11 10:16:32,501 >> {'loss': 0.0610, 'learning_rate': 1.8167e-06, 'epoch': 2.80, 'throughput': 1474.53}
+
+[INFO|callbacks.py:310] 2024-07-11 10:16:43,658 >> {'loss': 0.0572, 'learning_rate': 1.8250e-06, 'epoch': 2.82, 'throughput': 1474.35}
+
+[INFO|callbacks.py:310] 2024-07-11 10:16:54,814 >> {'loss': 0.0497, 'learning_rate': 1.8333e-06, 'epoch': 2.83, 'throughput': 1474.44}
+
+[INFO|callbacks.py:310] 2024-07-11 10:17:05,943 >> {'loss': 0.0672, 'learning_rate': 1.8417e-06, 'epoch': 2.84, 'throughput': 1474.31}
+
+[INFO|callbacks.py:310] 2024-07-11 10:17:17,069 >> {'loss': 0.0563, 'learning_rate': 1.8500e-06, 'epoch': 2.86, 'throughput': 1474.33}
+
+[INFO|callbacks.py:310] 2024-07-11 10:17:28,189 >> {'loss': 0.0690, 'learning_rate': 1.8583e-06, 'epoch': 2.87, 'throughput': 1474.50}
+
+[INFO|callbacks.py:310] 2024-07-11 10:17:39,309 >> {'loss': 0.0824, 'learning_rate': 1.8667e-06, 'epoch': 2.88, 'throughput': 1474.59}
+
+[INFO|callbacks.py:310] 2024-07-11 10:17:50,430 >> {'loss': 0.0570, 'learning_rate': 1.8750e-06, 'epoch': 2.89, 'throughput': 1474.72}
+
+[INFO|callbacks.py:310] 2024-07-11 10:18:01,560 >> {'loss': 0.0549, 'learning_rate': 1.8833e-06, 'epoch': 2.91, 'throughput': 1475.08}
+
+[INFO|callbacks.py:310] 2024-07-11 10:18:12,661 >> {'loss': 0.0652, 'learning_rate': 1.8917e-06, 'epoch': 2.92, 'throughput': 1474.98}
+
+[INFO|callbacks.py:310] 2024-07-11 10:18:23,802 >> {'loss': 0.0743, 'learning_rate': 1.9000e-06, 'epoch': 2.93, 'throughput': 1475.20}
+
+[INFO|callbacks.py:310] 2024-07-11 10:18:34,940 >> {'loss': 0.0416, 'learning_rate': 1.9083e-06, 'epoch': 2.95, 'throughput': 1475.35}
+
+[INFO|callbacks.py:310] 2024-07-11 10:18:46,091 >> {'loss': 0.0668, 'learning_rate': 1.9167e-06, 'epoch': 2.96, 'throughput': 1475.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:18:57,199 >> {'loss': 0.0603, 'learning_rate': 1.9250e-06, 'epoch': 2.97, 'throughput': 1475.60}
+
+[INFO|callbacks.py:310] 2024-07-11 10:19:08,355 >> {'loss': 0.0660, 'learning_rate': 1.9333e-06, 'epoch': 2.98, 'throughput': 1475.88}
+
+[INFO|callbacks.py:310] 2024-07-11 10:19:19,452 >> {'loss': 0.0692, 'learning_rate': 1.9417e-06, 'epoch': 3.00, 'throughput': 1475.87}
+
+[INFO|callbacks.py:310] 2024-07-11 10:19:30,549 >> {'loss': 0.0323, 'learning_rate': 1.9500e-06, 'epoch': 3.01, 'throughput': 1475.89}
+
+[INFO|callbacks.py:310] 2024-07-11 10:19:41,651 >> {'loss': 0.0403, 'learning_rate': 1.9583e-06, 'epoch': 3.02, 'throughput': 1475.72}
+
+[INFO|callbacks.py:310] 2024-07-11 10:19:52,776 >> {'loss': 0.0477, 'learning_rate': 1.9667e-06, 'epoch': 3.04, 'throughput': 1475.66}
+
+[INFO|callbacks.py:310] 2024-07-11 10:20:03,890 >> {'loss': 0.0356, 'learning_rate': 1.9750e-06, 'epoch': 3.05, 'throughput': 1475.41}
+
+[INFO|callbacks.py:310] 2024-07-11 10:20:14,994 >> {'loss': 0.0324, 'learning_rate': 1.9833e-06, 'epoch': 3.06, 'throughput': 1475.56}
+
+[INFO|callbacks.py:310] 2024-07-11 10:20:26,140 >> {'loss': 0.0428, 'learning_rate': 1.9917e-06, 'epoch': 3.07, 'throughput': 1475.76}
+
+[INFO|callbacks.py:310] 2024-07-11 10:20:37,270 >> {'loss': 0.0315, 'learning_rate': 2.0000e-06, 'epoch': 3.09, 'throughput': 1475.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:20:48,387 >> {'loss': 0.0366, 'learning_rate': 2.0083e-06, 'epoch': 3.10, 'throughput': 1475.78}
+
+[INFO|callbacks.py:310] 2024-07-11 10:20:59,460 >> {'loss': 0.0307, 'learning_rate': 2.0167e-06, 'epoch': 3.11, 'throughput': 1475.47}
+
+[INFO|callbacks.py:310] 2024-07-11 10:21:10,586 >> {'loss': 0.0338, 'learning_rate': 2.0250e-06, 'epoch': 3.13, 'throughput': 1475.52}
+
+[INFO|callbacks.py:310] 2024-07-11 10:21:21,705 >> {'loss': 0.0348, 'learning_rate': 2.0333e-06, 'epoch': 3.14, 'throughput': 1475.62}
+
+[INFO|callbacks.py:310] 2024-07-11 10:21:32,815 >> {'loss': 0.0355, 'learning_rate': 2.0417e-06, 'epoch': 3.15, 'throughput': 1475.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:21:43,960 >> {'loss': 0.0272, 'learning_rate': 2.0500e-06, 'epoch': 3.16, 'throughput': 1475.84}
+
+[INFO|callbacks.py:310] 2024-07-11 10:21:55,085 >> {'loss': 0.0464, 'learning_rate': 2.0583e-06, 'epoch': 3.18, 'throughput': 1475.83}
+
+[INFO|callbacks.py:310] 2024-07-11 10:22:06,205 >> {'loss': 0.0268, 'learning_rate': 2.0667e-06, 'epoch': 3.19, 'throughput': 1475.60}
+
+[INFO|callbacks.py:310] 2024-07-11 10:22:17,319 >> {'loss': 0.0229, 'learning_rate': 2.0750e-06, 'epoch': 3.20, 'throughput': 1475.38}
+
+[INFO|callbacks.py:310] 2024-07-11 10:22:28,439 >> {'loss': 0.0373, 'learning_rate': 2.0833e-06, 'epoch': 3.22, 'throughput': 1475.74}
+
+[INFO|callbacks.py:310] 2024-07-11 10:22:39,521 >> {'loss': 0.0361, 'learning_rate': 2.0917e-06, 'epoch': 3.23, 'throughput': 1475.59}
+
+[INFO|callbacks.py:310] 2024-07-11 10:22:50,624 >> {'loss': 0.0401, 'learning_rate': 2.1000e-06, 'epoch': 3.24, 'throughput': 1475.60}
+
+[INFO|callbacks.py:310] 2024-07-11 10:23:01,715 >> {'loss': 0.0262, 'learning_rate': 2.1083e-06, 'epoch': 3.25, 'throughput': 1475.38}
+
+[INFO|callbacks.py:310] 2024-07-11 10:23:12,825 >> {'loss': 0.0481, 'learning_rate': 2.1167e-06, 'epoch': 3.27, 'throughput': 1475.34}
+
+[INFO|callbacks.py:310] 2024-07-11 10:23:23,969 >> {'loss': 0.0417, 'learning_rate': 2.1250e-06, 'epoch': 3.28, 'throughput': 1475.21}
+
+[INFO|callbacks.py:310] 2024-07-11 10:23:35,127 >> {'loss': 0.0457, 'learning_rate': 2.1333e-06, 'epoch': 3.29, 'throughput': 1475.48}
+
+[INFO|callbacks.py:310] 2024-07-11 10:23:46,239 >> {'loss': 0.0175, 'learning_rate': 2.1417e-06, 'epoch': 3.31, 'throughput': 1475.19}
+
+[INFO|callbacks.py:310] 2024-07-11 10:23:57,391 >> {'loss': 0.0371, 'learning_rate': 2.1500e-06, 'epoch': 3.32, 'throughput': 1475.40}
+
+[INFO|callbacks.py:310] 2024-07-11 10:24:08,512 >> {'loss': 0.0268, 'learning_rate': 2.1583e-06, 'epoch': 3.33, 'throughput': 1475.58}
+
+[INFO|callbacks.py:310] 2024-07-11 10:24:19,598 >> {'loss': 0.0417, 'learning_rate': 2.1667e-06, 'epoch': 3.34, 'throughput': 1475.58}
+
+[INFO|callbacks.py:310] 2024-07-11 10:24:30,714 >> {'loss': 0.0264, 'learning_rate': 2.1750e-06, 'epoch': 3.36, 'throughput': 1475.32}
+
+[INFO|callbacks.py:310] 2024-07-11 10:24:41,829 >> {'loss': 0.0336, 'learning_rate': 2.1833e-06, 'epoch': 3.37, 'throughput': 1475.42}
+
+[INFO|callbacks.py:310] 2024-07-11 10:24:52,937 >> {'loss': 0.0434, 'learning_rate': 2.1917e-06, 'epoch': 3.38, 'throughput': 1475.28}
+
+[INFO|callbacks.py:310] 2024-07-11 10:25:04,052 >> {'loss': 0.0389, 'learning_rate': 2.2000e-06, 'epoch': 3.40, 'throughput': 1475.08}
+
+[INFO|callbacks.py:310] 2024-07-11 10:25:15,179 >> {'loss': 0.0450, 'learning_rate': 2.2083e-06, 'epoch': 3.41, 'throughput': 1474.84}
+
+[INFO|callbacks.py:310] 2024-07-11 10:25:26,284 >> {'loss': 0.0331, 'learning_rate': 2.2167e-06, 'epoch': 3.42, 'throughput': 1474.66}
+
+[INFO|callbacks.py:310] 2024-07-11 10:25:37,399 >> {'loss': 0.0228, 'learning_rate': 2.2250e-06, 'epoch': 3.43, 'throughput': 1474.73}
+
+[INFO|callbacks.py:310] 2024-07-11 10:25:48,485 >> {'loss': 0.0307, 'learning_rate': 2.2333e-06, 'epoch': 3.45, 'throughput': 1474.93}
+
+[INFO|callbacks.py:310] 2024-07-11 10:25:59,600 >> {'loss': 0.0332, 'learning_rate': 2.2417e-06, 'epoch': 3.46, 'throughput': 1474.79}
+
+[INFO|callbacks.py:310] 2024-07-11 10:26:10,724 >> {'loss': 0.0662, 'learning_rate': 2.2500e-06, 'epoch': 3.47, 'throughput': 1474.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:26:21,833 >> {'loss': 0.0431, 'learning_rate': 2.2583e-06, 'epoch': 3.49, 'throughput': 1474.76}
+
+[INFO|callbacks.py:310] 2024-07-11 10:26:32,988 >> {'loss': 0.0423, 'learning_rate': 2.2667e-06, 'epoch': 3.50, 'throughput': 1474.84}
+
+[INFO|callbacks.py:310] 2024-07-11 10:26:44,135 >> {'loss': 0.0447, 'learning_rate': 2.2750e-06, 'epoch': 3.51, 'throughput': 1474.86}
+
+[INFO|callbacks.py:310] 2024-07-11 10:26:55,240 >> {'loss': 0.0337, 'learning_rate': 2.2833e-06, 'epoch': 3.52, 'throughput': 1474.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:27:06,362 >> {'loss': 0.0319, 'learning_rate': 2.2917e-06, 'epoch': 3.54, 'throughput': 1474.61}
+
+[INFO|callbacks.py:310] 2024-07-11 10:27:17,507 >> {'loss': 0.0270, 'learning_rate': 2.3000e-06, 'epoch': 3.55, 'throughput': 1474.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:27:28,590 >> {'loss': 0.0244, 'learning_rate': 2.3083e-06, 'epoch': 3.56, 'throughput': 1474.91}
+
+[INFO|callbacks.py:310] 2024-07-11 10:27:39,702 >> {'loss': 0.0377, 'learning_rate': 2.3167e-06, 'epoch': 3.58, 'throughput': 1475.12}
+
+[INFO|callbacks.py:310] 2024-07-11 10:27:50,833 >> {'loss': 0.0477, 'learning_rate': 2.3250e-06, 'epoch': 3.59, 'throughput': 1475.12}
+
+[INFO|callbacks.py:310] 2024-07-11 10:28:01,921 >> {'loss': 0.0332, 'learning_rate': 2.3333e-06, 'epoch': 3.60, 'throughput': 1474.74}
+
+[INFO|callbacks.py:310] 2024-07-11 10:28:13,062 >> {'loss': 0.0285, 'learning_rate': 2.3417e-06, 'epoch': 3.61, 'throughput': 1474.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:28:24,197 >> {'loss': 0.0468, 'learning_rate': 2.3500e-06, 'epoch': 3.63, 'throughput': 1475.00}
+
+[INFO|callbacks.py:310] 2024-07-11 10:28:35,319 >> {'loss': 0.0370, 'learning_rate': 2.3583e-06, 'epoch': 3.64, 'throughput': 1475.13}
+
+[INFO|callbacks.py:310] 2024-07-11 10:28:46,466 >> {'loss': 0.0410, 'learning_rate': 2.3667e-06, 'epoch': 3.65, 'throughput': 1474.98}
+
+[INFO|callbacks.py:310] 2024-07-11 10:28:57,575 >> {'loss': 0.0461, 'learning_rate': 2.3750e-06, 'epoch': 3.67, 'throughput': 1475.10}
+
+[INFO|callbacks.py:310] 2024-07-11 10:29:08,687 >> {'loss': 0.0594, 'learning_rate': 2.3833e-06, 'epoch': 3.68, 'throughput': 1474.95}
+
+[INFO|callbacks.py:310] 2024-07-11 10:29:19,788 >> {'loss': 0.0728, 'learning_rate': 2.3917e-06, 'epoch': 3.69, 'throughput': 1475.07}
+
+[INFO|callbacks.py:310] 2024-07-11 10:29:30,916 >> {'loss': 0.0290, 'learning_rate': 2.4000e-06, 'epoch': 3.70, 'throughput': 1475.20}
+
+[INFO|callbacks.py:310] 2024-07-11 10:29:42,019 >> {'loss': 0.0412, 'learning_rate': 2.4083e-06, 'epoch': 3.72, 'throughput': 1475.44}
+
+[INFO|callbacks.py:310] 2024-07-11 10:29:53,160 >> {'loss': 0.0262, 'learning_rate': 2.4167e-06, 'epoch': 3.73, 'throughput': 1475.13}
+
+[INFO|callbacks.py:310] 2024-07-11 10:30:04,273 >> {'loss': 0.0796, 'learning_rate': 2.4250e-06, 'epoch': 3.74, 'throughput': 1474.96}
+
+[INFO|callbacks.py:310] 2024-07-11 10:30:15,400 >> {'loss': 0.0447, 'learning_rate': 2.4333e-06, 'epoch': 3.76, 'throughput': 1474.93}
+
+[INFO|callbacks.py:310] 2024-07-11 10:30:26,536 >> {'loss': 0.0252, 'learning_rate': 2.4417e-06, 'epoch': 3.77, 'throughput': 1474.95}
+
+[INFO|callbacks.py:310] 2024-07-11 10:30:37,624 >> {'loss': 0.0458, 'learning_rate': 2.4500e-06, 'epoch': 3.78, 'throughput': 1474.79}
+
+[INFO|callbacks.py:310] 2024-07-11 10:30:48,698 >> {'loss': 0.0431, 'learning_rate': 2.4583e-06, 'epoch': 3.79, 'throughput': 1474.62}
+
+[INFO|callbacks.py:310] 2024-07-11 10:30:59,827 >> {'loss': 0.0422, 'learning_rate': 2.4667e-06, 'epoch': 3.81, 'throughput': 1474.72}
+
+[INFO|callbacks.py:310] 2024-07-11 10:31:10,927 >> {'loss': 0.0428, 'learning_rate': 2.4750e-06, 'epoch': 3.82, 'throughput': 1474.85}
+
+[INFO|callbacks.py:310] 2024-07-11 10:31:22,040 >> {'loss': 0.0473, 'learning_rate': 2.4833e-06, 'epoch': 3.83, 'throughput': 1474.88}
+
+[INFO|callbacks.py:310] 2024-07-11 10:31:33,168 >> {'loss': 0.0223, 'learning_rate': 2.4917e-06, 'epoch': 3.85, 'throughput': 1474.81}
+
+[INFO|callbacks.py:310] 2024-07-11 10:31:44,304 >> {'loss': 0.0274, 'learning_rate': 2.5000e-06, 'epoch': 3.86, 'throughput': 1474.73}
+
+[INFO|callbacks.py:310] 2024-07-11 10:31:55,446 >> {'loss': 0.0454, 'learning_rate': 2.5083e-06, 'epoch': 3.87, 'throughput': 1474.68}
+
+[INFO|callbacks.py:310] 2024-07-11 10:32:06,565 >> {'loss': 0.0220, 'learning_rate': 2.5167e-06, 'epoch': 3.88, 'throughput': 1474.90}
+
+[INFO|callbacks.py:310] 2024-07-11 10:32:17,678 >> {'loss': 0.0357, 'learning_rate': 2.5250e-06, 'epoch': 3.90, 'throughput': 1474.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:32:28,794 >> {'loss': 0.0458, 'learning_rate': 2.5333e-06, 'epoch': 3.91, 'throughput': 1475.20}
+
+[INFO|callbacks.py:310] 2024-07-11 10:32:39,890 >> {'loss': 0.0454, 'learning_rate': 2.5417e-06, 'epoch': 3.92, 'throughput': 1475.13}
+
+[INFO|callbacks.py:310] 2024-07-11 10:32:51,000 >> {'loss': 0.0262, 'learning_rate': 2.5500e-06, 'epoch': 3.94, 'throughput': 1475.05}
+
+[INFO|callbacks.py:310] 2024-07-11 10:33:02,090 >> {'loss': 0.0234, 'learning_rate': 2.5583e-06, 'epoch': 3.95, 'throughput': 1474.86}
+
+[INFO|callbacks.py:310] 2024-07-11 10:33:13,206 >> {'loss': 0.0366, 'learning_rate': 2.5667e-06, 'epoch': 3.96, 'throughput': 1474.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:33:24,327 >> {'loss': 0.0236, 'learning_rate': 2.5750e-06, 'epoch': 3.97, 'throughput': 1474.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:33:35,481 >> {'loss': 0.0403, 'learning_rate': 2.5833e-06, 'epoch': 3.99, 'throughput': 1475.24}
+
+[INFO|callbacks.py:310] 2024-07-11 10:33:46,614 >> {'loss': 0.0395, 'learning_rate': 2.5917e-06, 'epoch': 4.00, 'throughput': 1475.24}
+
+[INFO|callbacks.py:310] 2024-07-11 10:33:57,694 >> {'loss': 0.0163, 'learning_rate': 2.6000e-06, 'epoch': 4.01, 'throughput': 1475.19}
+
+[INFO|callbacks.py:310] 2024-07-11 10:34:08,791 >> {'loss': 0.0181, 'learning_rate': 2.6083e-06, 'epoch': 4.03, 'throughput': 1475.00}
+
+[INFO|callbacks.py:310] 2024-07-11 10:34:19,898 >> {'loss': 0.0130, 'learning_rate': 2.6167e-06, 'epoch': 4.04, 'throughput': 1474.96}
+
+[INFO|callbacks.py:310] 2024-07-11 10:34:30,966 >> {'loss': 0.0178, 'learning_rate': 2.6250e-06, 'epoch': 4.05, 'throughput': 1474.78}
+
+[INFO|callbacks.py:310] 2024-07-11 10:34:42,123 >> {'loss': 0.0153, 'learning_rate': 2.6333e-06, 'epoch': 4.06, 'throughput': 1475.20}
+
+[INFO|callbacks.py:310] 2024-07-11 10:34:53,252 >> {'loss': 0.0205, 'learning_rate': 2.6417e-06, 'epoch': 4.08, 'throughput': 1475.09}
+
+[INFO|callbacks.py:310] 2024-07-11 10:35:04,398 >> {'loss': 0.0021, 'learning_rate': 2.6500e-06, 'epoch': 4.09, 'throughput': 1475.17}
+
+[INFO|callbacks.py:310] 2024-07-11 10:35:15,518 >> {'loss': 0.0220, 'learning_rate': 2.6583e-06, 'epoch': 4.10, 'throughput': 1475.20}
+
+[INFO|callbacks.py:310] 2024-07-11 10:35:26,625 >> {'loss': 0.0134, 'learning_rate': 2.6667e-06, 'epoch': 4.12, 'throughput': 1475.14}
+
+[INFO|callbacks.py:310] 2024-07-11 10:35:37,710 >> {'loss': 0.0175, 'learning_rate': 2.6750e-06, 'epoch': 4.13, 'throughput': 1474.95}
+
+[INFO|callbacks.py:310] 2024-07-11 10:35:48,853 >> {'loss': 0.0230, 'learning_rate': 2.6833e-06, 'epoch': 4.14, 'throughput': 1474.98}
+
+[INFO|callbacks.py:310] 2024-07-11 10:35:59,942 >> {'loss': 0.0303, 'learning_rate': 2.6917e-06, 'epoch': 4.15, 'throughput': 1474.77}
+
+[INFO|callbacks.py:310] 2024-07-11 10:36:11,072 >> {'loss': 0.0187, 'learning_rate': 2.7000e-06, 'epoch': 4.17, 'throughput': 1474.91}
+
+[INFO|callbacks.py:310] 2024-07-11 10:36:22,179 >> {'loss': 0.0126, 'learning_rate': 2.7083e-06, 'epoch': 4.18, 'throughput': 1474.71}
+
+[INFO|callbacks.py:310] 2024-07-11 10:36:33,308 >> {'loss': 0.0203, 'learning_rate': 2.7167e-06, 'epoch': 4.19, 'throughput': 1474.67}
+
+[INFO|callbacks.py:310] 2024-07-11 10:36:44,410 >> {'loss': 0.0078, 'learning_rate': 2.7250e-06, 'epoch': 4.21, 'throughput': 1474.62}
+
+[INFO|callbacks.py:310] 2024-07-11 10:36:55,536 >> {'loss': 0.0165, 'learning_rate': 2.7333e-06, 'epoch': 4.22, 'throughput': 1474.50}
+
+[INFO|callbacks.py:310] 2024-07-11 10:37:06,705 >> {'loss': 0.0113, 'learning_rate': 2.7417e-06, 'epoch': 4.23, 'throughput': 1474.78}
+
+[INFO|callbacks.py:310] 2024-07-11 10:37:17,796 >> {'loss': 0.0058, 'learning_rate': 2.7500e-06, 'epoch': 4.24, 'throughput': 1474.81}
+
+[INFO|callbacks.py:310] 2024-07-11 10:37:28,878 >> {'loss': 0.0070, 'learning_rate': 2.7583e-06, 'epoch': 4.26, 'throughput': 1474.67}
+
+[INFO|callbacks.py:310] 2024-07-11 10:37:39,999 >> {'loss': 0.0270, 'learning_rate': 2.7667e-06, 'epoch': 4.27, 'throughput': 1474.84}
+
+[INFO|callbacks.py:310] 2024-07-11 10:37:51,079 >> {'loss': 0.0276, 'learning_rate': 2.7750e-06, 'epoch': 4.28, 'throughput': 1474.61}
+
+[INFO|callbacks.py:310] 2024-07-11 10:38:02,192 >> {'loss': 0.0367, 'learning_rate': 2.7833e-06, 'epoch': 4.30, 'throughput': 1474.55}
+
+[INFO|callbacks.py:310] 2024-07-11 10:38:13,299 >> {'loss': 0.0161, 'learning_rate': 2.7917e-06, 'epoch': 4.31, 'throughput': 1474.44}
+
+[INFO|callbacks.py:310] 2024-07-11 10:38:24,431 >> {'loss': 0.0180, 'learning_rate': 2.8000e-06, 'epoch': 4.32, 'throughput': 1474.57}
+
+[INFO|callbacks.py:310] 2024-07-11 10:38:35,563 >> {'loss': 0.0044, 'learning_rate': 2.8083e-06, 'epoch': 4.33, 'throughput': 1474.64}
+
+[INFO|callbacks.py:310] 2024-07-11 10:38:46,691 >> {'loss': 0.0109, 'learning_rate': 2.8167e-06, 'epoch': 4.35, 'throughput': 1474.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:38:57,770 >> {'loss': 0.0173, 'learning_rate': 2.8250e-06, 'epoch': 4.36, 'throughput': 1474.67}
+
+[INFO|callbacks.py:310] 2024-07-11 10:39:08,939 >> {'loss': 0.0107, 'learning_rate': 2.8333e-06, 'epoch': 4.37, 'throughput': 1474.70}
+
+[INFO|callbacks.py:310] 2024-07-11 10:39:20,028 >> {'loss': 0.0116, 'learning_rate': 2.8417e-06, 'epoch': 4.39, 'throughput': 1474.55}
+
+[INFO|callbacks.py:310] 2024-07-11 10:39:31,145 >> {'loss': 0.0281, 'learning_rate': 2.8500e-06, 'epoch': 4.40, 'throughput': 1474.78}
+
+[INFO|callbacks.py:310] 2024-07-11 10:39:42,249 >> {'loss': 0.0246, 'learning_rate': 2.8583e-06, 'epoch': 4.41, 'throughput': 1474.86}
+
+[INFO|callbacks.py:310] 2024-07-11 10:39:53,373 >> {'loss': 0.0146, 'learning_rate': 2.8667e-06, 'epoch': 4.42, 'throughput': 1474.84}
+
+[INFO|callbacks.py:310] 2024-07-11 10:40:04,489 >> {'loss': 0.0439, 'learning_rate': 2.8750e-06, 'epoch': 4.44, 'throughput': 1474.62}
+
+[INFO|callbacks.py:310] 2024-07-11 10:40:15,615 >> {'loss': 0.0279, 'learning_rate': 2.8833e-06, 'epoch': 4.45, 'throughput': 1474.60}
+
+[INFO|callbacks.py:310] 2024-07-11 10:40:26,740 >> {'loss': 0.0276, 'learning_rate': 2.8917e-06, 'epoch': 4.46, 'throughput': 1474.70}
+
+[INFO|callbacks.py:310] 2024-07-11 10:40:37,836 >> {'loss': 0.0167, 'learning_rate': 2.9000e-06, 'epoch': 4.48, 'throughput': 1474.63}
+
+[INFO|callbacks.py:310] 2024-07-11 10:40:48,963 >> {'loss': 0.0258, 'learning_rate': 2.9083e-06, 'epoch': 4.49, 'throughput': 1474.83}
+
+[INFO|callbacks.py:310] 2024-07-11 10:41:00,055 >> {'loss': 0.0306, 'learning_rate': 2.9167e-06, 'epoch': 4.50, 'throughput': 1474.91}
+
+[INFO|callbacks.py:310] 2024-07-11 10:41:11,182 >> {'loss': 0.0395, 'learning_rate': 2.9250e-06, 'epoch': 4.51, 'throughput': 1475.01}
+
+[INFO|callbacks.py:310] 2024-07-11 10:41:22,328 >> {'loss': 0.0203, 'learning_rate': 2.9333e-06, 'epoch': 4.53, 'throughput': 1475.06}
+
+[INFO|callbacks.py:310] 2024-07-11 10:41:33,436 >> {'loss': 0.0241, 'learning_rate': 2.9417e-06, 'epoch': 4.54, 'throughput': 1475.01}
+
+[INFO|callbacks.py:310] 2024-07-11 10:41:44,523 >> {'loss': 0.0151, 'learning_rate': 2.9500e-06, 'epoch': 4.55, 'throughput': 1474.68}
+
+[INFO|callbacks.py:310] 2024-07-11 10:41:55,641 >> {'loss': 0.0355, 'learning_rate': 2.9583e-06, 'epoch': 4.57, 'throughput': 1474.59}
+
+[INFO|callbacks.py:310] 2024-07-11 10:42:06,750 >> {'loss': 0.0093, 'learning_rate': 2.9667e-06, 'epoch': 4.58, 'throughput': 1474.72}
+
+[INFO|callbacks.py:310] 2024-07-11 10:42:17,848 >> {'loss': 0.0314, 'learning_rate': 2.9750e-06, 'epoch': 4.59, 'throughput': 1474.75}
+
+[INFO|callbacks.py:310] 2024-07-11 10:42:28,952 >> {'loss': 0.0155, 'learning_rate': 2.9833e-06, 'epoch': 4.60, 'throughput': 1474.77}
+
+[INFO|callbacks.py:310] 2024-07-11 10:42:40,070 >> {'loss': 0.0138, 'learning_rate': 2.9917e-06, 'epoch': 4.62, 'throughput': 1474.94}
+
+[INFO|callbacks.py:310] 2024-07-11 10:42:51,212 >> {'loss': 0.0212, 'learning_rate': 3.0000e-06, 'epoch': 4.63, 'throughput': 1475.10}
+
+[INFO|callbacks.py:310] 2024-07-11 10:43:02,328 >> {'loss': 0.0255, 'learning_rate': 3.0083e-06, 'epoch': 4.64, 'throughput': 1475.11}
+
+[INFO|callbacks.py:310] 2024-07-11 10:43:13,439 >> {'loss': 0.0231, 'learning_rate': 3.0167e-06, 'epoch': 4.66, 'throughput': 1474.99}
+
+[INFO|callbacks.py:310] 2024-07-11 10:43:24,561 >> {'loss': 0.0083, 'learning_rate': 3.0250e-06, 'epoch': 4.67, 'throughput': 1474.99}
+
+[INFO|callbacks.py:310] 2024-07-11 10:43:35,710 >> {'loss': 0.0122, 'learning_rate': 3.0333e-06, 'epoch': 4.68, 'throughput': 1475.18}
+
+[INFO|callbacks.py:310] 2024-07-11 10:43:46,811 >> {'loss': 0.0142, 'learning_rate': 3.0417e-06, 'epoch': 4.69, 'throughput': 1475.37}
+
+[INFO|callbacks.py:310] 2024-07-11 10:43:57,895 >> {'loss': 0.0293, 'learning_rate': 3.0500e-06, 'epoch': 4.71, 'throughput': 1475.24}
+
+[INFO|callbacks.py:310] 2024-07-11 10:44:09,006 >> {'loss': 0.0292, 'learning_rate': 3.0583e-06, 'epoch': 4.72, 'throughput': 1475.50}
+
+[INFO|callbacks.py:310] 2024-07-11 10:44:20,120 >> {'loss': 0.0320, 'learning_rate': 3.0667e-06, 'epoch': 4.73, 'throughput': 1475.46}
+
+[INFO|callbacks.py:310] 2024-07-11 10:44:31,238 >> {'loss': 0.0189, 'learning_rate': 3.0750e-06, 'epoch': 4.75, 'throughput': 1475.50}
+
+[INFO|callbacks.py:310] 2024-07-11 10:44:42,405 >> {'loss': 0.0220, 'learning_rate': 3.0833e-06, 'epoch': 4.76, 'throughput': 1475.57}
+
+[INFO|callbacks.py:310] 2024-07-11 10:44:53,524 >> {'loss': 0.0242, 'learning_rate': 3.0917e-06, 'epoch': 4.77, 'throughput': 1475.68}
+
+[INFO|callbacks.py:310] 2024-07-11 10:45:04,670 >> {'loss': 0.0150, 'learning_rate': 3.1000e-06, 'epoch': 4.78, 'throughput': 1475.82}
+
+[INFO|callbacks.py:310] 2024-07-11 10:45:15,790 >> {'loss': 0.0154, 'learning_rate': 3.1083e-06, 'epoch': 4.80, 'throughput': 1475.81}
+
+[INFO|callbacks.py:310] 2024-07-11 10:45:26,907 >> {'loss': 0.0195, 'learning_rate': 3.1167e-06, 'epoch': 4.81, 'throughput': 1475.92}
+
+[INFO|callbacks.py:310] 2024-07-11 10:45:38,027 >> {'loss': 0.0310, 'learning_rate': 3.1250e-06, 'epoch': 4.82, 'throughput': 1476.19}
+
+[INFO|callbacks.py:310] 2024-07-11 10:45:49,107 >> {'loss': 0.0376, 'learning_rate': 3.1333e-06, 'epoch': 4.84, 'throughput': 1476.09}
+
+[INFO|callbacks.py:310] 2024-07-11 10:46:00,236 >> {'loss': 0.0093, 'learning_rate': 3.1417e-06, 'epoch': 4.85, 'throughput': 1476.24}
+
+[INFO|callbacks.py:310] 2024-07-11 10:46:11,360 >> {'loss': 0.0270, 'learning_rate': 3.1500e-06, 'epoch': 4.86, 'throughput': 1476.22}
+
+[INFO|callbacks.py:310] 2024-07-11 10:46:22,487 >> {'loss': 0.0191, 'learning_rate': 3.1583e-06, 'epoch': 4.87, 'throughput': 1476.16}
+
+[INFO|callbacks.py:310] 2024-07-11 10:46:33,618 >> {'loss': 0.0132, 'learning_rate': 3.1667e-06, 'epoch': 4.89, 'throughput': 1476.06}
+
+[INFO|callbacks.py:310] 2024-07-11 10:46:44,735 >> {'loss': 0.0111, 'learning_rate': 3.1750e-06, 'epoch': 4.90, 'throughput': 1475.94}
+
+[INFO|callbacks.py:310] 2024-07-11 10:46:55,863 >> {'loss': 0.0145, 'learning_rate': 3.1833e-06, 'epoch': 4.91, 'throughput': 1475.98}
+
+[INFO|callbacks.py:310] 2024-07-11 10:47:06,943 >> {'loss': 0.0082, 'learning_rate': 3.1917e-06, 'epoch': 4.93, 'throughput': 1475.97}
+
+[INFO|callbacks.py:310] 2024-07-11 10:47:18,028 >> {'loss': 0.0165, 'learning_rate': 3.2000e-06, 'epoch': 4.94, 'throughput': 1475.81}
+
+[INFO|callbacks.py:310] 2024-07-11 10:47:29,154 >> {'loss': 0.0243, 'learning_rate': 3.2083e-06, 'epoch': 4.95, 'throughput': 1476.08}
+
+[INFO|trainer.py:3478] 2024-07-11 10:47:35,554 >> Saving model checkpoint to saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/checkpoint-385
+
+[INFO|configuration_utils.py:472] 2024-07-11 10:47:35,557 >> Configuration saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/checkpoint-385/config.json
+
+[INFO|configuration_utils.py:769] 2024-07-11 10:47:35,558 >> Configuration saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/checkpoint-385/generation_config.json
+
+[INFO|modeling_utils.py:2698] 2024-07-11 10:47:49,264 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 3 checkpoint shards. You can find where each parameters has been saved in the index located at saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/checkpoint-385/model.safetensors.index.json.
+
+[INFO|tokenization_utils_base.py:2574] 2024-07-11 10:47:49,265 >> tokenizer config file saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/checkpoint-385/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2583] 2024-07-11 10:47:49,265 >> Special tokens file saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/checkpoint-385/special_tokens_map.json
+
+[INFO|trainer.py:2383] 2024-07-11 10:48:19,619 >> 
+
+Training completed. Do not forget to share your model on huggingface.co/models =)
+
+
+
+[INFO|trainer.py:3478] 2024-07-11 10:48:26,073 >> Saving model checkpoint to saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth
+
+[INFO|configuration_utils.py:472] 2024-07-11 10:48:26,076 >> Configuration saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/config.json
+
+[INFO|configuration_utils.py:769] 2024-07-11 10:48:26,076 >> Configuration saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/generation_config.json
+
+[INFO|modeling_utils.py:2698] 2024-07-11 10:48:39,859 >> The model is bigger than the maximum size per checkpoint (5GB) and is going to be split in 3 checkpoint shards. You can find where each parameters has been saved in the index located at saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/model.safetensors.index.json.
+
+[INFO|tokenization_utils_base.py:2574] 2024-07-11 10:48:39,860 >> tokenizer config file saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/tokenizer_config.json
+
+[INFO|tokenization_utils_base.py:2583] 2024-07-11 10:48:39,860 >> Special tokens file saved in saves/LLaMA2-7B-Chat/full/train_2024-07-11-09-30-54_llama2_inst_truth/special_tokens_map.json
+
+[WARNING|ploting.py:89] 2024-07-11 10:48:40,948 >> No metric eval_loss to plot.
+
+[WARNING|ploting.py:89] 2024-07-11 10:48:40,949 >> No metric eval_accuracy to plot.
+
+[INFO|modelcard.py:449] 2024-07-11 10:48:40,950 >> Dropping the following result as it does not have all the necessary fields:
+{'task': {'name': 'Causal Language Modeling', 'type': 'text-generation'}}
+

special_tokens_map.json

@@ -0,0 +1,24 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": "</s>",
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

tokenizer_config.json

@@ -0,0 +1,44 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "add_prefix_space": null,
+  "added_tokens_decoder": {
+    "0": {
+      "content": "<unk>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "1": {
+      "content": "<s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "2": {
+      "content": "</s>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<s>",
+  "chat_template": "{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if loop.index0 == 0 and system_message is defined %}{% set content = '<<SYS>>\n' + system_message + '\n<</SYS>>\n\n' + message['content'] %}{% endif %}{% if message['role'] == 'user' %}{{ '<s>' + '[INST] ' + content + ' [/INST]' }}{% elif message['role'] == 'assistant' %}{{ content + '</s>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": false,
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "</s>",
+  "padding_side": "right",
+  "sp_model_kwargs": {},
+  "split_special_tokens": false,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>",
+  "use_default_system_prompt": false
+}

train_results.json

@@ -0,0 +1,9 @@
+{
+    "epoch": 4.951768488745981,
+    "num_input_tokens_seen": 6318560,
+    "total_flos": 2.5049338265940787e+17,
+    "train_loss": 0.9364173337708336,
+    "train_runtime": 4331.1035,
+    "train_samples_per_second": 22.95,
+    "train_steps_per_second": 0.089
+}

trainer_log.jsonl

@@ -0,0 +1,386 @@
+{"current_steps": 1, "total_steps": 385, "loss": 8.1083, "learning_rate": 8.333333333333335e-09, "epoch": 0.012861736334405145, "percentage": 0.26, "elapsed_time": "0:00:11", "remaining_time": "1:15:00", "throughput": "1411.72", "total_tokens": 16544}
+{"current_steps": 2, "total_steps": 385, "loss": 8.1052, "learning_rate": 1.666666666666667e-08, "epoch": 0.02572347266881029, "percentage": 0.52, "elapsed_time": "0:00:22", "remaining_time": "1:12:54", "throughput": "1455.41", "total_tokens": 33248}
+{"current_steps": 3, "total_steps": 385, "loss": 8.1366, "learning_rate": 2.5000000000000002e-08, "epoch": 0.03858520900321544, "percentage": 0.78, "elapsed_time": "0:00:33", "remaining_time": "1:11:59", "throughput": "1466.88", "total_tokens": 49760}
+{"current_steps": 4, "total_steps": 385, "loss": 8.0413, "learning_rate": 3.333333333333334e-08, "epoch": 0.05144694533762058, "percentage": 1.04, "elapsed_time": "0:00:45", "remaining_time": "1:11:27", "throughput": "1462.41", "total_tokens": 65824}
+{"current_steps": 5, "total_steps": 385, "loss": 8.1055, "learning_rate": 4.166666666666667e-08, "epoch": 0.06430868167202572, "percentage": 1.3, "elapsed_time": "0:00:56", "remaining_time": "1:11:04", "throughput": "1471.37", "total_tokens": 82560}
+{"current_steps": 6, "total_steps": 385, "loss": 7.9666, "learning_rate": 5.0000000000000004e-08, "epoch": 0.07717041800643087, "percentage": 1.56, "elapsed_time": "0:01:07", "remaining_time": "1:10:47", "throughput": "1484.19", "total_tokens": 99808}
+{"current_steps": 7, "total_steps": 385, "loss": 8.0182, "learning_rate": 5.833333333333334e-08, "epoch": 0.09003215434083602, "percentage": 1.82, "elapsed_time": "0:01:18", "remaining_time": "1:10:31", "throughput": "1477.83", "total_tokens": 115808}
+{"current_steps": 8, "total_steps": 385, "loss": 8.0363, "learning_rate": 6.666666666666668e-08, "epoch": 0.10289389067524116, "percentage": 2.08, "elapsed_time": "0:01:29", "remaining_time": "1:10:17", "throughput": "1468.21", "total_tokens": 131392}
+{"current_steps": 9, "total_steps": 385, "loss": 8.0918, "learning_rate": 7.500000000000001e-08, "epoch": 0.1157556270096463, "percentage": 2.34, "elapsed_time": "0:01:40", "remaining_time": "1:10:04", "throughput": "1469.05", "total_tokens": 147840}
+{"current_steps": 10, "total_steps": 385, "loss": 8.0197, "learning_rate": 8.333333333333334e-08, "epoch": 0.12861736334405144, "percentage": 2.6, "elapsed_time": "0:01:51", "remaining_time": "1:09:51", "throughput": "1467.95", "total_tokens": 164064}
+{"current_steps": 11, "total_steps": 385, "loss": 7.9714, "learning_rate": 9.166666666666668e-08, "epoch": 0.1414790996784566, "percentage": 2.86, "elapsed_time": "0:02:02", "remaining_time": "1:09:38", "throughput": "1470.48", "total_tokens": 180704}
+{"current_steps": 12, "total_steps": 385, "loss": 8.0131, "learning_rate": 1.0000000000000001e-07, "epoch": 0.15434083601286175, "percentage": 3.12, "elapsed_time": "0:02:13", "remaining_time": "1:09:23", "throughput": "1463.77", "total_tokens": 196064}
+{"current_steps": 13, "total_steps": 385, "loss": 7.9796, "learning_rate": 1.0833333333333335e-07, "epoch": 0.16720257234726688, "percentage": 3.38, "elapsed_time": "0:02:25", "remaining_time": "1:09:09", "throughput": "1457.86", "total_tokens": 211424}
+{"current_steps": 14, "total_steps": 385, "loss": 7.9999, "learning_rate": 1.1666666666666668e-07, "epoch": 0.18006430868167203, "percentage": 3.64, "elapsed_time": "0:02:36", "remaining_time": "1:08:57", "throughput": "1460.84", "total_tokens": 228096}
+{"current_steps": 15, "total_steps": 385, "loss": 8.0168, "learning_rate": 1.2500000000000002e-07, "epoch": 0.19292604501607716, "percentage": 3.9, "elapsed_time": "0:02:47", "remaining_time": "1:08:46", "throughput": "1461.12", "total_tokens": 244416}
+{"current_steps": 16, "total_steps": 385, "loss": 7.8423, "learning_rate": 1.3333333333333336e-07, "epoch": 0.2057877813504823, "percentage": 4.16, "elapsed_time": "0:02:58", "remaining_time": "1:08:34", "throughput": "1463.26", "total_tokens": 261056}
+{"current_steps": 17, "total_steps": 385, "loss": 7.9208, "learning_rate": 1.4166666666666668e-07, "epoch": 0.21864951768488747, "percentage": 4.42, "elapsed_time": "0:03:09", "remaining_time": "1:08:23", "throughput": "1461.36", "total_tokens": 276992}
+{"current_steps": 18, "total_steps": 385, "loss": 7.6898, "learning_rate": 1.5000000000000002e-07, "epoch": 0.2315112540192926, "percentage": 4.68, "elapsed_time": "0:03:20", "remaining_time": "1:08:11", "throughput": "1462.71", "total_tokens": 293536}
+{"current_steps": 19, "total_steps": 385, "loss": 7.9285, "learning_rate": 1.5833333333333336e-07, "epoch": 0.24437299035369775, "percentage": 4.94, "elapsed_time": "0:03:31", "remaining_time": "1:07:59", "throughput": "1459.08", "total_tokens": 309024}
+{"current_steps": 20, "total_steps": 385, "loss": 7.8924, "learning_rate": 1.6666666666666668e-07, "epoch": 0.2572347266881029, "percentage": 5.19, "elapsed_time": "0:03:42", "remaining_time": "1:07:47", "throughput": "1458.15", "total_tokens": 325024}
+{"current_steps": 21, "total_steps": 385, "loss": 7.6173, "learning_rate": 1.7500000000000002e-07, "epoch": 0.27009646302250806, "percentage": 5.45, "elapsed_time": "0:03:54", "remaining_time": "1:07:36", "throughput": "1460.88", "total_tokens": 341856}
+{"current_steps": 22, "total_steps": 385, "loss": 7.7661, "learning_rate": 1.8333333333333336e-07, "epoch": 0.2829581993569132, "percentage": 5.71, "elapsed_time": "0:04:05", "remaining_time": "1:07:24", "throughput": "1463.97", "total_tokens": 358848}
+{"current_steps": 23, "total_steps": 385, "loss": 7.2936, "learning_rate": 1.9166666666666668e-07, "epoch": 0.2958199356913183, "percentage": 5.97, "elapsed_time": "0:04:16", "remaining_time": "1:07:12", "throughput": "1464.98", "total_tokens": 375360}
+{"current_steps": 24, "total_steps": 385, "loss": 7.3044, "learning_rate": 2.0000000000000002e-07, "epoch": 0.3086816720257235, "percentage": 6.23, "elapsed_time": "0:04:27", "remaining_time": "1:07:01", "throughput": "1468.54", "total_tokens": 392608}
+{"current_steps": 25, "total_steps": 385, "loss": 7.35, "learning_rate": 2.0833333333333333e-07, "epoch": 0.3215434083601286, "percentage": 6.49, "elapsed_time": "0:04:38", "remaining_time": "1:06:49", "throughput": "1468.11", "total_tokens": 408800}
+{"current_steps": 26, "total_steps": 385, "loss": 7.1213, "learning_rate": 2.166666666666667e-07, "epoch": 0.33440514469453375, "percentage": 6.75, "elapsed_time": "0:04:49", "remaining_time": "1:06:38", "throughput": "1469.84", "total_tokens": 425632}
+{"current_steps": 27, "total_steps": 385, "loss": 7.1363, "learning_rate": 2.2500000000000002e-07, "epoch": 0.34726688102893893, "percentage": 7.01, "elapsed_time": "0:05:00", "remaining_time": "1:06:26", "throughput": "1469.80", "total_tokens": 441952}
+{"current_steps": 28, "total_steps": 385, "loss": 6.9016, "learning_rate": 2.3333333333333336e-07, "epoch": 0.36012861736334406, "percentage": 7.27, "elapsed_time": "0:05:11", "remaining_time": "1:06:16", "throughput": "1468.87", "total_tokens": 458080}
+{"current_steps": 29, "total_steps": 385, "loss": 6.9764, "learning_rate": 2.416666666666667e-07, "epoch": 0.3729903536977492, "percentage": 7.53, "elapsed_time": "0:05:22", "remaining_time": "1:06:04", "throughput": "1467.16", "total_tokens": 473824}
+{"current_steps": 30, "total_steps": 385, "loss": 6.9005, "learning_rate": 2.5000000000000004e-07, "epoch": 0.3858520900321543, "percentage": 7.79, "elapsed_time": "0:05:34", "remaining_time": "1:05:53", "throughput": "1467.82", "total_tokens": 490336}
+{"current_steps": 31, "total_steps": 385, "loss": 6.9852, "learning_rate": 2.5833333333333333e-07, "epoch": 0.3987138263665595, "percentage": 8.05, "elapsed_time": "0:05:45", "remaining_time": "1:05:41", "throughput": "1468.73", "total_tokens": 506976}
+{"current_steps": 32, "total_steps": 385, "loss": 5.834, "learning_rate": 2.666666666666667e-07, "epoch": 0.4115755627009646, "percentage": 8.31, "elapsed_time": "0:05:56", "remaining_time": "1:05:30", "throughput": "1469.26", "total_tokens": 523520}
+{"current_steps": 33, "total_steps": 385, "loss": 5.5452, "learning_rate": 2.75e-07, "epoch": 0.42443729903536975, "percentage": 8.57, "elapsed_time": "0:06:07", "remaining_time": "1:05:19", "throughput": "1471.43", "total_tokens": 540672}
+{"current_steps": 34, "total_steps": 385, "loss": 5.5189, "learning_rate": 2.8333333333333336e-07, "epoch": 0.43729903536977494, "percentage": 8.83, "elapsed_time": "0:06:18", "remaining_time": "1:05:08", "throughput": "1473.12", "total_tokens": 557696}
+{"current_steps": 35, "total_steps": 385, "loss": 5.4654, "learning_rate": 2.916666666666667e-07, "epoch": 0.45016077170418006, "percentage": 9.09, "elapsed_time": "0:06:29", "remaining_time": "1:04:57", "throughput": "1473.94", "total_tokens": 574400}
+{"current_steps": 36, "total_steps": 385, "loss": 5.3117, "learning_rate": 3.0000000000000004e-07, "epoch": 0.4630225080385852, "percentage": 9.35, "elapsed_time": "0:06:40", "remaining_time": "1:04:45", "throughput": "1472.23", "total_tokens": 590112}
+{"current_steps": 37, "total_steps": 385, "loss": 5.2744, "learning_rate": 3.083333333333334e-07, "epoch": 0.4758842443729904, "percentage": 9.61, "elapsed_time": "0:06:51", "remaining_time": "1:04:34", "throughput": "1470.87", "total_tokens": 605920}
+{"current_steps": 38, "total_steps": 385, "loss": 5.2821, "learning_rate": 3.166666666666667e-07, "epoch": 0.4887459807073955, "percentage": 9.87, "elapsed_time": "0:07:03", "remaining_time": "1:04:23", "throughput": "1470.77", "total_tokens": 622208}
+{"current_steps": 39, "total_steps": 385, "loss": 5.0967, "learning_rate": 3.25e-07, "epoch": 0.5016077170418006, "percentage": 10.13, "elapsed_time": "0:07:14", "remaining_time": "1:04:11", "throughput": "1470.64", "total_tokens": 638464}
+{"current_steps": 40, "total_steps": 385, "loss": 5.2167, "learning_rate": 3.3333333333333335e-07, "epoch": 0.5144694533762058, "percentage": 10.39, "elapsed_time": "0:07:25", "remaining_time": "1:04:00", "throughput": "1470.41", "total_tokens": 654656}
+{"current_steps": 41, "total_steps": 385, "loss": 5.1517, "learning_rate": 3.416666666666667e-07, "epoch": 0.5273311897106109, "percentage": 10.65, "elapsed_time": "0:07:36", "remaining_time": "1:03:49", "throughput": "1471.24", "total_tokens": 671424}
+{"current_steps": 42, "total_steps": 385, "loss": 4.6959, "learning_rate": 3.5000000000000004e-07, "epoch": 0.5401929260450161, "percentage": 10.91, "elapsed_time": "0:07:47", "remaining_time": "1:03:37", "throughput": "1471.59", "total_tokens": 687968}
+{"current_steps": 43, "total_steps": 385, "loss": 4.1458, "learning_rate": 3.583333333333334e-07, "epoch": 0.5530546623794212, "percentage": 11.17, "elapsed_time": "0:07:58", "remaining_time": "1:03:26", "throughput": "1471.34", "total_tokens": 704224}
+{"current_steps": 44, "total_steps": 385, "loss": 3.6692, "learning_rate": 3.666666666666667e-07, "epoch": 0.5659163987138264, "percentage": 11.43, "elapsed_time": "0:08:09", "remaining_time": "1:03:15", "throughput": "1471.55", "total_tokens": 720704}
+{"current_steps": 45, "total_steps": 385, "loss": 3.3322, "learning_rate": 3.75e-07, "epoch": 0.5787781350482315, "percentage": 11.69, "elapsed_time": "0:08:20", "remaining_time": "1:03:04", "throughput": "1472.18", "total_tokens": 737408}
+{"current_steps": 46, "total_steps": 385, "loss": 3.2857, "learning_rate": 3.8333333333333335e-07, "epoch": 0.5916398713826366, "percentage": 11.95, "elapsed_time": "0:08:32", "remaining_time": "1:02:53", "throughput": "1472.04", "total_tokens": 753728}
+{"current_steps": 47, "total_steps": 385, "loss": 3.0806, "learning_rate": 3.9166666666666675e-07, "epoch": 0.6045016077170418, "percentage": 12.21, "elapsed_time": "0:08:43", "remaining_time": "1:02:42", "throughput": "1472.06", "total_tokens": 770080}
+{"current_steps": 48, "total_steps": 385, "loss": 2.8239, "learning_rate": 4.0000000000000003e-07, "epoch": 0.617363344051447, "percentage": 12.47, "elapsed_time": "0:08:54", "remaining_time": "1:02:30", "throughput": "1472.90", "total_tokens": 786912}
+{"current_steps": 49, "total_steps": 385, "loss": 2.8695, "learning_rate": 4.083333333333334e-07, "epoch": 0.6302250803858521, "percentage": 12.73, "elapsed_time": "0:09:05", "remaining_time": "1:02:19", "throughput": "1474.53", "total_tokens": 804192}
+{"current_steps": 50, "total_steps": 385, "loss": 2.6218, "learning_rate": 4.1666666666666667e-07, "epoch": 0.6430868167202572, "percentage": 12.99, "elapsed_time": "0:09:16", "remaining_time": "1:02:08", "throughput": "1475.14", "total_tokens": 820928}
+{"current_steps": 51, "total_steps": 385, "loss": 2.3993, "learning_rate": 4.2500000000000006e-07, "epoch": 0.6559485530546624, "percentage": 13.25, "elapsed_time": "0:09:27", "remaining_time": "1:01:57", "throughput": "1474.86", "total_tokens": 837152}
+{"current_steps": 52, "total_steps": 385, "loss": 2.2828, "learning_rate": 4.333333333333334e-07, "epoch": 0.6688102893890675, "percentage": 13.51, "elapsed_time": "0:09:38", "remaining_time": "1:01:46", "throughput": "1474.81", "total_tokens": 853568}
+{"current_steps": 53, "total_steps": 385, "loss": 2.2063, "learning_rate": 4.416666666666667e-07, "epoch": 0.6816720257234726, "percentage": 13.77, "elapsed_time": "0:09:49", "remaining_time": "1:01:35", "throughput": "1475.84", "total_tokens": 870592}
+{"current_steps": 54, "total_steps": 385, "loss": 1.8878, "learning_rate": 4.5000000000000003e-07, "epoch": 0.6945337620578779, "percentage": 14.03, "elapsed_time": "0:10:01", "remaining_time": "1:01:24", "throughput": "1474.30", "total_tokens": 886080}
+{"current_steps": 55, "total_steps": 385, "loss": 1.5971, "learning_rate": 4.583333333333333e-07, "epoch": 0.707395498392283, "percentage": 14.29, "elapsed_time": "0:10:12", "remaining_time": "1:01:12", "throughput": "1475.65", "total_tokens": 903296}
+{"current_steps": 56, "total_steps": 385, "loss": 0.9604, "learning_rate": 4.666666666666667e-07, "epoch": 0.7202572347266881, "percentage": 14.55, "elapsed_time": "0:10:23", "remaining_time": "1:01:01", "throughput": "1475.05", "total_tokens": 919296}
+{"current_steps": 57, "total_steps": 385, "loss": 0.6287, "learning_rate": 4.7500000000000006e-07, "epoch": 0.7331189710610932, "percentage": 14.81, "elapsed_time": "0:10:34", "remaining_time": "1:00:50", "throughput": "1475.70", "total_tokens": 936096}
+{"current_steps": 58, "total_steps": 385, "loss": 0.4844, "learning_rate": 4.833333333333334e-07, "epoch": 0.7459807073954984, "percentage": 15.06, "elapsed_time": "0:10:45", "remaining_time": "1:00:39", "throughput": "1476.45", "total_tokens": 953024}
+{"current_steps": 59, "total_steps": 385, "loss": 0.3891, "learning_rate": 4.916666666666667e-07, "epoch": 0.7588424437299035, "percentage": 15.32, "elapsed_time": "0:10:56", "remaining_time": "1:00:27", "throughput": "1475.83", "total_tokens": 969024}
+{"current_steps": 60, "total_steps": 385, "loss": 0.357, "learning_rate": 5.000000000000001e-07, "epoch": 0.7717041800643086, "percentage": 15.58, "elapsed_time": "0:11:07", "remaining_time": "1:00:16", "throughput": "1476.05", "total_tokens": 985600}
+{"current_steps": 61, "total_steps": 385, "loss": 0.2854, "learning_rate": 5.083333333333334e-07, "epoch": 0.7845659163987139, "percentage": 15.84, "elapsed_time": "0:11:18", "remaining_time": "1:00:05", "throughput": "1476.53", "total_tokens": 1002336}
+{"current_steps": 62, "total_steps": 385, "loss": 0.2403, "learning_rate": 5.166666666666667e-07, "epoch": 0.797427652733119, "percentage": 16.1, "elapsed_time": "0:11:29", "remaining_time": "0:59:54", "throughput": "1476.18", "total_tokens": 1018496}
+{"current_steps": 63, "total_steps": 385, "loss": 0.2522, "learning_rate": 5.250000000000001e-07, "epoch": 0.8102893890675241, "percentage": 16.36, "elapsed_time": "0:11:41", "remaining_time": "0:59:43", "throughput": "1477.05", "total_tokens": 1035552}
+{"current_steps": 64, "total_steps": 385, "loss": 0.2339, "learning_rate": 5.333333333333335e-07, "epoch": 0.8231511254019293, "percentage": 16.62, "elapsed_time": "0:11:52", "remaining_time": "0:59:32", "throughput": "1476.51", "total_tokens": 1051584}
+{"current_steps": 65, "total_steps": 385, "loss": 0.2214, "learning_rate": 5.416666666666667e-07, "epoch": 0.8360128617363344, "percentage": 16.88, "elapsed_time": "0:12:03", "remaining_time": "0:59:20", "throughput": "1477.43", "total_tokens": 1068640}
+{"current_steps": 66, "total_steps": 385, "loss": 0.1912, "learning_rate": 5.5e-07, "epoch": 0.8488745980707395, "percentage": 17.14, "elapsed_time": "0:12:14", "remaining_time": "0:59:09", "throughput": "1478.13", "total_tokens": 1085568}
+{"current_steps": 67, "total_steps": 385, "loss": 0.2157, "learning_rate": 5.583333333333333e-07, "epoch": 0.8617363344051447, "percentage": 17.4, "elapsed_time": "0:12:25", "remaining_time": "0:58:58", "throughput": "1479.16", "total_tokens": 1102784}
+{"current_steps": 68, "total_steps": 385, "loss": 0.2045, "learning_rate": 5.666666666666667e-07, "epoch": 0.8745980707395499, "percentage": 17.66, "elapsed_time": "0:12:36", "remaining_time": "0:58:47", "throughput": "1478.13", "total_tokens": 1118432}
+{"current_steps": 69, "total_steps": 385, "loss": 0.1964, "learning_rate": 5.750000000000001e-07, "epoch": 0.887459807073955, "percentage": 17.92, "elapsed_time": "0:12:47", "remaining_time": "0:58:36", "throughput": "1478.54", "total_tokens": 1135232}
+{"current_steps": 70, "total_steps": 385, "loss": 0.1901, "learning_rate": 5.833333333333334e-07, "epoch": 0.9003215434083601, "percentage": 18.18, "elapsed_time": "0:12:58", "remaining_time": "0:58:25", "throughput": "1477.07", "total_tokens": 1150496}
+{"current_steps": 71, "total_steps": 385, "loss": 0.1891, "learning_rate": 5.916666666666667e-07, "epoch": 0.9131832797427653, "percentage": 18.44, "elapsed_time": "0:13:10", "remaining_time": "0:58:13", "throughput": "1477.00", "total_tokens": 1166848}
+{"current_steps": 72, "total_steps": 385, "loss": 0.1841, "learning_rate": 6.000000000000001e-07, "epoch": 0.9260450160771704, "percentage": 18.7, "elapsed_time": "0:13:21", "remaining_time": "0:58:02", "throughput": "1477.26", "total_tokens": 1183552}
+{"current_steps": 73, "total_steps": 385, "loss": 0.1803, "learning_rate": 6.083333333333334e-07, "epoch": 0.9389067524115756, "percentage": 18.96, "elapsed_time": "0:13:32", "remaining_time": "0:57:51", "throughput": "1477.28", "total_tokens": 1199968}
+{"current_steps": 74, "total_steps": 385, "loss": 0.1657, "learning_rate": 6.166666666666668e-07, "epoch": 0.9517684887459807, "percentage": 19.22, "elapsed_time": "0:13:43", "remaining_time": "0:57:40", "throughput": "1477.17", "total_tokens": 1216256}
+{"current_steps": 75, "total_steps": 385, "loss": 0.2157, "learning_rate": 6.25e-07, "epoch": 0.9646302250803859, "percentage": 19.48, "elapsed_time": "0:13:54", "remaining_time": "0:57:29", "throughput": "1477.26", "total_tokens": 1232736}
+{"current_steps": 76, "total_steps": 385, "loss": 0.2, "learning_rate": 6.333333333333334e-07, "epoch": 0.977491961414791, "percentage": 19.74, "elapsed_time": "0:14:05", "remaining_time": "0:57:17", "throughput": "1477.40", "total_tokens": 1249248}
+{"current_steps": 77, "total_steps": 385, "loss": 0.1519, "learning_rate": 6.416666666666667e-07, "epoch": 0.9903536977491961, "percentage": 20.0, "elapsed_time": "0:14:16", "remaining_time": "0:57:06", "throughput": "1476.86", "total_tokens": 1265152}
+{"current_steps": 78, "total_steps": 385, "loss": 0.1765, "learning_rate": 6.5e-07, "epoch": 1.0032154340836013, "percentage": 20.26, "elapsed_time": "0:14:27", "remaining_time": "0:56:55", "throughput": "1477.71", "total_tokens": 1282336}
+{"current_steps": 79, "total_steps": 385, "loss": 0.1907, "learning_rate": 6.583333333333333e-07, "epoch": 1.0160771704180065, "percentage": 20.52, "elapsed_time": "0:14:38", "remaining_time": "0:56:44", "throughput": "1478.10", "total_tokens": 1299168}
+{"current_steps": 80, "total_steps": 385, "loss": 0.1593, "learning_rate": 6.666666666666667e-07, "epoch": 1.0289389067524115, "percentage": 20.78, "elapsed_time": "0:14:50", "remaining_time": "0:56:33", "throughput": "1477.21", "total_tokens": 1314816}
+{"current_steps": 81, "total_steps": 385, "loss": 0.1659, "learning_rate": 6.750000000000001e-07, "epoch": 1.0418006430868167, "percentage": 21.04, "elapsed_time": "0:15:01", "remaining_time": "0:56:22", "throughput": "1477.14", "total_tokens": 1331168}
+{"current_steps": 82, "total_steps": 385, "loss": 0.1619, "learning_rate": 6.833333333333334e-07, "epoch": 1.0546623794212218, "percentage": 21.3, "elapsed_time": "0:15:12", "remaining_time": "0:56:10", "throughput": "1477.58", "total_tokens": 1347968}
+{"current_steps": 83, "total_steps": 385, "loss": 0.168, "learning_rate": 6.916666666666668e-07, "epoch": 1.067524115755627, "percentage": 21.56, "elapsed_time": "0:15:23", "remaining_time": "0:55:59", "throughput": "1477.08", "total_tokens": 1363872}
+{"current_steps": 84, "total_steps": 385, "loss": 0.1627, "learning_rate": 7.000000000000001e-07, "epoch": 1.0803858520900322, "percentage": 21.82, "elapsed_time": "0:15:34", "remaining_time": "0:55:48", "throughput": "1477.67", "total_tokens": 1380896}
+{"current_steps": 85, "total_steps": 385, "loss": 0.1421, "learning_rate": 7.083333333333334e-07, "epoch": 1.0932475884244373, "percentage": 22.08, "elapsed_time": "0:15:45", "remaining_time": "0:55:37", "throughput": "1478.13", "total_tokens": 1397792}
+{"current_steps": 86, "total_steps": 385, "loss": 0.1517, "learning_rate": 7.166666666666668e-07, "epoch": 1.1061093247588425, "percentage": 22.34, "elapsed_time": "0:15:56", "remaining_time": "0:55:26", "throughput": "1478.20", "total_tokens": 1414272}
+{"current_steps": 87, "total_steps": 385, "loss": 0.1519, "learning_rate": 7.25e-07, "epoch": 1.1189710610932475, "percentage": 22.6, "elapsed_time": "0:16:07", "remaining_time": "0:55:15", "throughput": "1477.17", "total_tokens": 1429696}
+{"current_steps": 88, "total_steps": 385, "loss": 0.128, "learning_rate": 7.333333333333334e-07, "epoch": 1.1318327974276527, "percentage": 22.86, "elapsed_time": "0:16:19", "remaining_time": "0:55:04", "throughput": "1477.75", "total_tokens": 1446720}
+{"current_steps": 89, "total_steps": 385, "loss": 0.1481, "learning_rate": 7.416666666666668e-07, "epoch": 1.144694533762058, "percentage": 23.12, "elapsed_time": "0:16:30", "remaining_time": "0:54:53", "throughput": "1477.59", "total_tokens": 1463008}
+{"current_steps": 90, "total_steps": 385, "loss": 0.1636, "learning_rate": 7.5e-07, "epoch": 1.157556270096463, "percentage": 23.38, "elapsed_time": "0:16:41", "remaining_time": "0:54:41", "throughput": "1478.21", "total_tokens": 1480096}
+{"current_steps": 91, "total_steps": 385, "loss": 0.1443, "learning_rate": 7.583333333333334e-07, "epoch": 1.1704180064308682, "percentage": 23.64, "elapsed_time": "0:16:52", "remaining_time": "0:54:30", "throughput": "1478.70", "total_tokens": 1497024}
+{"current_steps": 92, "total_steps": 385, "loss": 0.1592, "learning_rate": 7.666666666666667e-07, "epoch": 1.1832797427652733, "percentage": 23.9, "elapsed_time": "0:17:03", "remaining_time": "0:54:19", "throughput": "1479.03", "total_tokens": 1513792}
+{"current_steps": 93, "total_steps": 385, "loss": 0.1744, "learning_rate": 7.750000000000001e-07, "epoch": 1.1961414790996785, "percentage": 24.16, "elapsed_time": "0:17:14", "remaining_time": "0:54:08", "throughput": "1478.61", "total_tokens": 1529696}
+{"current_steps": 94, "total_steps": 385, "loss": 0.1615, "learning_rate": 7.833333333333335e-07, "epoch": 1.2090032154340835, "percentage": 24.42, "elapsed_time": "0:17:25", "remaining_time": "0:53:57", "throughput": "1478.86", "total_tokens": 1546400}
+{"current_steps": 95, "total_steps": 385, "loss": 0.1466, "learning_rate": 7.916666666666667e-07, "epoch": 1.2218649517684887, "percentage": 24.68, "elapsed_time": "0:17:36", "remaining_time": "0:53:46", "throughput": "1480.34", "total_tokens": 1564448}
+{"current_steps": 96, "total_steps": 385, "loss": 0.1333, "learning_rate": 8.000000000000001e-07, "epoch": 1.234726688102894, "percentage": 24.94, "elapsed_time": "0:17:47", "remaining_time": "0:53:34", "throughput": "1480.18", "total_tokens": 1580736}
+{"current_steps": 97, "total_steps": 385, "loss": 0.1475, "learning_rate": 8.083333333333334e-07, "epoch": 1.247588424437299, "percentage": 25.19, "elapsed_time": "0:17:59", "remaining_time": "0:53:23", "throughput": "1479.91", "total_tokens": 1596928}
+{"current_steps": 98, "total_steps": 385, "loss": 0.142, "learning_rate": 8.166666666666668e-07, "epoch": 1.2604501607717042, "percentage": 25.45, "elapsed_time": "0:18:10", "remaining_time": "0:53:12", "throughput": "1480.14", "total_tokens": 1613728}
+{"current_steps": 99, "total_steps": 385, "loss": 0.148, "learning_rate": 8.250000000000001e-07, "epoch": 1.2733118971061093, "percentage": 25.71, "elapsed_time": "0:18:21", "remaining_time": "0:53:01", "throughput": "1480.08", "total_tokens": 1630112}
+{"current_steps": 100, "total_steps": 385, "loss": 0.1539, "learning_rate": 8.333333333333333e-07, "epoch": 1.2861736334405145, "percentage": 25.97, "elapsed_time": "0:18:32", "remaining_time": "0:52:50", "throughput": "1478.98", "total_tokens": 1645248}
+{"current_steps": 101, "total_steps": 385, "loss": 0.1496, "learning_rate": 8.416666666666667e-07, "epoch": 1.2990353697749195, "percentage": 26.23, "elapsed_time": "0:18:43", "remaining_time": "0:52:39", "throughput": "1479.63", "total_tokens": 1662400}
+{"current_steps": 102, "total_steps": 385, "loss": 0.1563, "learning_rate": 8.500000000000001e-07, "epoch": 1.3118971061093248, "percentage": 26.49, "elapsed_time": "0:18:54", "remaining_time": "0:52:28", "throughput": "1479.38", "total_tokens": 1678528}
+{"current_steps": 103, "total_steps": 385, "loss": 0.1295, "learning_rate": 8.583333333333334e-07, "epoch": 1.32475884244373, "percentage": 26.75, "elapsed_time": "0:19:05", "remaining_time": "0:52:16", "throughput": "1478.95", "total_tokens": 1694432}
+{"current_steps": 104, "total_steps": 385, "loss": 0.1353, "learning_rate": 8.666666666666668e-07, "epoch": 1.337620578778135, "percentage": 27.01, "elapsed_time": "0:19:16", "remaining_time": "0:52:05", "throughput": "1478.71", "total_tokens": 1710624}
+{"current_steps": 105, "total_steps": 385, "loss": 0.1376, "learning_rate": 8.75e-07, "epoch": 1.3504823151125402, "percentage": 27.27, "elapsed_time": "0:19:27", "remaining_time": "0:51:54", "throughput": "1478.25", "total_tokens": 1726560}
+{"current_steps": 106, "total_steps": 385, "loss": 0.1313, "learning_rate": 8.833333333333334e-07, "epoch": 1.3633440514469453, "percentage": 27.53, "elapsed_time": "0:19:39", "remaining_time": "0:51:43", "throughput": "1478.13", "total_tokens": 1742912}
+{"current_steps": 107, "total_steps": 385, "loss": 0.1367, "learning_rate": 8.916666666666668e-07, "epoch": 1.3762057877813505, "percentage": 27.79, "elapsed_time": "0:19:50", "remaining_time": "0:51:32", "throughput": "1477.58", "total_tokens": 1758688}
+{"current_steps": 108, "total_steps": 385, "loss": 0.135, "learning_rate": 9.000000000000001e-07, "epoch": 1.3890675241157555, "percentage": 28.05, "elapsed_time": "0:20:01", "remaining_time": "0:51:21", "throughput": "1477.67", "total_tokens": 1775264}
+{"current_steps": 109, "total_steps": 385, "loss": 0.1212, "learning_rate": 9.083333333333335e-07, "epoch": 1.4019292604501608, "percentage": 28.31, "elapsed_time": "0:20:12", "remaining_time": "0:51:10", "throughput": "1477.90", "total_tokens": 1791904}
+{"current_steps": 110, "total_steps": 385, "loss": 0.1355, "learning_rate": 9.166666666666666e-07, "epoch": 1.414790996784566, "percentage": 28.57, "elapsed_time": "0:20:23", "remaining_time": "0:50:58", "throughput": "1478.26", "total_tokens": 1808736}
+{"current_steps": 111, "total_steps": 385, "loss": 0.1439, "learning_rate": 9.25e-07, "epoch": 1.427652733118971, "percentage": 28.83, "elapsed_time": "0:20:34", "remaining_time": "0:50:47", "throughput": "1478.53", "total_tokens": 1825504}
+{"current_steps": 112, "total_steps": 385, "loss": 0.1323, "learning_rate": 9.333333333333334e-07, "epoch": 1.4405144694533762, "percentage": 29.09, "elapsed_time": "0:20:45", "remaining_time": "0:50:36", "throughput": "1478.60", "total_tokens": 1842080}
+{"current_steps": 113, "total_steps": 385, "loss": 0.1376, "learning_rate": 9.416666666666667e-07, "epoch": 1.4533762057877815, "percentage": 29.35, "elapsed_time": "0:20:56", "remaining_time": "0:50:25", "throughput": "1478.52", "total_tokens": 1858432}
+{"current_steps": 114, "total_steps": 385, "loss": 0.1191, "learning_rate": 9.500000000000001e-07, "epoch": 1.4662379421221865, "percentage": 29.61, "elapsed_time": "0:21:08", "remaining_time": "0:50:14", "throughput": "1478.08", "total_tokens": 1874304}
+{"current_steps": 115, "total_steps": 385, "loss": 0.1045, "learning_rate": 9.583333333333334e-07, "epoch": 1.4790996784565915, "percentage": 29.87, "elapsed_time": "0:21:19", "remaining_time": "0:50:03", "throughput": "1477.61", "total_tokens": 1890144}
+{"current_steps": 116, "total_steps": 385, "loss": 0.1467, "learning_rate": 9.666666666666668e-07, "epoch": 1.4919614147909968, "percentage": 30.13, "elapsed_time": "0:21:30", "remaining_time": "0:49:52", "throughput": "1476.92", "total_tokens": 1905664}
+{"current_steps": 117, "total_steps": 385, "loss": 0.1142, "learning_rate": 9.750000000000002e-07, "epoch": 1.504823151125402, "percentage": 30.39, "elapsed_time": "0:21:41", "remaining_time": "0:49:41", "throughput": "1477.03", "total_tokens": 1922240}
+{"current_steps": 118, "total_steps": 385, "loss": 0.1107, "learning_rate": 9.833333333333334e-07, "epoch": 1.517684887459807, "percentage": 30.65, "elapsed_time": "0:21:52", "remaining_time": "0:49:29", "throughput": "1476.73", "total_tokens": 1938240}
+{"current_steps": 119, "total_steps": 385, "loss": 0.146, "learning_rate": 9.916666666666668e-07, "epoch": 1.5305466237942122, "percentage": 30.91, "elapsed_time": "0:22:03", "remaining_time": "0:49:18", "throughput": "1477.00", "total_tokens": 1955008}
+{"current_steps": 120, "total_steps": 385, "loss": 0.1533, "learning_rate": 1.0000000000000002e-06, "epoch": 1.5434083601286175, "percentage": 31.17, "elapsed_time": "0:22:14", "remaining_time": "0:49:07", "throughput": "1477.02", "total_tokens": 1971456}
+{"current_steps": 121, "total_steps": 385, "loss": 0.1315, "learning_rate": 1.0083333333333333e-06, "epoch": 1.5562700964630225, "percentage": 31.43, "elapsed_time": "0:22:25", "remaining_time": "0:48:56", "throughput": "1476.81", "total_tokens": 1987552}
+{"current_steps": 122, "total_steps": 385, "loss": 0.1197, "learning_rate": 1.0166666666666667e-06, "epoch": 1.5691318327974275, "percentage": 31.69, "elapsed_time": "0:22:37", "remaining_time": "0:48:45", "throughput": "1476.92", "total_tokens": 2004192}
+{"current_steps": 123, "total_steps": 385, "loss": 0.1228, "learning_rate": 1.025e-06, "epoch": 1.5819935691318328, "percentage": 31.95, "elapsed_time": "0:22:48", "remaining_time": "0:48:34", "throughput": "1477.06", "total_tokens": 2020832}
+{"current_steps": 124, "total_steps": 385, "loss": 0.1273, "learning_rate": 1.0333333333333333e-06, "epoch": 1.594855305466238, "percentage": 32.21, "elapsed_time": "0:22:59", "remaining_time": "0:48:23", "throughput": "1477.06", "total_tokens": 2037312}
+{"current_steps": 125, "total_steps": 385, "loss": 0.141, "learning_rate": 1.0416666666666667e-06, "epoch": 1.607717041800643, "percentage": 32.47, "elapsed_time": "0:23:10", "remaining_time": "0:48:12", "throughput": "1477.45", "total_tokens": 2054304}
+{"current_steps": 126, "total_steps": 385, "loss": 0.1315, "learning_rate": 1.0500000000000001e-06, "epoch": 1.6205787781350482, "percentage": 32.73, "elapsed_time": "0:23:21", "remaining_time": "0:48:00", "throughput": "1477.73", "total_tokens": 2071136}
+{"current_steps": 127, "total_steps": 385, "loss": 0.1136, "learning_rate": 1.0583333333333335e-06, "epoch": 1.6334405144694535, "percentage": 32.99, "elapsed_time": "0:23:32", "remaining_time": "0:47:49", "throughput": "1478.09", "total_tokens": 2088064}
+{"current_steps": 128, "total_steps": 385, "loss": 0.1013, "learning_rate": 1.066666666666667e-06, "epoch": 1.6463022508038585, "percentage": 33.25, "elapsed_time": "0:23:43", "remaining_time": "0:47:38", "throughput": "1478.54", "total_tokens": 2105120}
+{"current_steps": 129, "total_steps": 385, "loss": 0.1056, "learning_rate": 1.075e-06, "epoch": 1.6591639871382635, "percentage": 33.51, "elapsed_time": "0:23:54", "remaining_time": "0:47:27", "throughput": "1478.14", "total_tokens": 2120992}
+{"current_steps": 130, "total_steps": 385, "loss": 0.1071, "learning_rate": 1.0833333333333335e-06, "epoch": 1.6720257234726688, "percentage": 33.77, "elapsed_time": "0:24:06", "remaining_time": "0:47:16", "throughput": "1478.54", "total_tokens": 2137984}
+{"current_steps": 131, "total_steps": 385, "loss": 0.1357, "learning_rate": 1.0916666666666667e-06, "epoch": 1.684887459807074, "percentage": 34.03, "elapsed_time": "0:24:17", "remaining_time": "0:47:05", "throughput": "1478.41", "total_tokens": 2154240}
+{"current_steps": 132, "total_steps": 385, "loss": 0.1181, "learning_rate": 1.1e-06, "epoch": 1.697749196141479, "percentage": 34.29, "elapsed_time": "0:24:28", "remaining_time": "0:46:54", "throughput": "1478.26", "total_tokens": 2170464}
+{"current_steps": 133, "total_steps": 385, "loss": 0.0826, "learning_rate": 1.1083333333333335e-06, "epoch": 1.7106109324758842, "percentage": 34.55, "elapsed_time": "0:24:39", "remaining_time": "0:46:43", "throughput": "1478.30", "total_tokens": 2187008}
+{"current_steps": 134, "total_steps": 385, "loss": 0.1221, "learning_rate": 1.1166666666666666e-06, "epoch": 1.7234726688102895, "percentage": 34.81, "elapsed_time": "0:24:50", "remaining_time": "0:46:31", "throughput": "1477.92", "total_tokens": 2202880}
+{"current_steps": 135, "total_steps": 385, "loss": 0.1021, "learning_rate": 1.125e-06, "epoch": 1.7363344051446945, "percentage": 35.06, "elapsed_time": "0:25:01", "remaining_time": "0:46:20", "throughput": "1478.20", "total_tokens": 2219744}
+{"current_steps": 136, "total_steps": 385, "loss": 0.098, "learning_rate": 1.1333333333333334e-06, "epoch": 1.7491961414790995, "percentage": 35.32, "elapsed_time": "0:25:12", "remaining_time": "0:46:09", "throughput": "1478.17", "total_tokens": 2236064}
+{"current_steps": 137, "total_steps": 385, "loss": 0.1085, "learning_rate": 1.1416666666666668e-06, "epoch": 1.762057877813505, "percentage": 35.58, "elapsed_time": "0:25:23", "remaining_time": "0:45:58", "throughput": "1478.40", "total_tokens": 2252832}
+{"current_steps": 138, "total_steps": 385, "loss": 0.1038, "learning_rate": 1.1500000000000002e-06, "epoch": 1.77491961414791, "percentage": 35.84, "elapsed_time": "0:25:34", "remaining_time": "0:45:47", "throughput": "1478.07", "total_tokens": 2268704}
+{"current_steps": 139, "total_steps": 385, "loss": 0.1081, "learning_rate": 1.1583333333333334e-06, "epoch": 1.787781350482315, "percentage": 36.1, "elapsed_time": "0:25:46", "remaining_time": "0:45:36", "throughput": "1478.19", "total_tokens": 2285280}
+{"current_steps": 140, "total_steps": 385, "loss": 0.1287, "learning_rate": 1.1666666666666668e-06, "epoch": 1.8006430868167203, "percentage": 36.36, "elapsed_time": "0:25:57", "remaining_time": "0:45:24", "throughput": "1478.10", "total_tokens": 2301600}
+{"current_steps": 141, "total_steps": 385, "loss": 0.1068, "learning_rate": 1.175e-06, "epoch": 1.8135048231511255, "percentage": 36.62, "elapsed_time": "0:26:08", "remaining_time": "0:45:13", "throughput": "1478.11", "total_tokens": 2318048}
+{"current_steps": 142, "total_steps": 385, "loss": 0.1202, "learning_rate": 1.1833333333333334e-06, "epoch": 1.8263665594855305, "percentage": 36.88, "elapsed_time": "0:26:19", "remaining_time": "0:45:02", "throughput": "1477.67", "total_tokens": 2333728}
+{"current_steps": 143, "total_steps": 385, "loss": 0.119, "learning_rate": 1.1916666666666668e-06, "epoch": 1.8392282958199357, "percentage": 37.14, "elapsed_time": "0:26:30", "remaining_time": "0:44:51", "throughput": "1477.42", "total_tokens": 2349760}
+{"current_steps": 144, "total_steps": 385, "loss": 0.1273, "learning_rate": 1.2000000000000002e-06, "epoch": 1.852090032154341, "percentage": 37.4, "elapsed_time": "0:26:41", "remaining_time": "0:44:40", "throughput": "1477.51", "total_tokens": 2366336}
+{"current_steps": 145, "total_steps": 385, "loss": 0.1024, "learning_rate": 1.2083333333333333e-06, "epoch": 1.864951768488746, "percentage": 37.66, "elapsed_time": "0:26:52", "remaining_time": "0:44:29", "throughput": "1477.11", "total_tokens": 2382016}
+{"current_steps": 146, "total_steps": 385, "loss": 0.1143, "learning_rate": 1.2166666666666667e-06, "epoch": 1.877813504823151, "percentage": 37.92, "elapsed_time": "0:27:03", "remaining_time": "0:44:17", "throughput": "1476.97", "total_tokens": 2398176}
+{"current_steps": 147, "total_steps": 385, "loss": 0.1229, "learning_rate": 1.2250000000000001e-06, "epoch": 1.8906752411575563, "percentage": 38.18, "elapsed_time": "0:27:14", "remaining_time": "0:44:06", "throughput": "1477.01", "total_tokens": 2414656}
+{"current_steps": 148, "total_steps": 385, "loss": 0.0964, "learning_rate": 1.2333333333333335e-06, "epoch": 1.9035369774919615, "percentage": 38.44, "elapsed_time": "0:27:25", "remaining_time": "0:43:55", "throughput": "1476.61", "total_tokens": 2430400}
+{"current_steps": 149, "total_steps": 385, "loss": 0.1132, "learning_rate": 1.2416666666666667e-06, "epoch": 1.9163987138263665, "percentage": 38.7, "elapsed_time": "0:27:37", "remaining_time": "0:43:44", "throughput": "1477.03", "total_tokens": 2447520}
+{"current_steps": 150, "total_steps": 385, "loss": 0.0734, "learning_rate": 1.25e-06, "epoch": 1.9292604501607717, "percentage": 38.96, "elapsed_time": "0:27:48", "remaining_time": "0:43:33", "throughput": "1476.98", "total_tokens": 2463872}
+{"current_steps": 151, "total_steps": 385, "loss": 0.0939, "learning_rate": 1.2583333333333333e-06, "epoch": 1.942122186495177, "percentage": 39.22, "elapsed_time": "0:27:59", "remaining_time": "0:43:22", "throughput": "1477.57", "total_tokens": 2481344}
+{"current_steps": 152, "total_steps": 385, "loss": 0.1143, "learning_rate": 1.2666666666666669e-06, "epoch": 1.954983922829582, "percentage": 39.48, "elapsed_time": "0:28:10", "remaining_time": "0:43:11", "throughput": "1477.48", "total_tokens": 2497632}
+{"current_steps": 153, "total_steps": 385, "loss": 0.1114, "learning_rate": 1.275e-06, "epoch": 1.967845659163987, "percentage": 39.74, "elapsed_time": "0:28:21", "remaining_time": "0:43:00", "throughput": "1476.92", "total_tokens": 2513024}
+{"current_steps": 154, "total_steps": 385, "loss": 0.0948, "learning_rate": 1.2833333333333335e-06, "epoch": 1.9807073954983923, "percentage": 40.0, "elapsed_time": "0:28:32", "remaining_time": "0:42:48", "throughput": "1476.58", "total_tokens": 2528800}
+{"current_steps": 155, "total_steps": 385, "loss": 0.0805, "learning_rate": 1.2916666666666669e-06, "epoch": 1.9935691318327975, "percentage": 40.26, "elapsed_time": "0:28:43", "remaining_time": "0:42:37", "throughput": "1476.85", "total_tokens": 2545696}
+{"current_steps": 156, "total_steps": 385, "loss": 0.1001, "learning_rate": 1.3e-06, "epoch": 2.0064308681672025, "percentage": 40.52, "elapsed_time": "0:28:54", "remaining_time": "0:42:26", "throughput": "1476.62", "total_tokens": 2561728}
+{"current_steps": 157, "total_steps": 385, "loss": 0.1002, "learning_rate": 1.3083333333333334e-06, "epoch": 2.0192926045016075, "percentage": 40.78, "elapsed_time": "0:29:05", "remaining_time": "0:42:15", "throughput": "1476.44", "total_tokens": 2577856}
+{"current_steps": 158, "total_steps": 385, "loss": 0.0847, "learning_rate": 1.3166666666666666e-06, "epoch": 2.032154340836013, "percentage": 41.04, "elapsed_time": "0:29:17", "remaining_time": "0:42:04", "throughput": "1476.26", "total_tokens": 2593984}
+{"current_steps": 159, "total_steps": 385, "loss": 0.071, "learning_rate": 1.3250000000000002e-06, "epoch": 2.045016077170418, "percentage": 41.3, "elapsed_time": "0:29:28", "remaining_time": "0:41:53", "throughput": "1476.41", "total_tokens": 2610720}
+{"current_steps": 160, "total_steps": 385, "loss": 0.0791, "learning_rate": 1.3333333333333334e-06, "epoch": 2.057877813504823, "percentage": 41.56, "elapsed_time": "0:29:39", "remaining_time": "0:41:42", "throughput": "1476.00", "total_tokens": 2626400}
+{"current_steps": 161, "total_steps": 385, "loss": 0.0769, "learning_rate": 1.3416666666666666e-06, "epoch": 2.0707395498392285, "percentage": 41.82, "elapsed_time": "0:29:50", "remaining_time": "0:41:31", "throughput": "1475.83", "total_tokens": 2642496}
+{"current_steps": 162, "total_steps": 385, "loss": 0.0916, "learning_rate": 1.3500000000000002e-06, "epoch": 2.0836012861736335, "percentage": 42.08, "elapsed_time": "0:30:01", "remaining_time": "0:41:19", "throughput": "1475.46", "total_tokens": 2658144}
+{"current_steps": 163, "total_steps": 385, "loss": 0.0584, "learning_rate": 1.3583333333333334e-06, "epoch": 2.0964630225080385, "percentage": 42.34, "elapsed_time": "0:30:12", "remaining_time": "0:41:08", "throughput": "1475.48", "total_tokens": 2674560}
+{"current_steps": 164, "total_steps": 385, "loss": 0.0811, "learning_rate": 1.3666666666666668e-06, "epoch": 2.1093247588424435, "percentage": 42.6, "elapsed_time": "0:30:23", "remaining_time": "0:40:57", "throughput": "1475.35", "total_tokens": 2690688}
+{"current_steps": 165, "total_steps": 385, "loss": 0.0663, "learning_rate": 1.3750000000000002e-06, "epoch": 2.122186495176849, "percentage": 42.86, "elapsed_time": "0:30:34", "remaining_time": "0:40:46", "throughput": "1475.17", "total_tokens": 2706720}
+{"current_steps": 166, "total_steps": 385, "loss": 0.0802, "learning_rate": 1.3833333333333336e-06, "epoch": 2.135048231511254, "percentage": 43.12, "elapsed_time": "0:30:45", "remaining_time": "0:40:35", "throughput": "1475.13", "total_tokens": 2723008}
+{"current_steps": 167, "total_steps": 385, "loss": 0.0587, "learning_rate": 1.3916666666666668e-06, "epoch": 2.147909967845659, "percentage": 43.38, "elapsed_time": "0:30:57", "remaining_time": "0:40:24", "throughput": "1474.82", "total_tokens": 2738784}
+{"current_steps": 168, "total_steps": 385, "loss": 0.0953, "learning_rate": 1.4000000000000001e-06, "epoch": 2.1607717041800645, "percentage": 43.64, "elapsed_time": "0:31:08", "remaining_time": "0:40:13", "throughput": "1474.73", "total_tokens": 2755072}
+{"current_steps": 169, "total_steps": 385, "loss": 0.0795, "learning_rate": 1.4083333333333335e-06, "epoch": 2.1736334405144695, "percentage": 43.9, "elapsed_time": "0:31:19", "remaining_time": "0:40:01", "throughput": "1474.63", "total_tokens": 2771296}
+{"current_steps": 170, "total_steps": 385, "loss": 0.1015, "learning_rate": 1.4166666666666667e-06, "epoch": 2.1864951768488745, "percentage": 44.16, "elapsed_time": "0:31:30", "remaining_time": "0:39:50", "throughput": "1474.63", "total_tokens": 2787712}
+{"current_steps": 171, "total_steps": 385, "loss": 0.0614, "learning_rate": 1.425e-06, "epoch": 2.19935691318328, "percentage": 44.42, "elapsed_time": "0:31:41", "remaining_time": "0:39:39", "throughput": "1474.37", "total_tokens": 2803552}
+{"current_steps": 172, "total_steps": 385, "loss": 0.0753, "learning_rate": 1.4333333333333335e-06, "epoch": 2.212218649517685, "percentage": 44.68, "elapsed_time": "0:31:52", "remaining_time": "0:39:28", "throughput": "1474.59", "total_tokens": 2820352}
+{"current_steps": 173, "total_steps": 385, "loss": 0.08, "learning_rate": 1.4416666666666667e-06, "epoch": 2.22508038585209, "percentage": 44.94, "elapsed_time": "0:32:03", "remaining_time": "0:39:17", "throughput": "1475.05", "total_tokens": 2837696}
+{"current_steps": 174, "total_steps": 385, "loss": 0.0603, "learning_rate": 1.45e-06, "epoch": 2.237942122186495, "percentage": 45.19, "elapsed_time": "0:32:14", "remaining_time": "0:39:06", "throughput": "1475.13", "total_tokens": 2854272}
+{"current_steps": 175, "total_steps": 385, "loss": 0.0874, "learning_rate": 1.4583333333333335e-06, "epoch": 2.2508038585209005, "percentage": 45.45, "elapsed_time": "0:32:26", "remaining_time": "0:38:55", "throughput": "1475.01", "total_tokens": 2870464}
+{"current_steps": 176, "total_steps": 385, "loss": 0.0833, "learning_rate": 1.4666666666666669e-06, "epoch": 2.2636655948553055, "percentage": 45.71, "elapsed_time": "0:32:37", "remaining_time": "0:38:44", "throughput": "1475.02", "total_tokens": 2886880}
+{"current_steps": 177, "total_steps": 385, "loss": 0.0676, "learning_rate": 1.475e-06, "epoch": 2.2765273311897105, "percentage": 45.97, "elapsed_time": "0:32:48", "remaining_time": "0:38:33", "throughput": "1474.80", "total_tokens": 2902880}
+{"current_steps": 178, "total_steps": 385, "loss": 0.086, "learning_rate": 1.4833333333333337e-06, "epoch": 2.289389067524116, "percentage": 46.23, "elapsed_time": "0:32:59", "remaining_time": "0:38:21", "throughput": "1474.96", "total_tokens": 2919584}
+{"current_steps": 179, "total_steps": 385, "loss": 0.0662, "learning_rate": 1.4916666666666669e-06, "epoch": 2.302250803858521, "percentage": 46.49, "elapsed_time": "0:33:10", "remaining_time": "0:38:10", "throughput": "1474.55", "total_tokens": 2935136}
+{"current_steps": 180, "total_steps": 385, "loss": 0.1028, "learning_rate": 1.5e-06, "epoch": 2.315112540192926, "percentage": 46.75, "elapsed_time": "0:33:21", "remaining_time": "0:37:59", "throughput": "1474.63", "total_tokens": 2951680}
+{"current_steps": 181, "total_steps": 385, "loss": 0.0882, "learning_rate": 1.5083333333333336e-06, "epoch": 2.327974276527331, "percentage": 47.01, "elapsed_time": "0:33:32", "remaining_time": "0:37:48", "throughput": "1474.45", "total_tokens": 2967712}
+{"current_steps": 182, "total_steps": 385, "loss": 0.0806, "learning_rate": 1.5166666666666668e-06, "epoch": 2.3408360128617365, "percentage": 47.27, "elapsed_time": "0:33:43", "remaining_time": "0:37:37", "throughput": "1474.52", "total_tokens": 2984224}
+{"current_steps": 183, "total_steps": 385, "loss": 0.1108, "learning_rate": 1.525e-06, "epoch": 2.3536977491961415, "percentage": 47.53, "elapsed_time": "0:33:54", "remaining_time": "0:37:26", "throughput": "1474.19", "total_tokens": 2999904}
+{"current_steps": 184, "total_steps": 385, "loss": 0.0687, "learning_rate": 1.5333333333333334e-06, "epoch": 2.3665594855305465, "percentage": 47.79, "elapsed_time": "0:34:06", "remaining_time": "0:37:15", "throughput": "1474.68", "total_tokens": 3017376}
+{"current_steps": 185, "total_steps": 385, "loss": 0.0736, "learning_rate": 1.5416666666666668e-06, "epoch": 2.379421221864952, "percentage": 48.05, "elapsed_time": "0:34:17", "remaining_time": "0:37:04", "throughput": "1474.67", "total_tokens": 3033728}
+{"current_steps": 186, "total_steps": 385, "loss": 0.0779, "learning_rate": 1.5500000000000002e-06, "epoch": 2.392282958199357, "percentage": 48.31, "elapsed_time": "0:34:28", "remaining_time": "0:36:52", "throughput": "1474.70", "total_tokens": 3050176}
+{"current_steps": 187, "total_steps": 385, "loss": 0.0709, "learning_rate": 1.5583333333333334e-06, "epoch": 2.405144694533762, "percentage": 48.57, "elapsed_time": "0:34:39", "remaining_time": "0:36:41", "throughput": "1474.61", "total_tokens": 3066400}
+{"current_steps": 188, "total_steps": 385, "loss": 0.0652, "learning_rate": 1.566666666666667e-06, "epoch": 2.418006430868167, "percentage": 48.83, "elapsed_time": "0:34:50", "remaining_time": "0:36:30", "throughput": "1474.76", "total_tokens": 3083104}
+{"current_steps": 189, "total_steps": 385, "loss": 0.1095, "learning_rate": 1.5750000000000002e-06, "epoch": 2.4308681672025725, "percentage": 49.09, "elapsed_time": "0:35:01", "remaining_time": "0:36:19", "throughput": "1475.00", "total_tokens": 3100000}
+{"current_steps": 190, "total_steps": 385, "loss": 0.0618, "learning_rate": 1.5833333333333333e-06, "epoch": 2.4437299035369775, "percentage": 49.35, "elapsed_time": "0:35:12", "remaining_time": "0:36:08", "throughput": "1475.18", "total_tokens": 3116800}
+{"current_steps": 191, "total_steps": 385, "loss": 0.0666, "learning_rate": 1.591666666666667e-06, "epoch": 2.4565916398713825, "percentage": 49.61, "elapsed_time": "0:35:23", "remaining_time": "0:35:57", "throughput": "1475.36", "total_tokens": 3133568}
+{"current_steps": 192, "total_steps": 385, "loss": 0.0573, "learning_rate": 1.6000000000000001e-06, "epoch": 2.469453376205788, "percentage": 49.87, "elapsed_time": "0:35:35", "remaining_time": "0:35:46", "throughput": "1475.40", "total_tokens": 3150048}
+{"current_steps": 193, "total_steps": 385, "loss": 0.0577, "learning_rate": 1.6083333333333333e-06, "epoch": 2.482315112540193, "percentage": 50.13, "elapsed_time": "0:35:46", "remaining_time": "0:35:35", "throughput": "1475.50", "total_tokens": 3166688}
+{"current_steps": 194, "total_steps": 385, "loss": 0.0813, "learning_rate": 1.6166666666666667e-06, "epoch": 2.495176848874598, "percentage": 50.39, "elapsed_time": "0:35:57", "remaining_time": "0:35:23", "throughput": "1475.34", "total_tokens": 3182752}
+{"current_steps": 195, "total_steps": 385, "loss": 0.066, "learning_rate": 1.6250000000000001e-06, "epoch": 2.508038585209003, "percentage": 50.65, "elapsed_time": "0:36:08", "remaining_time": "0:35:12", "throughput": "1475.24", "total_tokens": 3198944}
+{"current_steps": 196, "total_steps": 385, "loss": 0.0622, "learning_rate": 1.6333333333333335e-06, "epoch": 2.5209003215434085, "percentage": 50.91, "elapsed_time": "0:36:19", "remaining_time": "0:35:01", "throughput": "1475.32", "total_tokens": 3215552}
+{"current_steps": 197, "total_steps": 385, "loss": 0.0616, "learning_rate": 1.6416666666666667e-06, "epoch": 2.5337620578778135, "percentage": 51.17, "elapsed_time": "0:36:30", "remaining_time": "0:34:50", "throughput": "1475.12", "total_tokens": 3231488}
+{"current_steps": 198, "total_steps": 385, "loss": 0.0993, "learning_rate": 1.6500000000000003e-06, "epoch": 2.5466237942122185, "percentage": 51.43, "elapsed_time": "0:36:41", "remaining_time": "0:34:39", "throughput": "1475.12", "total_tokens": 3247840}
+{"current_steps": 199, "total_steps": 385, "loss": 0.0702, "learning_rate": 1.6583333333333335e-06, "epoch": 2.559485530546624, "percentage": 51.69, "elapsed_time": "0:36:52", "remaining_time": "0:34:28", "throughput": "1475.10", "total_tokens": 3264128}
+{"current_steps": 200, "total_steps": 385, "loss": 0.0743, "learning_rate": 1.6666666666666667e-06, "epoch": 2.572347266881029, "percentage": 51.95, "elapsed_time": "0:37:03", "remaining_time": "0:34:17", "throughput": "1474.97", "total_tokens": 3280192}
+{"current_steps": 201, "total_steps": 385, "loss": 0.0647, "learning_rate": 1.6750000000000003e-06, "epoch": 2.585209003215434, "percentage": 52.21, "elapsed_time": "0:37:15", "remaining_time": "0:34:05", "throughput": "1474.93", "total_tokens": 3296480}
+{"current_steps": 202, "total_steps": 385, "loss": 0.0814, "learning_rate": 1.6833333333333335e-06, "epoch": 2.598070739549839, "percentage": 52.47, "elapsed_time": "0:37:26", "remaining_time": "0:33:54", "throughput": "1475.03", "total_tokens": 3313120}
+{"current_steps": 203, "total_steps": 385, "loss": 0.0861, "learning_rate": 1.6916666666666666e-06, "epoch": 2.6109324758842445, "percentage": 52.73, "elapsed_time": "0:37:37", "remaining_time": "0:33:43", "throughput": "1474.89", "total_tokens": 3329184}
+{"current_steps": 204, "total_steps": 385, "loss": 0.0769, "learning_rate": 1.7000000000000002e-06, "epoch": 2.6237942122186495, "percentage": 52.99, "elapsed_time": "0:37:48", "remaining_time": "0:33:32", "throughput": "1475.15", "total_tokens": 3346208}
+{"current_steps": 205, "total_steps": 385, "loss": 0.0888, "learning_rate": 1.7083333333333334e-06, "epoch": 2.6366559485530545, "percentage": 53.25, "elapsed_time": "0:37:59", "remaining_time": "0:33:21", "throughput": "1475.31", "total_tokens": 3363008}
+{"current_steps": 206, "total_steps": 385, "loss": 0.1017, "learning_rate": 1.7166666666666668e-06, "epoch": 2.64951768488746, "percentage": 53.51, "elapsed_time": "0:38:10", "remaining_time": "0:33:10", "throughput": "1475.34", "total_tokens": 3379488}
+{"current_steps": 207, "total_steps": 385, "loss": 0.0677, "learning_rate": 1.725e-06, "epoch": 2.662379421221865, "percentage": 53.77, "elapsed_time": "0:38:21", "remaining_time": "0:32:59", "throughput": "1475.25", "total_tokens": 3395648}
+{"current_steps": 208, "total_steps": 385, "loss": 0.0861, "learning_rate": 1.7333333333333336e-06, "epoch": 2.67524115755627, "percentage": 54.03, "elapsed_time": "0:38:32", "remaining_time": "0:32:48", "throughput": "1475.55", "total_tokens": 3412736}
+{"current_steps": 209, "total_steps": 385, "loss": 0.0562, "learning_rate": 1.7416666666666668e-06, "epoch": 2.688102893890675, "percentage": 54.29, "elapsed_time": "0:38:43", "remaining_time": "0:32:37", "throughput": "1475.69", "total_tokens": 3429472}
+{"current_steps": 210, "total_steps": 385, "loss": 0.0641, "learning_rate": 1.75e-06, "epoch": 2.7009646302250805, "percentage": 54.55, "elapsed_time": "0:38:55", "remaining_time": "0:32:25", "throughput": "1475.59", "total_tokens": 3445632}
+{"current_steps": 211, "total_steps": 385, "loss": 0.0829, "learning_rate": 1.7583333333333336e-06, "epoch": 2.7138263665594855, "percentage": 54.81, "elapsed_time": "0:39:06", "remaining_time": "0:32:14", "throughput": "1475.88", "total_tokens": 3462720}
+{"current_steps": 212, "total_steps": 385, "loss": 0.0632, "learning_rate": 1.7666666666666668e-06, "epoch": 2.7266881028938905, "percentage": 55.06, "elapsed_time": "0:39:17", "remaining_time": "0:32:03", "throughput": "1475.70", "total_tokens": 3478720}
+{"current_steps": 213, "total_steps": 385, "loss": 0.062, "learning_rate": 1.7750000000000002e-06, "epoch": 2.739549839228296, "percentage": 55.32, "elapsed_time": "0:39:28", "remaining_time": "0:31:52", "throughput": "1475.36", "total_tokens": 3494304}
+{"current_steps": 214, "total_steps": 385, "loss": 0.0816, "learning_rate": 1.7833333333333336e-06, "epoch": 2.752411575562701, "percentage": 55.58, "elapsed_time": "0:39:39", "remaining_time": "0:31:41", "throughput": "1475.14", "total_tokens": 3510176}
+{"current_steps": 215, "total_steps": 385, "loss": 0.096, "learning_rate": 1.7916666666666667e-06, "epoch": 2.765273311897106, "percentage": 55.84, "elapsed_time": "0:39:50", "remaining_time": "0:31:30", "throughput": "1475.26", "total_tokens": 3526848}
+{"current_steps": 216, "total_steps": 385, "loss": 0.0639, "learning_rate": 1.8000000000000001e-06, "epoch": 2.778135048231511, "percentage": 56.1, "elapsed_time": "0:40:01", "remaining_time": "0:31:19", "throughput": "1475.21", "total_tokens": 3543136}
+{"current_steps": 217, "total_steps": 385, "loss": 0.0915, "learning_rate": 1.8083333333333335e-06, "epoch": 2.7909967845659165, "percentage": 56.36, "elapsed_time": "0:40:12", "remaining_time": "0:31:08", "throughput": "1474.69", "total_tokens": 3558240}
+{"current_steps": 218, "total_steps": 385, "loss": 0.061, "learning_rate": 1.816666666666667e-06, "epoch": 2.8038585209003215, "percentage": 56.62, "elapsed_time": "0:40:23", "remaining_time": "0:30:56", "throughput": "1474.53", "total_tokens": 3574240}
+{"current_steps": 219, "total_steps": 385, "loss": 0.0572, "learning_rate": 1.825e-06, "epoch": 2.816720257234727, "percentage": 56.88, "elapsed_time": "0:40:35", "remaining_time": "0:30:45", "throughput": "1474.35", "total_tokens": 3590240}
+{"current_steps": 220, "total_steps": 385, "loss": 0.0497, "learning_rate": 1.8333333333333333e-06, "epoch": 2.829581993569132, "percentage": 57.14, "elapsed_time": "0:40:46", "remaining_time": "0:30:34", "throughput": "1474.44", "total_tokens": 3606912}
+{"current_steps": 221, "total_steps": 385, "loss": 0.0672, "learning_rate": 1.8416666666666669e-06, "epoch": 2.842443729903537, "percentage": 57.4, "elapsed_time": "0:40:57", "remaining_time": "0:30:23", "throughput": "1474.31", "total_tokens": 3623008}
+{"current_steps": 222, "total_steps": 385, "loss": 0.0563, "learning_rate": 1.85e-06, "epoch": 2.855305466237942, "percentage": 57.66, "elapsed_time": "0:41:08", "remaining_time": "0:30:12", "throughput": "1474.33", "total_tokens": 3639456}
+{"current_steps": 223, "total_steps": 385, "loss": 0.069, "learning_rate": 1.8583333333333335e-06, "epoch": 2.868167202572347, "percentage": 57.92, "elapsed_time": "0:41:19", "remaining_time": "0:30:01", "throughput": "1474.50", "total_tokens": 3656288}
+{"current_steps": 224, "total_steps": 385, "loss": 0.0824, "learning_rate": 1.8666666666666669e-06, "epoch": 2.8810289389067525, "percentage": 58.18, "elapsed_time": "0:41:30", "remaining_time": "0:29:50", "throughput": "1474.59", "total_tokens": 3672896}
+{"current_steps": 225, "total_steps": 385, "loss": 0.057, "learning_rate": 1.8750000000000003e-06, "epoch": 2.8938906752411575, "percentage": 58.44, "elapsed_time": "0:41:41", "remaining_time": "0:29:39", "throughput": "1474.72", "total_tokens": 3689632}
+{"current_steps": 226, "total_steps": 385, "loss": 0.0549, "learning_rate": 1.8833333333333334e-06, "epoch": 2.906752411575563, "percentage": 58.7, "elapsed_time": "0:41:53", "remaining_time": "0:29:28", "throughput": "1475.08", "total_tokens": 3706944}
+{"current_steps": 227, "total_steps": 385, "loss": 0.0652, "learning_rate": 1.8916666666666668e-06, "epoch": 2.919614147909968, "percentage": 58.96, "elapsed_time": "0:42:04", "remaining_time": "0:29:16", "throughput": "1474.98", "total_tokens": 3723072}
+{"current_steps": 228, "total_steps": 385, "loss": 0.0743, "learning_rate": 1.9000000000000002e-06, "epoch": 2.932475884244373, "percentage": 59.22, "elapsed_time": "0:42:15", "remaining_time": "0:29:05", "throughput": "1475.20", "total_tokens": 3740064}
+{"current_steps": 229, "total_steps": 385, "loss": 0.0416, "learning_rate": 1.9083333333333334e-06, "epoch": 2.945337620578778, "percentage": 59.48, "elapsed_time": "0:42:26", "remaining_time": "0:28:54", "throughput": "1475.35", "total_tokens": 3756864}
+{"current_steps": 230, "total_steps": 385, "loss": 0.0668, "learning_rate": 1.916666666666667e-06, "epoch": 2.958199356913183, "percentage": 59.74, "elapsed_time": "0:42:37", "remaining_time": "0:28:43", "throughput": "1475.75", "total_tokens": 3774336}
+{"current_steps": 231, "total_steps": 385, "loss": 0.0603, "learning_rate": 1.925e-06, "epoch": 2.9710610932475885, "percentage": 60.0, "elapsed_time": "0:42:48", "remaining_time": "0:28:32", "throughput": "1475.60", "total_tokens": 3790336}
+{"current_steps": 232, "total_steps": 385, "loss": 0.066, "learning_rate": 1.9333333333333336e-06, "epoch": 2.9839228295819935, "percentage": 60.26, "elapsed_time": "0:42:59", "remaining_time": "0:28:21", "throughput": "1475.88", "total_tokens": 3807520}
+{"current_steps": 233, "total_steps": 385, "loss": 0.0692, "learning_rate": 1.9416666666666666e-06, "epoch": 2.996784565916399, "percentage": 60.52, "elapsed_time": "0:43:10", "remaining_time": "0:28:10", "throughput": "1475.87", "total_tokens": 3823872}
+{"current_steps": 234, "total_steps": 385, "loss": 0.0323, "learning_rate": 1.9500000000000004e-06, "epoch": 3.009646302250804, "percentage": 60.78, "elapsed_time": "0:43:22", "remaining_time": "0:27:59", "throughput": "1475.89", "total_tokens": 3840320}
+{"current_steps": 235, "total_steps": 385, "loss": 0.0403, "learning_rate": 1.9583333333333334e-06, "epoch": 3.022508038585209, "percentage": 61.04, "elapsed_time": "0:43:33", "remaining_time": "0:27:47", "throughput": "1475.72", "total_tokens": 3856256}
+{"current_steps": 236, "total_steps": 385, "loss": 0.0477, "learning_rate": 1.9666666666666668e-06, "epoch": 3.035369774919614, "percentage": 61.3, "elapsed_time": "0:43:44", "remaining_time": "0:27:36", "throughput": "1475.66", "total_tokens": 3872512}
+{"current_steps": 237, "total_steps": 385, "loss": 0.0356, "learning_rate": 1.975e-06, "epoch": 3.0482315112540195, "percentage": 61.56, "elapsed_time": "0:43:55", "remaining_time": "0:27:25", "throughput": "1475.41", "total_tokens": 3888256}
+{"current_steps": 238, "total_steps": 385, "loss": 0.0324, "learning_rate": 1.9833333333333335e-06, "epoch": 3.0610932475884245, "percentage": 61.82, "elapsed_time": "0:44:06", "remaining_time": "0:27:14", "throughput": "1475.56", "total_tokens": 3905024}
+{"current_steps": 239, "total_steps": 385, "loss": 0.0428, "learning_rate": 1.991666666666667e-06, "epoch": 3.0739549839228295, "percentage": 62.08, "elapsed_time": "0:44:17", "remaining_time": "0:27:03", "throughput": "1475.76", "total_tokens": 3922016}
+{"current_steps": 240, "total_steps": 385, "loss": 0.0315, "learning_rate": 2.0000000000000003e-06, "epoch": 3.0868167202572345, "percentage": 62.34, "elapsed_time": "0:44:28", "remaining_time": "0:26:52", "throughput": "1475.75", "total_tokens": 3938400}
+{"current_steps": 241, "total_steps": 385, "loss": 0.0366, "learning_rate": 2.0083333333333337e-06, "epoch": 3.09967845659164, "percentage": 62.6, "elapsed_time": "0:44:39", "remaining_time": "0:26:41", "throughput": "1475.78", "total_tokens": 3954912}
+{"current_steps": 242, "total_steps": 385, "loss": 0.0307, "learning_rate": 2.0166666666666667e-06, "epoch": 3.112540192926045, "percentage": 62.86, "elapsed_time": "0:44:50", "remaining_time": "0:26:30", "throughput": "1475.47", "total_tokens": 3970400}
+{"current_steps": 243, "total_steps": 385, "loss": 0.0338, "learning_rate": 2.025e-06, "epoch": 3.12540192926045, "percentage": 63.12, "elapsed_time": "0:45:02", "remaining_time": "0:26:18", "throughput": "1475.52", "total_tokens": 3986944}
+{"current_steps": 244, "total_steps": 385, "loss": 0.0348, "learning_rate": 2.0333333333333335e-06, "epoch": 3.1382636655948555, "percentage": 63.38, "elapsed_time": "0:45:13", "remaining_time": "0:26:07", "throughput": "1475.62", "total_tokens": 4003648}
+{"current_steps": 245, "total_steps": 385, "loss": 0.0355, "learning_rate": 2.041666666666667e-06, "epoch": 3.1511254019292605, "percentage": 63.64, "elapsed_time": "0:45:24", "remaining_time": "0:25:56", "throughput": "1475.75", "total_tokens": 4020384}
+{"current_steps": 246, "total_steps": 385, "loss": 0.0272, "learning_rate": 2.05e-06, "epoch": 3.1639871382636655, "percentage": 63.9, "elapsed_time": "0:45:35", "remaining_time": "0:25:45", "throughput": "1475.84", "total_tokens": 4037088}
+{"current_steps": 247, "total_steps": 385, "loss": 0.0464, "learning_rate": 2.0583333333333337e-06, "epoch": 3.176848874598071, "percentage": 64.16, "elapsed_time": "0:45:46", "remaining_time": "0:25:34", "throughput": "1475.83", "total_tokens": 4053472}
+{"current_steps": 248, "total_steps": 385, "loss": 0.0268, "learning_rate": 2.0666666666666666e-06, "epoch": 3.189710610932476, "percentage": 64.42, "elapsed_time": "0:45:57", "remaining_time": "0:25:23", "throughput": "1475.60", "total_tokens": 4069248}
+{"current_steps": 249, "total_steps": 385, "loss": 0.0229, "learning_rate": 2.075e-06, "epoch": 3.202572347266881, "percentage": 64.68, "elapsed_time": "0:46:08", "remaining_time": "0:25:12", "throughput": "1475.38", "total_tokens": 4085024}
+{"current_steps": 250, "total_steps": 385, "loss": 0.0373, "learning_rate": 2.0833333333333334e-06, "epoch": 3.215434083601286, "percentage": 64.94, "elapsed_time": "0:46:19", "remaining_time": "0:25:01", "throughput": "1475.74", "total_tokens": 4102432}
+{"current_steps": 251, "total_steps": 385, "loss": 0.0361, "learning_rate": 2.091666666666667e-06, "epoch": 3.2282958199356915, "percentage": 65.19, "elapsed_time": "0:46:31", "remaining_time": "0:24:50", "throughput": "1475.59", "total_tokens": 4118368}
+{"current_steps": 252, "total_steps": 385, "loss": 0.0401, "learning_rate": 2.1000000000000002e-06, "epoch": 3.2411575562700965, "percentage": 65.45, "elapsed_time": "0:46:42", "remaining_time": "0:24:38", "throughput": "1475.60", "total_tokens": 4134784}
+{"current_steps": 253, "total_steps": 385, "loss": 0.0262, "learning_rate": 2.1083333333333336e-06, "epoch": 3.2540192926045015, "percentage": 65.71, "elapsed_time": "0:46:53", "remaining_time": "0:24:27", "throughput": "1475.38", "total_tokens": 4150528}
+{"current_steps": 254, "total_steps": 385, "loss": 0.0481, "learning_rate": 2.116666666666667e-06, "epoch": 3.266881028938907, "percentage": 65.97, "elapsed_time": "0:47:04", "remaining_time": "0:24:16", "throughput": "1475.34", "total_tokens": 4166816}
+{"current_steps": 255, "total_steps": 385, "loss": 0.0417, "learning_rate": 2.125e-06, "epoch": 3.279742765273312, "percentage": 66.23, "elapsed_time": "0:47:15", "remaining_time": "0:24:05", "throughput": "1475.21", "total_tokens": 4182880}
+{"current_steps": 256, "total_steps": 385, "loss": 0.0457, "learning_rate": 2.133333333333334e-06, "epoch": 3.292604501607717, "percentage": 66.49, "elapsed_time": "0:47:26", "remaining_time": "0:23:54", "throughput": "1475.48", "total_tokens": 4200128}
+{"current_steps": 257, "total_steps": 385, "loss": 0.0175, "learning_rate": 2.1416666666666668e-06, "epoch": 3.305466237942122, "percentage": 66.75, "elapsed_time": "0:47:37", "remaining_time": "0:23:43", "throughput": "1475.19", "total_tokens": 4215680}
+{"current_steps": 258, "total_steps": 385, "loss": 0.0371, "learning_rate": 2.15e-06, "epoch": 3.3183279742765275, "percentage": 67.01, "elapsed_time": "0:47:48", "remaining_time": "0:23:32", "throughput": "1475.40", "total_tokens": 4232736}
+{"current_steps": 259, "total_steps": 385, "loss": 0.0268, "learning_rate": 2.1583333333333336e-06, "epoch": 3.3311897106109325, "percentage": 67.27, "elapsed_time": "0:47:59", "remaining_time": "0:23:21", "throughput": "1475.58", "total_tokens": 4249664}
+{"current_steps": 260, "total_steps": 385, "loss": 0.0417, "learning_rate": 2.166666666666667e-06, "epoch": 3.3440514469453375, "percentage": 67.53, "elapsed_time": "0:48:11", "remaining_time": "0:23:09", "throughput": "1475.58", "total_tokens": 4266016}
+{"current_steps": 261, "total_steps": 385, "loss": 0.0264, "learning_rate": 2.1750000000000004e-06, "epoch": 3.356913183279743, "percentage": 67.79, "elapsed_time": "0:48:22", "remaining_time": "0:22:58", "throughput": "1475.32", "total_tokens": 4281664}
+{"current_steps": 262, "total_steps": 385, "loss": 0.0336, "learning_rate": 2.1833333333333333e-06, "epoch": 3.369774919614148, "percentage": 68.05, "elapsed_time": "0:48:33", "remaining_time": "0:22:47", "throughput": "1475.42", "total_tokens": 4298368}
+{"current_steps": 263, "total_steps": 385, "loss": 0.0434, "learning_rate": 2.191666666666667e-06, "epoch": 3.382636655948553, "percentage": 68.31, "elapsed_time": "0:48:44", "remaining_time": "0:22:36", "throughput": "1475.28", "total_tokens": 4314336}
+{"current_steps": 264, "total_steps": 385, "loss": 0.0389, "learning_rate": 2.2e-06, "epoch": 3.395498392282958, "percentage": 68.57, "elapsed_time": "0:48:55", "remaining_time": "0:22:25", "throughput": "1475.08", "total_tokens": 4330144}
+{"current_steps": 265, "total_steps": 385, "loss": 0.045, "learning_rate": 2.2083333333333335e-06, "epoch": 3.4083601286173635, "percentage": 68.83, "elapsed_time": "0:49:06", "remaining_time": "0:22:14", "throughput": "1474.84", "total_tokens": 4345856}
+{"current_steps": 266, "total_steps": 385, "loss": 0.0331, "learning_rate": 2.216666666666667e-06, "epoch": 3.4212218649517685, "percentage": 69.09, "elapsed_time": "0:49:17", "remaining_time": "0:22:03", "throughput": "1474.66", "total_tokens": 4361696}
+{"current_steps": 267, "total_steps": 385, "loss": 0.0228, "learning_rate": 2.2250000000000003e-06, "epoch": 3.4340836012861735, "percentage": 69.35, "elapsed_time": "0:49:28", "remaining_time": "0:21:52", "throughput": "1474.73", "total_tokens": 4378304}
+{"current_steps": 268, "total_steps": 385, "loss": 0.0307, "learning_rate": 2.2333333333333333e-06, "epoch": 3.446945337620579, "percentage": 69.61, "elapsed_time": "0:49:39", "remaining_time": "0:21:40", "throughput": "1474.93", "total_tokens": 4395232}
+{"current_steps": 269, "total_steps": 385, "loss": 0.0332, "learning_rate": 2.2416666666666667e-06, "epoch": 3.459807073954984, "percentage": 69.87, "elapsed_time": "0:49:51", "remaining_time": "0:21:29", "throughput": "1474.79", "total_tokens": 4411232}
+{"current_steps": 270, "total_steps": 385, "loss": 0.0662, "learning_rate": 2.25e-06, "epoch": 3.472668810289389, "percentage": 70.13, "elapsed_time": "0:50:02", "remaining_time": "0:21:18", "throughput": "1474.75", "total_tokens": 4427520}
+{"current_steps": 271, "total_steps": 385, "loss": 0.0431, "learning_rate": 2.2583333333333335e-06, "epoch": 3.485530546623794, "percentage": 70.39, "elapsed_time": "0:50:13", "remaining_time": "0:21:07", "throughput": "1474.76", "total_tokens": 4443904}
+{"current_steps": 272, "total_steps": 385, "loss": 0.0423, "learning_rate": 2.266666666666667e-06, "epoch": 3.4983922829581995, "percentage": 70.65, "elapsed_time": "0:50:24", "remaining_time": "0:20:56", "throughput": "1474.84", "total_tokens": 4460608}
+{"current_steps": 273, "total_steps": 385, "loss": 0.0447, "learning_rate": 2.2750000000000002e-06, "epoch": 3.5112540192926045, "percentage": 70.91, "elapsed_time": "0:50:35", "remaining_time": "0:20:45", "throughput": "1474.86", "total_tokens": 4477120}
+{"current_steps": 274, "total_steps": 385, "loss": 0.0337, "learning_rate": 2.2833333333333336e-06, "epoch": 3.5241157556270095, "percentage": 71.17, "elapsed_time": "0:50:46", "remaining_time": "0:20:34", "throughput": "1474.75", "total_tokens": 4493152}
+{"current_steps": 275, "total_steps": 385, "loss": 0.0319, "learning_rate": 2.2916666666666666e-06, "epoch": 3.536977491961415, "percentage": 71.43, "elapsed_time": "0:50:57", "remaining_time": "0:20:23", "throughput": "1474.61", "total_tokens": 4509120}
+{"current_steps": 276, "total_steps": 385, "loss": 0.027, "learning_rate": 2.3000000000000004e-06, "epoch": 3.54983922829582, "percentage": 71.69, "elapsed_time": "0:51:08", "remaining_time": "0:20:12", "throughput": "1474.75", "total_tokens": 4525984}
+{"current_steps": 277, "total_steps": 385, "loss": 0.0244, "learning_rate": 2.3083333333333334e-06, "epoch": 3.562700964630225, "percentage": 71.95, "elapsed_time": "0:51:20", "remaining_time": "0:20:00", "throughput": "1474.91", "total_tokens": 4542816}
+{"current_steps": 278, "total_steps": 385, "loss": 0.0377, "learning_rate": 2.316666666666667e-06, "epoch": 3.57556270096463, "percentage": 72.21, "elapsed_time": "0:51:31", "remaining_time": "0:19:49", "throughput": "1475.12", "total_tokens": 4559872}
+{"current_steps": 279, "total_steps": 385, "loss": 0.0477, "learning_rate": 2.325e-06, "epoch": 3.5884244372990355, "percentage": 72.47, "elapsed_time": "0:51:42", "remaining_time": "0:19:38", "throughput": "1475.12", "total_tokens": 4576288}
+{"current_steps": 280, "total_steps": 385, "loss": 0.0332, "learning_rate": 2.3333333333333336e-06, "epoch": 3.6012861736334405, "percentage": 72.73, "elapsed_time": "0:51:53", "remaining_time": "0:19:27", "throughput": "1474.74", "total_tokens": 4591456}
+{"current_steps": 281, "total_steps": 385, "loss": 0.0285, "learning_rate": 2.341666666666667e-06, "epoch": 3.6141479099678455, "percentage": 72.99, "elapsed_time": "0:52:04", "remaining_time": "0:19:16", "throughput": "1474.97", "total_tokens": 4608608}
+{"current_steps": 282, "total_steps": 385, "loss": 0.0468, "learning_rate": 2.35e-06, "epoch": 3.627009646302251, "percentage": 73.25, "elapsed_time": "0:52:15", "remaining_time": "0:19:05", "throughput": "1475.00", "total_tokens": 4625120}
+{"current_steps": 283, "total_steps": 385, "loss": 0.037, "learning_rate": 2.3583333333333338e-06, "epoch": 3.639871382636656, "percentage": 73.51, "elapsed_time": "0:52:26", "remaining_time": "0:18:54", "throughput": "1475.13", "total_tokens": 4641952}
+{"current_steps": 284, "total_steps": 385, "loss": 0.041, "learning_rate": 2.3666666666666667e-06, "epoch": 3.652733118971061, "percentage": 73.77, "elapsed_time": "0:52:37", "remaining_time": "0:18:43", "throughput": "1474.98", "total_tokens": 4657920}
+{"current_steps": 285, "total_steps": 385, "loss": 0.0461, "learning_rate": 2.375e-06, "epoch": 3.665594855305466, "percentage": 74.03, "elapsed_time": "0:52:49", "remaining_time": "0:18:31", "throughput": "1475.10", "total_tokens": 4674688}
+{"current_steps": 286, "total_steps": 385, "loss": 0.0594, "learning_rate": 2.3833333333333335e-06, "epoch": 3.6784565916398715, "percentage": 74.29, "elapsed_time": "0:53:00", "remaining_time": "0:18:20", "throughput": "1474.95", "total_tokens": 4690592}
+{"current_steps": 287, "total_steps": 385, "loss": 0.0728, "learning_rate": 2.391666666666667e-06, "epoch": 3.6913183279742765, "percentage": 74.55, "elapsed_time": "0:53:11", "remaining_time": "0:18:09", "throughput": "1475.07", "total_tokens": 4707360}
+{"current_steps": 288, "total_steps": 385, "loss": 0.029, "learning_rate": 2.4000000000000003e-06, "epoch": 3.7041800643086815, "percentage": 74.81, "elapsed_time": "0:53:22", "remaining_time": "0:17:58", "throughput": "1475.20", "total_tokens": 4724192}
+{"current_steps": 289, "total_steps": 385, "loss": 0.0412, "learning_rate": 2.4083333333333337e-06, "epoch": 3.717041800643087, "percentage": 75.06, "elapsed_time": "0:53:33", "remaining_time": "0:17:47", "throughput": "1475.44", "total_tokens": 4741344}
+{"current_steps": 290, "total_steps": 385, "loss": 0.0262, "learning_rate": 2.4166666666666667e-06, "epoch": 3.729903536977492, "percentage": 75.32, "elapsed_time": "0:53:44", "remaining_time": "0:17:36", "throughput": "1475.13", "total_tokens": 4756768}
+{"current_steps": 291, "total_steps": 385, "loss": 0.0796, "learning_rate": 2.425e-06, "epoch": 3.742765273311897, "percentage": 75.58, "elapsed_time": "0:53:55", "remaining_time": "0:17:25", "throughput": "1474.96", "total_tokens": 4772608}
+{"current_steps": 292, "total_steps": 385, "loss": 0.0447, "learning_rate": 2.4333333333333335e-06, "epoch": 3.755627009646302, "percentage": 75.84, "elapsed_time": "0:54:06", "remaining_time": "0:17:14", "throughput": "1474.93", "total_tokens": 4788928}
+{"current_steps": 293, "total_steps": 385, "loss": 0.0252, "learning_rate": 2.441666666666667e-06, "epoch": 3.7684887459807075, "percentage": 76.1, "elapsed_time": "0:54:18", "remaining_time": "0:17:02", "throughput": "1474.95", "total_tokens": 4805408}
+{"current_steps": 294, "total_steps": 385, "loss": 0.0458, "learning_rate": 2.4500000000000003e-06, "epoch": 3.7813504823151125, "percentage": 76.36, "elapsed_time": "0:54:29", "remaining_time": "0:16:51", "throughput": "1474.79", "total_tokens": 4821248}
+{"current_steps": 295, "total_steps": 385, "loss": 0.0431, "learning_rate": 2.4583333333333332e-06, "epoch": 3.7942122186495175, "percentage": 76.62, "elapsed_time": "0:54:40", "remaining_time": "0:16:40", "throughput": "1474.62", "total_tokens": 4837024}
+{"current_steps": 296, "total_steps": 385, "loss": 0.0422, "learning_rate": 2.466666666666667e-06, "epoch": 3.807073954983923, "percentage": 76.88, "elapsed_time": "0:54:51", "remaining_time": "0:16:29", "throughput": "1474.72", "total_tokens": 4853760}
+{"current_steps": 297, "total_steps": 385, "loss": 0.0428, "learning_rate": 2.475e-06, "epoch": 3.819935691318328, "percentage": 77.14, "elapsed_time": "0:55:02", "remaining_time": "0:16:18", "throughput": "1474.85", "total_tokens": 4870560}
+{"current_steps": 298, "total_steps": 385, "loss": 0.0473, "learning_rate": 2.4833333333333334e-06, "epoch": 3.832797427652733, "percentage": 77.4, "elapsed_time": "0:55:13", "remaining_time": "0:16:07", "throughput": "1474.88", "total_tokens": 4887040}
+{"current_steps": 299, "total_steps": 385, "loss": 0.0223, "learning_rate": 2.491666666666667e-06, "epoch": 3.845659163987138, "percentage": 77.66, "elapsed_time": "0:55:24", "remaining_time": "0:15:56", "throughput": "1474.81", "total_tokens": 4903232}
+{"current_steps": 300, "total_steps": 385, "loss": 0.0274, "learning_rate": 2.5e-06, "epoch": 3.8585209003215435, "percentage": 77.92, "elapsed_time": "0:55:35", "remaining_time": "0:15:45", "throughput": "1474.73", "total_tokens": 4919392}
+{"current_steps": 301, "total_steps": 385, "loss": 0.0454, "learning_rate": 2.5083333333333336e-06, "epoch": 3.8713826366559485, "percentage": 78.18, "elapsed_time": "0:55:46", "remaining_time": "0:15:34", "throughput": "1474.68", "total_tokens": 4935648}
+{"current_steps": 302, "total_steps": 385, "loss": 0.022, "learning_rate": 2.5166666666666666e-06, "epoch": 3.884244372990354, "percentage": 78.44, "elapsed_time": "0:55:58", "remaining_time": "0:15:22", "throughput": "1474.90", "total_tokens": 4952768}
+{"current_steps": 303, "total_steps": 385, "loss": 0.0357, "learning_rate": 2.5250000000000004e-06, "epoch": 3.897106109324759, "percentage": 78.7, "elapsed_time": "0:56:09", "remaining_time": "0:15:11", "throughput": "1474.97", "total_tokens": 4969408}
+{"current_steps": 304, "total_steps": 385, "loss": 0.0458, "learning_rate": 2.5333333333333338e-06, "epoch": 3.909967845659164, "percentage": 78.96, "elapsed_time": "0:56:20", "remaining_time": "0:15:00", "throughput": "1475.20", "total_tokens": 4986592}
+{"current_steps": 305, "total_steps": 385, "loss": 0.0454, "learning_rate": 2.5416666666666668e-06, "epoch": 3.922829581993569, "percentage": 79.22, "elapsed_time": "0:56:31", "remaining_time": "0:14:49", "throughput": "1475.13", "total_tokens": 5002720}
+{"current_steps": 306, "total_steps": 385, "loss": 0.0262, "learning_rate": 2.55e-06, "epoch": 3.935691318327974, "percentage": 79.48, "elapsed_time": "0:56:42", "remaining_time": "0:14:38", "throughput": "1475.05", "total_tokens": 5018848}
+{"current_steps": 307, "total_steps": 385, "loss": 0.0234, "learning_rate": 2.558333333333334e-06, "epoch": 3.9485530546623795, "percentage": 79.74, "elapsed_time": "0:56:53", "remaining_time": "0:14:27", "throughput": "1474.86", "total_tokens": 5034528}
+{"current_steps": 308, "total_steps": 385, "loss": 0.0366, "learning_rate": 2.566666666666667e-06, "epoch": 3.9614147909967845, "percentage": 80.0, "elapsed_time": "0:57:04", "remaining_time": "0:14:16", "throughput": "1474.97", "total_tokens": 5051328}
+{"current_steps": 309, "total_steps": 385, "loss": 0.0236, "learning_rate": 2.5750000000000003e-06, "epoch": 3.97427652733119, "percentage": 80.26, "elapsed_time": "0:57:15", "remaining_time": "0:14:05", "throughput": "1474.97", "total_tokens": 5067712}
+{"current_steps": 310, "total_steps": 385, "loss": 0.0403, "learning_rate": 2.5833333333333337e-06, "epoch": 3.987138263665595, "percentage": 80.52, "elapsed_time": "0:57:26", "remaining_time": "0:13:53", "throughput": "1475.24", "total_tokens": 5085088}
+{"current_steps": 311, "total_steps": 385, "loss": 0.0395, "learning_rate": 2.5916666666666667e-06, "epoch": 4.0, "percentage": 80.78, "elapsed_time": "0:57:38", "remaining_time": "0:13:42", "throughput": "1475.24", "total_tokens": 5101536}
+{"current_steps": 312, "total_steps": 385, "loss": 0.0163, "learning_rate": 2.6e-06, "epoch": 4.012861736334405, "percentage": 81.04, "elapsed_time": "0:57:49", "remaining_time": "0:13:31", "throughput": "1475.19", "total_tokens": 5117696}
+{"current_steps": 313, "total_steps": 385, "loss": 0.0181, "learning_rate": 2.608333333333333e-06, "epoch": 4.02572347266881, "percentage": 81.3, "elapsed_time": "0:58:00", "remaining_time": "0:13:20", "throughput": "1475.00", "total_tokens": 5133408}
+{"current_steps": 314, "total_steps": 385, "loss": 0.013, "learning_rate": 2.616666666666667e-06, "epoch": 4.038585209003215, "percentage": 81.56, "elapsed_time": "0:58:11", "remaining_time": "0:13:09", "throughput": "1474.96", "total_tokens": 5149664}
+{"current_steps": 315, "total_steps": 385, "loss": 0.0178, "learning_rate": 2.6250000000000003e-06, "epoch": 4.051446945337621, "percentage": 81.82, "elapsed_time": "0:58:22", "remaining_time": "0:12:58", "throughput": "1474.78", "total_tokens": 5165344}
+{"current_steps": 316, "total_steps": 385, "loss": 0.0153, "learning_rate": 2.6333333333333332e-06, "epoch": 4.064308681672026, "percentage": 82.08, "elapsed_time": "0:58:33", "remaining_time": "0:12:47", "throughput": "1475.20", "total_tokens": 5183264}
+{"current_steps": 317, "total_steps": 385, "loss": 0.0205, "learning_rate": 2.6416666666666666e-06, "epoch": 4.077170418006431, "percentage": 82.34, "elapsed_time": "0:58:44", "remaining_time": "0:12:36", "throughput": "1475.09", "total_tokens": 5199296}
+{"current_steps": 318, "total_steps": 385, "loss": 0.0021, "learning_rate": 2.6500000000000005e-06, "epoch": 4.090032154340836, "percentage": 82.6, "elapsed_time": "0:58:55", "remaining_time": "0:12:24", "throughput": "1475.17", "total_tokens": 5216032}
+{"current_steps": 319, "total_steps": 385, "loss": 0.022, "learning_rate": 2.6583333333333334e-06, "epoch": 4.102893890675241, "percentage": 82.86, "elapsed_time": "0:59:07", "remaining_time": "0:12:13", "throughput": "1475.20", "total_tokens": 5232544}
+{"current_steps": 320, "total_steps": 385, "loss": 0.0134, "learning_rate": 2.666666666666667e-06, "epoch": 4.115755627009646, "percentage": 83.12, "elapsed_time": "0:59:18", "remaining_time": "0:12:02", "throughput": "1475.14", "total_tokens": 5248704}
+{"current_steps": 321, "total_steps": 385, "loss": 0.0175, "learning_rate": 2.6750000000000002e-06, "epoch": 4.128617363344051, "percentage": 83.38, "elapsed_time": "0:59:29", "remaining_time": "0:11:51", "throughput": "1474.95", "total_tokens": 5264384}
+{"current_steps": 322, "total_steps": 385, "loss": 0.023, "learning_rate": 2.683333333333333e-06, "epoch": 4.141479099678457, "percentage": 83.64, "elapsed_time": "0:59:40", "remaining_time": "0:11:40", "throughput": "1474.98", "total_tokens": 5280928}
+{"current_steps": 323, "total_steps": 385, "loss": 0.0303, "learning_rate": 2.691666666666667e-06, "epoch": 4.154340836012862, "percentage": 83.9, "elapsed_time": "0:59:51", "remaining_time": "0:11:29", "throughput": "1474.77", "total_tokens": 5296512}
+{"current_steps": 324, "total_steps": 385, "loss": 0.0187, "learning_rate": 2.7000000000000004e-06, "epoch": 4.167202572347267, "percentage": 84.16, "elapsed_time": "1:00:02", "remaining_time": "0:11:18", "throughput": "1474.91", "total_tokens": 5313440}
+{"current_steps": 325, "total_steps": 385, "loss": 0.0126, "learning_rate": 2.7083333333333334e-06, "epoch": 4.180064308681672, "percentage": 84.42, "elapsed_time": "1:00:13", "remaining_time": "0:11:07", "throughput": "1474.71", "total_tokens": 5329088}
+{"current_steps": 326, "total_steps": 385, "loss": 0.0203, "learning_rate": 2.7166666666666668e-06, "epoch": 4.192926045016077, "percentage": 84.68, "elapsed_time": "1:00:24", "remaining_time": "0:10:56", "throughput": "1474.67", "total_tokens": 5345376}
+{"current_steps": 327, "total_steps": 385, "loss": 0.0078, "learning_rate": 2.7250000000000006e-06, "epoch": 4.205787781350482, "percentage": 84.94, "elapsed_time": "1:00:35", "remaining_time": "0:10:44", "throughput": "1474.62", "total_tokens": 5361568}
+{"current_steps": 328, "total_steps": 385, "loss": 0.0165, "learning_rate": 2.7333333333333336e-06, "epoch": 4.218649517684887, "percentage": 85.19, "elapsed_time": "1:00:47", "remaining_time": "0:10:33", "throughput": "1474.50", "total_tokens": 5377536}
+{"current_steps": 329, "total_steps": 385, "loss": 0.0113, "learning_rate": 2.741666666666667e-06, "epoch": 4.231511254019293, "percentage": 85.45, "elapsed_time": "1:00:58", "remaining_time": "0:10:22", "throughput": "1474.78", "total_tokens": 5395040}
+{"current_steps": 330, "total_steps": 385, "loss": 0.0058, "learning_rate": 2.7500000000000004e-06, "epoch": 4.244372990353698, "percentage": 85.71, "elapsed_time": "1:01:09", "remaining_time": "0:10:11", "throughput": "1474.81", "total_tokens": 5411488}
+{"current_steps": 331, "total_steps": 385, "loss": 0.007, "learning_rate": 2.7583333333333333e-06, "epoch": 4.257234726688103, "percentage": 85.97, "elapsed_time": "1:01:20", "remaining_time": "0:10:00", "throughput": "1474.67", "total_tokens": 5427328}
+{"current_steps": 332, "total_steps": 385, "loss": 0.027, "learning_rate": 2.766666666666667e-06, "epoch": 4.270096463022508, "percentage": 86.23, "elapsed_time": "1:01:31", "remaining_time": "0:09:49", "throughput": "1474.84", "total_tokens": 5444352}
+{"current_steps": 333, "total_steps": 385, "loss": 0.0276, "learning_rate": 2.7750000000000005e-06, "epoch": 4.282958199356913, "percentage": 86.49, "elapsed_time": "1:01:42", "remaining_time": "0:09:38", "throughput": "1474.61", "total_tokens": 5459840}
+{"current_steps": 334, "total_steps": 385, "loss": 0.0367, "learning_rate": 2.7833333333333335e-06, "epoch": 4.295819935691318, "percentage": 86.75, "elapsed_time": "1:01:53", "remaining_time": "0:09:27", "throughput": "1474.55", "total_tokens": 5476000}
+{"current_steps": 335, "total_steps": 385, "loss": 0.0161, "learning_rate": 2.791666666666667e-06, "epoch": 4.308681672025724, "percentage": 87.01, "elapsed_time": "1:02:04", "remaining_time": "0:09:15", "throughput": "1474.44", "total_tokens": 5491968}
+{"current_steps": 336, "total_steps": 385, "loss": 0.018, "learning_rate": 2.8000000000000003e-06, "epoch": 4.321543408360129, "percentage": 87.27, "elapsed_time": "1:02:15", "remaining_time": "0:09:04", "throughput": "1474.57", "total_tokens": 5508864}
+{"current_steps": 337, "total_steps": 385, "loss": 0.0044, "learning_rate": 2.8083333333333333e-06, "epoch": 4.334405144694534, "percentage": 87.53, "elapsed_time": "1:02:27", "remaining_time": "0:08:53", "throughput": "1474.64", "total_tokens": 5525536}
+{"current_steps": 338, "total_steps": 385, "loss": 0.0109, "learning_rate": 2.816666666666667e-06, "epoch": 4.347266881028939, "percentage": 87.79, "elapsed_time": "1:02:38", "remaining_time": "0:08:42", "throughput": "1474.75", "total_tokens": 5542368}
+{"current_steps": 339, "total_steps": 385, "loss": 0.0173, "learning_rate": 2.825e-06, "epoch": 4.360128617363344, "percentage": 88.05, "elapsed_time": "1:02:49", "remaining_time": "0:08:31", "throughput": "1474.67", "total_tokens": 5558400}
+{"current_steps": 340, "total_steps": 385, "loss": 0.0107, "learning_rate": 2.8333333333333335e-06, "epoch": 4.372990353697749, "percentage": 88.31, "elapsed_time": "1:03:00", "remaining_time": "0:08:20", "throughput": "1474.70", "total_tokens": 5575008}
+{"current_steps": 341, "total_steps": 385, "loss": 0.0116, "learning_rate": 2.841666666666667e-06, "epoch": 4.385852090032154, "percentage": 88.57, "elapsed_time": "1:03:11", "remaining_time": "0:08:09", "throughput": "1474.55", "total_tokens": 5590784}
+{"current_steps": 342, "total_steps": 385, "loss": 0.0281, "learning_rate": 2.85e-06, "epoch": 4.39871382636656, "percentage": 88.83, "elapsed_time": "1:03:22", "remaining_time": "0:07:58", "throughput": "1474.78", "total_tokens": 5608032}
+{"current_steps": 343, "total_steps": 385, "loss": 0.0246, "learning_rate": 2.8583333333333336e-06, "epoch": 4.411575562700965, "percentage": 89.09, "elapsed_time": "1:03:33", "remaining_time": "0:07:46", "throughput": "1474.86", "total_tokens": 5624704}
+{"current_steps": 344, "total_steps": 385, "loss": 0.0146, "learning_rate": 2.866666666666667e-06, "epoch": 4.42443729903537, "percentage": 89.35, "elapsed_time": "1:03:44", "remaining_time": "0:07:35", "throughput": "1474.84", "total_tokens": 5641056}
+{"current_steps": 345, "total_steps": 385, "loss": 0.0439, "learning_rate": 2.875e-06, "epoch": 4.437299035369775, "percentage": 89.61, "elapsed_time": "1:03:55", "remaining_time": "0:07:24", "throughput": "1474.62", "total_tokens": 5656608}
+{"current_steps": 346, "total_steps": 385, "loss": 0.0279, "learning_rate": 2.8833333333333334e-06, "epoch": 4.45016077170418, "percentage": 89.87, "elapsed_time": "1:04:07", "remaining_time": "0:07:13", "throughput": "1474.60", "total_tokens": 5672928}
+{"current_steps": 347, "total_steps": 385, "loss": 0.0276, "learning_rate": 2.8916666666666672e-06, "epoch": 4.463022508038585, "percentage": 90.13, "elapsed_time": "1:04:18", "remaining_time": "0:07:02", "throughput": "1474.70", "total_tokens": 5689728}
+{"current_steps": 348, "total_steps": 385, "loss": 0.0167, "learning_rate": 2.9e-06, "epoch": 4.47588424437299, "percentage": 90.39, "elapsed_time": "1:04:29", "remaining_time": "0:06:51", "throughput": "1474.63", "total_tokens": 5705824}
+{"current_steps": 349, "total_steps": 385, "loss": 0.0258, "learning_rate": 2.9083333333333336e-06, "epoch": 4.488745980707396, "percentage": 90.65, "elapsed_time": "1:04:40", "remaining_time": "0:06:40", "throughput": "1474.83", "total_tokens": 5723008}
+{"current_steps": 350, "total_steps": 385, "loss": 0.0306, "learning_rate": 2.916666666666667e-06, "epoch": 4.501607717041801, "percentage": 90.91, "elapsed_time": "1:04:51", "remaining_time": "0:06:29", "throughput": "1474.91", "total_tokens": 5739680}
+{"current_steps": 351, "total_steps": 385, "loss": 0.0395, "learning_rate": 2.925e-06, "epoch": 4.514469453376206, "percentage": 91.17, "elapsed_time": "1:05:02", "remaining_time": "0:06:18", "throughput": "1475.01", "total_tokens": 5756480}
+{"current_steps": 352, "total_steps": 385, "loss": 0.0203, "learning_rate": 2.9333333333333338e-06, "epoch": 4.527331189710611, "percentage": 91.43, "elapsed_time": "1:05:13", "remaining_time": "0:06:06", "throughput": "1475.06", "total_tokens": 5773088}
+{"current_steps": 353, "total_steps": 385, "loss": 0.0241, "learning_rate": 2.941666666666667e-06, "epoch": 4.540192926045016, "percentage": 91.69, "elapsed_time": "1:05:24", "remaining_time": "0:05:55", "throughput": "1475.01", "total_tokens": 5789312}
+{"current_steps": 354, "total_steps": 385, "loss": 0.0151, "learning_rate": 2.95e-06, "epoch": 4.553054662379421, "percentage": 91.95, "elapsed_time": "1:05:36", "remaining_time": "0:05:44", "throughput": "1474.68", "total_tokens": 5804352}
+{"current_steps": 355, "total_steps": 385, "loss": 0.0355, "learning_rate": 2.9583333333333335e-06, "epoch": 4.565916398713826, "percentage": 92.21, "elapsed_time": "1:05:47", "remaining_time": "0:05:33", "throughput": "1474.59", "total_tokens": 5820384}
+{"current_steps": 356, "total_steps": 385, "loss": 0.0093, "learning_rate": 2.9666666666666673e-06, "epoch": 4.578778135048232, "percentage": 92.47, "elapsed_time": "1:05:58", "remaining_time": "0:05:22", "throughput": "1474.72", "total_tokens": 5837280}
+{"current_steps": 357, "total_steps": 385, "loss": 0.0314, "learning_rate": 2.9750000000000003e-06, "epoch": 4.591639871382637, "percentage": 92.73, "elapsed_time": "1:06:09", "remaining_time": "0:05:11", "throughput": "1474.75", "total_tokens": 5853760}
+{"current_steps": 358, "total_steps": 385, "loss": 0.0155, "learning_rate": 2.9833333333333337e-06, "epoch": 4.604501607717042, "percentage": 92.99, "elapsed_time": "1:06:20", "remaining_time": "0:05:00", "throughput": "1474.77", "total_tokens": 5870240}
+{"current_steps": 359, "total_steps": 385, "loss": 0.0138, "learning_rate": 2.991666666666667e-06, "epoch": 4.617363344051447, "percentage": 93.25, "elapsed_time": "1:06:31", "remaining_time": "0:04:49", "throughput": "1474.94", "total_tokens": 5887296}
+{"current_steps": 360, "total_steps": 385, "loss": 0.0212, "learning_rate": 3e-06, "epoch": 4.630225080385852, "percentage": 93.51, "elapsed_time": "1:06:42", "remaining_time": "0:04:37", "throughput": "1475.10", "total_tokens": 5904384}
+{"current_steps": 361, "total_steps": 385, "loss": 0.0255, "learning_rate": 3.0083333333333335e-06, "epoch": 4.643086816720257, "percentage": 93.77, "elapsed_time": "1:06:53", "remaining_time": "0:04:26", "throughput": "1475.11", "total_tokens": 5920832}
+{"current_steps": 362, "total_steps": 385, "loss": 0.0231, "learning_rate": 3.0166666666666673e-06, "epoch": 4.655948553054662, "percentage": 94.03, "elapsed_time": "1:07:04", "remaining_time": "0:04:15", "throughput": "1474.99", "total_tokens": 5936736}
+{"current_steps": 363, "total_steps": 385, "loss": 0.0083, "learning_rate": 3.0250000000000003e-06, "epoch": 4.668810289389068, "percentage": 94.29, "elapsed_time": "1:07:16", "remaining_time": "0:04:04", "throughput": "1474.99", "total_tokens": 5953120}
+{"current_steps": 364, "total_steps": 385, "loss": 0.0122, "learning_rate": 3.0333333333333337e-06, "epoch": 4.681672025723473, "percentage": 94.55, "elapsed_time": "1:07:27", "remaining_time": "0:03:53", "throughput": "1475.18", "total_tokens": 5970336}
+{"current_steps": 365, "total_steps": 385, "loss": 0.0142, "learning_rate": 3.0416666666666666e-06, "epoch": 4.694533762057878, "percentage": 94.81, "elapsed_time": "1:07:38", "remaining_time": "0:03:42", "throughput": "1475.37", "total_tokens": 5987488}
+{"current_steps": 366, "total_steps": 385, "loss": 0.0293, "learning_rate": 3.05e-06, "epoch": 4.707395498392283, "percentage": 95.06, "elapsed_time": "1:07:49", "remaining_time": "0:03:31", "throughput": "1475.24", "total_tokens": 6003296}
+{"current_steps": 367, "total_steps": 385, "loss": 0.0292, "learning_rate": 3.058333333333334e-06, "epoch": 4.720257234726688, "percentage": 95.32, "elapsed_time": "1:08:00", "remaining_time": "0:03:20", "throughput": "1475.50", "total_tokens": 6020768}
+{"current_steps": 368, "total_steps": 385, "loss": 0.032, "learning_rate": 3.066666666666667e-06, "epoch": 4.733118971061093, "percentage": 95.58, "elapsed_time": "1:08:11", "remaining_time": "0:03:09", "throughput": "1475.46", "total_tokens": 6036992}
+{"current_steps": 369, "total_steps": 385, "loss": 0.0189, "learning_rate": 3.075e-06, "epoch": 4.745980707395498, "percentage": 95.84, "elapsed_time": "1:08:22", "remaining_time": "0:02:57", "throughput": "1475.50", "total_tokens": 6053568}
+{"current_steps": 370, "total_steps": 385, "loss": 0.022, "learning_rate": 3.0833333333333336e-06, "epoch": 4.758842443729904, "percentage": 96.1, "elapsed_time": "1:08:33", "remaining_time": "0:02:46", "throughput": "1475.57", "total_tokens": 6070336}
+{"current_steps": 371, "total_steps": 385, "loss": 0.0242, "learning_rate": 3.0916666666666666e-06, "epoch": 4.771704180064309, "percentage": 96.36, "elapsed_time": "1:08:45", "remaining_time": "0:02:35", "throughput": "1475.68", "total_tokens": 6087200}
+{"current_steps": 372, "total_steps": 385, "loss": 0.015, "learning_rate": 3.1000000000000004e-06, "epoch": 4.784565916398714, "percentage": 96.62, "elapsed_time": "1:08:56", "remaining_time": "0:02:24", "throughput": "1475.82", "total_tokens": 6104224}
+{"current_steps": 373, "total_steps": 385, "loss": 0.0154, "learning_rate": 3.1083333333333338e-06, "epoch": 4.797427652733119, "percentage": 96.88, "elapsed_time": "1:09:07", "remaining_time": "0:02:13", "throughput": "1475.81", "total_tokens": 6120576}
+{"current_steps": 374, "total_steps": 385, "loss": 0.0195, "learning_rate": 3.1166666666666668e-06, "epoch": 4.810289389067524, "percentage": 97.14, "elapsed_time": "1:09:18", "remaining_time": "0:02:02", "throughput": "1475.92", "total_tokens": 6137440}
+{"current_steps": 375, "total_steps": 385, "loss": 0.031, "learning_rate": 3.125e-06, "epoch": 4.823151125401929, "percentage": 97.4, "elapsed_time": "1:09:29", "remaining_time": "0:01:51", "throughput": "1476.19", "total_tokens": 6154976}
+{"current_steps": 376, "total_steps": 385, "loss": 0.0376, "learning_rate": 3.133333333333334e-06, "epoch": 4.836012861736334, "percentage": 97.66, "elapsed_time": "1:09:40", "remaining_time": "0:01:40", "throughput": "1476.09", "total_tokens": 6170912}
+{"current_steps": 377, "total_steps": 385, "loss": 0.0093, "learning_rate": 3.141666666666667e-06, "epoch": 4.84887459807074, "percentage": 97.92, "elapsed_time": "1:09:51", "remaining_time": "0:01:28", "throughput": "1476.24", "total_tokens": 6187968}
+{"current_steps": 378, "total_steps": 385, "loss": 0.027, "learning_rate": 3.1500000000000003e-06, "epoch": 4.861736334405145, "percentage": 98.18, "elapsed_time": "1:10:02", "remaining_time": "0:01:17", "throughput": "1476.22", "total_tokens": 6204320}
+{"current_steps": 379, "total_steps": 385, "loss": 0.0191, "learning_rate": 3.1583333333333337e-06, "epoch": 4.87459807073955, "percentage": 98.44, "elapsed_time": "1:10:13", "remaining_time": "0:01:06", "throughput": "1476.16", "total_tokens": 6220512}
+{"current_steps": 380, "total_steps": 385, "loss": 0.0132, "learning_rate": 3.1666666666666667e-06, "epoch": 4.887459807073955, "percentage": 98.7, "elapsed_time": "1:10:25", "remaining_time": "0:00:55", "throughput": "1476.06", "total_tokens": 6236512}
+{"current_steps": 381, "total_steps": 385, "loss": 0.0111, "learning_rate": 3.175e-06, "epoch": 4.90032154340836, "percentage": 98.96, "elapsed_time": "1:10:36", "remaining_time": "0:00:44", "throughput": "1475.94", "total_tokens": 6252416}
+{"current_steps": 382, "total_steps": 385, "loss": 0.0145, "learning_rate": 3.183333333333334e-06, "epoch": 4.913183279742765, "percentage": 99.22, "elapsed_time": "1:10:47", "remaining_time": "0:00:33", "throughput": "1475.98", "total_tokens": 6268992}
+{"current_steps": 383, "total_steps": 385, "loss": 0.0082, "learning_rate": 3.191666666666667e-06, "epoch": 4.92604501607717, "percentage": 99.48, "elapsed_time": "1:10:58", "remaining_time": "0:00:22", "throughput": "1475.97", "total_tokens": 6285312}
+{"current_steps": 384, "total_steps": 385, "loss": 0.0165, "learning_rate": 3.2000000000000003e-06, "epoch": 4.938906752411576, "percentage": 99.74, "elapsed_time": "1:11:09", "remaining_time": "0:00:11", "throughput": "1475.81", "total_tokens": 6300992}
+{"current_steps": 385, "total_steps": 385, "loss": 0.0243, "learning_rate": 3.2083333333333337e-06, "epoch": 4.951768488745981, "percentage": 100.0, "elapsed_time": "1:11:20", "remaining_time": "0:00:00", "throughput": "1476.08", "total_tokens": 6318560}
+{"current_steps": 385, "total_steps": 385, "epoch": 4.951768488745981, "percentage": 100.0, "elapsed_time": "1:12:11", "remaining_time": "0:00:00", "throughput": "1458.88", "total_tokens": 6318560}

trainer_state.json

@@ -0,0 +1,3123 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 4.951768488745981,
+  "eval_steps": 500,
+  "global_step": 385,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.012861736334405145,
+      "grad_norm": 258.1509094238281,
+      "learning_rate": 8.333333333333335e-09,
+      "loss": 8.1083,
+      "num_input_tokens_seen": 16544,
+      "step": 1
+    },
+    {
+      "epoch": 0.02572347266881029,
+      "grad_norm": 266.15087890625,
+      "learning_rate": 1.666666666666667e-08,
+      "loss": 8.1052,
+      "num_input_tokens_seen": 33248,
+      "step": 2
+    },
+    {
+      "epoch": 0.03858520900321544,
+      "grad_norm": 276.69952392578125,
+      "learning_rate": 2.5000000000000002e-08,
+      "loss": 8.1366,
+      "num_input_tokens_seen": 49760,
+      "step": 3
+    },
+    {
+      "epoch": 0.05144694533762058,
+      "grad_norm": 268.43804931640625,
+      "learning_rate": 3.333333333333334e-08,
+      "loss": 8.0413,
+      "num_input_tokens_seen": 65824,
+      "step": 4
+    },
+    {
+      "epoch": 0.06430868167202572,
+      "grad_norm": 269.6364440917969,
+      "learning_rate": 4.166666666666667e-08,
+      "loss": 8.1055,
+      "num_input_tokens_seen": 82560,
+      "step": 5
+    },
+    {
+      "epoch": 0.07717041800643087,
+      "grad_norm": 274.33538818359375,
+      "learning_rate": 5.0000000000000004e-08,
+      "loss": 7.9666,
+      "num_input_tokens_seen": 99808,
+      "step": 6
+    },
+    {
+      "epoch": 0.09003215434083602,
+      "grad_norm": 266.3240051269531,
+      "learning_rate": 5.833333333333334e-08,
+      "loss": 8.0182,
+      "num_input_tokens_seen": 115808,
+      "step": 7
+    },
+    {
+      "epoch": 0.10289389067524116,
+      "grad_norm": 274.253662109375,
+      "learning_rate": 6.666666666666668e-08,
+      "loss": 8.0363,
+      "num_input_tokens_seen": 131392,
+      "step": 8
+    },
+    {
+      "epoch": 0.1157556270096463,
+      "grad_norm": 263.321533203125,
+      "learning_rate": 7.500000000000001e-08,
+      "loss": 8.0918,
+      "num_input_tokens_seen": 147840,
+      "step": 9
+    },
+    {
+      "epoch": 0.12861736334405144,
+      "grad_norm": 266.48394775390625,
+      "learning_rate": 8.333333333333334e-08,
+      "loss": 8.0197,
+      "num_input_tokens_seen": 164064,
+      "step": 10
+    },
+    {
+      "epoch": 0.1414790996784566,
+      "grad_norm": 258.7409362792969,
+      "learning_rate": 9.166666666666668e-08,
+      "loss": 7.9714,
+      "num_input_tokens_seen": 180704,
+      "step": 11
+    },
+    {
+      "epoch": 0.15434083601286175,
+      "grad_norm": 271.2587585449219,
+      "learning_rate": 1.0000000000000001e-07,
+      "loss": 8.0131,
+      "num_input_tokens_seen": 196064,
+      "step": 12
+    },
+    {
+      "epoch": 0.16720257234726688,
+      "grad_norm": 255.18431091308594,
+      "learning_rate": 1.0833333333333335e-07,
+      "loss": 7.9796,
+      "num_input_tokens_seen": 211424,
+      "step": 13
+    },
+    {
+      "epoch": 0.18006430868167203,
+      "grad_norm": 269.7242736816406,
+      "learning_rate": 1.1666666666666668e-07,
+      "loss": 7.9999,
+      "num_input_tokens_seen": 228096,
+      "step": 14
+    },
+    {
+      "epoch": 0.19292604501607716,
+      "grad_norm": 264.5142517089844,
+      "learning_rate": 1.2500000000000002e-07,
+      "loss": 8.0168,
+      "num_input_tokens_seen": 244416,
+      "step": 15
+    },
+    {
+      "epoch": 0.2057877813504823,
+      "grad_norm": 267.4490661621094,
+      "learning_rate": 1.3333333333333336e-07,
+      "loss": 7.8423,
+      "num_input_tokens_seen": 261056,
+      "step": 16
+    },
+    {
+      "epoch": 0.21864951768488747,
+      "grad_norm": 269.4408874511719,
+      "learning_rate": 1.4166666666666668e-07,
+      "loss": 7.9208,
+      "num_input_tokens_seen": 276992,
+      "step": 17
+    },
+    {
+      "epoch": 0.2315112540192926,
+      "grad_norm": 256.512939453125,
+      "learning_rate": 1.5000000000000002e-07,
+      "loss": 7.6898,
+      "num_input_tokens_seen": 293536,
+      "step": 18
+    },
+    {
+      "epoch": 0.24437299035369775,
+      "grad_norm": 265.34490966796875,
+      "learning_rate": 1.5833333333333336e-07,
+      "loss": 7.9285,
+      "num_input_tokens_seen": 309024,
+      "step": 19
+    },
+    {
+      "epoch": 0.2572347266881029,
+      "grad_norm": 268.9532470703125,
+      "learning_rate": 1.6666666666666668e-07,
+      "loss": 7.8924,
+      "num_input_tokens_seen": 325024,
+      "step": 20
+    },
+    {
+      "epoch": 0.27009646302250806,
+      "grad_norm": 268.1192321777344,
+      "learning_rate": 1.7500000000000002e-07,
+      "loss": 7.6173,
+      "num_input_tokens_seen": 341856,
+      "step": 21
+    },
+    {
+      "epoch": 0.2829581993569132,
+      "grad_norm": 271.13006591796875,
+      "learning_rate": 1.8333333333333336e-07,
+      "loss": 7.7661,
+      "num_input_tokens_seen": 358848,
+      "step": 22
+    },
+    {
+      "epoch": 0.2958199356913183,
+      "grad_norm": 260.5618591308594,
+      "learning_rate": 1.9166666666666668e-07,
+      "loss": 7.2936,
+      "num_input_tokens_seen": 375360,
+      "step": 23
+    },
+    {
+      "epoch": 0.3086816720257235,
+      "grad_norm": 249.86729431152344,
+      "learning_rate": 2.0000000000000002e-07,
+      "loss": 7.3044,
+      "num_input_tokens_seen": 392608,
+      "step": 24
+    },
+    {
+      "epoch": 0.3215434083601286,
+      "grad_norm": 257.8570861816406,
+      "learning_rate": 2.0833333333333333e-07,
+      "loss": 7.35,
+      "num_input_tokens_seen": 408800,
+      "step": 25
+    },
+    {
+      "epoch": 0.33440514469453375,
+      "grad_norm": 246.40611267089844,
+      "learning_rate": 2.166666666666667e-07,
+      "loss": 7.1213,
+      "num_input_tokens_seen": 425632,
+      "step": 26
+    },
+    {
+      "epoch": 0.34726688102893893,
+      "grad_norm": 252.2755889892578,
+      "learning_rate": 2.2500000000000002e-07,
+      "loss": 7.1363,
+      "num_input_tokens_seen": 441952,
+      "step": 27
+    },
+    {
+      "epoch": 0.36012861736334406,
+      "grad_norm": 235.94859313964844,
+      "learning_rate": 2.3333333333333336e-07,
+      "loss": 6.9016,
+      "num_input_tokens_seen": 458080,
+      "step": 28
+    },
+    {
+      "epoch": 0.3729903536977492,
+      "grad_norm": 243.25863647460938,
+      "learning_rate": 2.416666666666667e-07,
+      "loss": 6.9764,
+      "num_input_tokens_seen": 473824,
+      "step": 29
+    },
+    {
+      "epoch": 0.3858520900321543,
+      "grad_norm": 250.39564514160156,
+      "learning_rate": 2.5000000000000004e-07,
+      "loss": 6.9005,
+      "num_input_tokens_seen": 490336,
+      "step": 30
+    },
+    {
+      "epoch": 0.3987138263665595,
+      "grad_norm": 253.6007843017578,
+      "learning_rate": 2.5833333333333333e-07,
+      "loss": 6.9852,
+      "num_input_tokens_seen": 506976,
+      "step": 31
+    },
+    {
+      "epoch": 0.4115755627009646,
+      "grad_norm": 155.81201171875,
+      "learning_rate": 2.666666666666667e-07,
+      "loss": 5.834,
+      "num_input_tokens_seen": 523520,
+      "step": 32
+    },
+    {
+      "epoch": 0.42443729903536975,
+      "grad_norm": 124.55789947509766,
+      "learning_rate": 2.75e-07,
+      "loss": 5.5452,
+      "num_input_tokens_seen": 540672,
+      "step": 33
+    },
+    {
+      "epoch": 0.43729903536977494,
+      "grad_norm": 130.3427276611328,
+      "learning_rate": 2.8333333333333336e-07,
+      "loss": 5.5189,
+      "num_input_tokens_seen": 557696,
+      "step": 34
+    },
+    {
+      "epoch": 0.45016077170418006,
+      "grad_norm": 126.62003326416016,
+      "learning_rate": 2.916666666666667e-07,
+      "loss": 5.4654,
+      "num_input_tokens_seen": 574400,
+      "step": 35
+    },
+    {
+      "epoch": 0.4630225080385852,
+      "grad_norm": 120.25413513183594,
+      "learning_rate": 3.0000000000000004e-07,
+      "loss": 5.3117,
+      "num_input_tokens_seen": 590112,
+      "step": 36
+    },
+    {
+      "epoch": 0.4758842443729904,
+      "grad_norm": 123.52056884765625,
+      "learning_rate": 3.083333333333334e-07,
+      "loss": 5.2744,
+      "num_input_tokens_seen": 605920,
+      "step": 37
+    },
+    {
+      "epoch": 0.4887459807073955,
+      "grad_norm": 124.25894927978516,
+      "learning_rate": 3.166666666666667e-07,
+      "loss": 5.2821,
+      "num_input_tokens_seen": 622208,
+      "step": 38
+    },
+    {
+      "epoch": 0.5016077170418006,
+      "grad_norm": 111.38367462158203,
+      "learning_rate": 3.25e-07,
+      "loss": 5.0967,
+      "num_input_tokens_seen": 638464,
+      "step": 39
+    },
+    {
+      "epoch": 0.5144694533762058,
+      "grad_norm": 119.48200225830078,
+      "learning_rate": 3.3333333333333335e-07,
+      "loss": 5.2167,
+      "num_input_tokens_seen": 654656,
+      "step": 40
+    },
+    {
+      "epoch": 0.5273311897106109,
+      "grad_norm": 108.62528991699219,
+      "learning_rate": 3.416666666666667e-07,
+      "loss": 5.1517,
+      "num_input_tokens_seen": 671424,
+      "step": 41
+    },
+    {
+      "epoch": 0.5401929260450161,
+      "grad_norm": 109.66475677490234,
+      "learning_rate": 3.5000000000000004e-07,
+      "loss": 4.6959,
+      "num_input_tokens_seen": 687968,
+      "step": 42
+    },
+    {
+      "epoch": 0.5530546623794212,
+      "grad_norm": 107.45171356201172,
+      "learning_rate": 3.583333333333334e-07,
+      "loss": 4.1458,
+      "num_input_tokens_seen": 704224,
+      "step": 43
+    },
+    {
+      "epoch": 0.5659163987138264,
+      "grad_norm": 102.30545806884766,
+      "learning_rate": 3.666666666666667e-07,
+      "loss": 3.6692,
+      "num_input_tokens_seen": 720704,
+      "step": 44
+    },
+    {
+      "epoch": 0.5787781350482315,
+      "grad_norm": 96.8830795288086,
+      "learning_rate": 3.75e-07,
+      "loss": 3.3322,
+      "num_input_tokens_seen": 737408,
+      "step": 45
+    },
+    {
+      "epoch": 0.5916398713826366,
+      "grad_norm": 102.91023254394531,
+      "learning_rate": 3.8333333333333335e-07,
+      "loss": 3.2857,
+      "num_input_tokens_seen": 753728,
+      "step": 46
+    },
+    {
+      "epoch": 0.6045016077170418,
+      "grad_norm": 105.37537384033203,
+      "learning_rate": 3.9166666666666675e-07,
+      "loss": 3.0806,
+      "num_input_tokens_seen": 770080,
+      "step": 47
+    },
+    {
+      "epoch": 0.617363344051447,
+      "grad_norm": 98.45983123779297,
+      "learning_rate": 4.0000000000000003e-07,
+      "loss": 2.8239,
+      "num_input_tokens_seen": 786912,
+      "step": 48
+    },
+    {
+      "epoch": 0.6302250803858521,
+      "grad_norm": 108.93148040771484,
+      "learning_rate": 4.083333333333334e-07,
+      "loss": 2.8695,
+      "num_input_tokens_seen": 804192,
+      "step": 49
+    },
+    {
+      "epoch": 0.6430868167202572,
+      "grad_norm": 100.31180572509766,
+      "learning_rate": 4.1666666666666667e-07,
+      "loss": 2.6218,
+      "num_input_tokens_seen": 820928,
+      "step": 50
+    },
+    {
+      "epoch": 0.6559485530546624,
+      "grad_norm": 101.31784057617188,
+      "learning_rate": 4.2500000000000006e-07,
+      "loss": 2.3993,
+      "num_input_tokens_seen": 837152,
+      "step": 51
+    },
+    {
+      "epoch": 0.6688102893890675,
+      "grad_norm": 101.09466552734375,
+      "learning_rate": 4.333333333333334e-07,
+      "loss": 2.2828,
+      "num_input_tokens_seen": 853568,
+      "step": 52
+    },
+    {
+      "epoch": 0.6816720257234726,
+      "grad_norm": 99.55025482177734,
+      "learning_rate": 4.416666666666667e-07,
+      "loss": 2.2063,
+      "num_input_tokens_seen": 870592,
+      "step": 53
+    },
+    {
+      "epoch": 0.6945337620578779,
+      "grad_norm": 104.6786117553711,
+      "learning_rate": 4.5000000000000003e-07,
+      "loss": 1.8878,
+      "num_input_tokens_seen": 886080,
+      "step": 54
+    },
+    {
+      "epoch": 0.707395498392283,
+      "grad_norm": 106.05968475341797,
+      "learning_rate": 4.583333333333333e-07,
+      "loss": 1.5971,
+      "num_input_tokens_seen": 903296,
+      "step": 55
+    },
+    {
+      "epoch": 0.7202572347266881,
+      "grad_norm": 93.43476867675781,
+      "learning_rate": 4.666666666666667e-07,
+      "loss": 0.9604,
+      "num_input_tokens_seen": 919296,
+      "step": 56
+    },
+    {
+      "epoch": 0.7331189710610932,
+      "grad_norm": 60.13694763183594,
+      "learning_rate": 4.7500000000000006e-07,
+      "loss": 0.6287,
+      "num_input_tokens_seen": 936096,
+      "step": 57
+    },
+    {
+      "epoch": 0.7459807073954984,
+      "grad_norm": 42.870086669921875,
+      "learning_rate": 4.833333333333334e-07,
+      "loss": 0.4844,
+      "num_input_tokens_seen": 953024,
+      "step": 58
+    },
+    {
+      "epoch": 0.7588424437299035,
+      "grad_norm": 45.984989166259766,
+      "learning_rate": 4.916666666666667e-07,
+      "loss": 0.3891,
+      "num_input_tokens_seen": 969024,
+      "step": 59
+    },
+    {
+      "epoch": 0.7717041800643086,
+      "grad_norm": 44.375484466552734,
+      "learning_rate": 5.000000000000001e-07,
+      "loss": 0.357,
+      "num_input_tokens_seen": 985600,
+      "step": 60
+    },
+    {
+      "epoch": 0.7845659163987139,
+      "grad_norm": 28.2038516998291,
+      "learning_rate": 5.083333333333334e-07,
+      "loss": 0.2854,
+      "num_input_tokens_seen": 1002336,
+      "step": 61
+    },
+    {
+      "epoch": 0.797427652733119,
+      "grad_norm": 13.842344284057617,
+      "learning_rate": 5.166666666666667e-07,
+      "loss": 0.2403,
+      "num_input_tokens_seen": 1018496,
+      "step": 62
+    },
+    {
+      "epoch": 0.8102893890675241,
+      "grad_norm": 21.351642608642578,
+      "learning_rate": 5.250000000000001e-07,
+      "loss": 0.2522,
+      "num_input_tokens_seen": 1035552,
+      "step": 63
+    },
+    {
+      "epoch": 0.8231511254019293,
+      "grad_norm": 21.38069725036621,
+      "learning_rate": 5.333333333333335e-07,
+      "loss": 0.2339,
+      "num_input_tokens_seen": 1051584,
+      "step": 64
+    },
+    {
+      "epoch": 0.8360128617363344,
+      "grad_norm": 9.553906440734863,
+      "learning_rate": 5.416666666666667e-07,
+      "loss": 0.2214,
+      "num_input_tokens_seen": 1068640,
+      "step": 65
+    },
+    {
+      "epoch": 0.8488745980707395,
+      "grad_norm": 6.100700855255127,
+      "learning_rate": 5.5e-07,
+      "loss": 0.1912,
+      "num_input_tokens_seen": 1085568,
+      "step": 66
+    },
+    {
+      "epoch": 0.8617363344051447,
+      "grad_norm": 18.959564208984375,
+      "learning_rate": 5.583333333333333e-07,
+      "loss": 0.2157,
+      "num_input_tokens_seen": 1102784,
+      "step": 67
+    },
+    {
+      "epoch": 0.8745980707395499,
+      "grad_norm": 16.303438186645508,
+      "learning_rate": 5.666666666666667e-07,
+      "loss": 0.2045,
+      "num_input_tokens_seen": 1118432,
+      "step": 68
+    },
+    {
+      "epoch": 0.887459807073955,
+      "grad_norm": 12.491047859191895,
+      "learning_rate": 5.750000000000001e-07,
+      "loss": 0.1964,
+      "num_input_tokens_seen": 1135232,
+      "step": 69
+    },
+    {
+      "epoch": 0.9003215434083601,
+      "grad_norm": 16.655309677124023,
+      "learning_rate": 5.833333333333334e-07,
+      "loss": 0.1901,
+      "num_input_tokens_seen": 1150496,
+      "step": 70
+    },
+    {
+      "epoch": 0.9131832797427653,
+      "grad_norm": 12.015859603881836,
+      "learning_rate": 5.916666666666667e-07,
+      "loss": 0.1891,
+      "num_input_tokens_seen": 1166848,
+      "step": 71
+    },
+    {
+      "epoch": 0.9260450160771704,
+      "grad_norm": 12.255152702331543,
+      "learning_rate": 6.000000000000001e-07,
+      "loss": 0.1841,
+      "num_input_tokens_seen": 1183552,
+      "step": 72
+    },
+    {
+      "epoch": 0.9389067524115756,
+      "grad_norm": 5.163099765777588,
+      "learning_rate": 6.083333333333334e-07,
+      "loss": 0.1803,
+      "num_input_tokens_seen": 1199968,
+      "step": 73
+    },
+    {
+      "epoch": 0.9517684887459807,
+      "grad_norm": 10.502095222473145,
+      "learning_rate": 6.166666666666668e-07,
+      "loss": 0.1657,
+      "num_input_tokens_seen": 1216256,
+      "step": 74
+    },
+    {
+      "epoch": 0.9646302250803859,
+      "grad_norm": 23.80043601989746,
+      "learning_rate": 6.25e-07,
+      "loss": 0.2157,
+      "num_input_tokens_seen": 1232736,
+      "step": 75
+    },
+    {
+      "epoch": 0.977491961414791,
+      "grad_norm": 19.7410831451416,
+      "learning_rate": 6.333333333333334e-07,
+      "loss": 0.2,
+      "num_input_tokens_seen": 1249248,
+      "step": 76
+    },
+    {
+      "epoch": 0.9903536977491961,
+      "grad_norm": 4.730998516082764,
+      "learning_rate": 6.416666666666667e-07,
+      "loss": 0.1519,
+      "num_input_tokens_seen": 1265152,
+      "step": 77
+    },
+    {
+      "epoch": 1.0032154340836013,
+      "grad_norm": 14.700318336486816,
+      "learning_rate": 6.5e-07,
+      "loss": 0.1765,
+      "num_input_tokens_seen": 1282336,
+      "step": 78
+    },
+    {
+      "epoch": 1.0160771704180065,
+      "grad_norm": 18.6468448638916,
+      "learning_rate": 6.583333333333333e-07,
+      "loss": 0.1907,
+      "num_input_tokens_seen": 1299168,
+      "step": 79
+    },
+    {
+      "epoch": 1.0289389067524115,
+      "grad_norm": 11.44071102142334,
+      "learning_rate": 6.666666666666667e-07,
+      "loss": 0.1593,
+      "num_input_tokens_seen": 1314816,
+      "step": 80
+    },
+    {
+      "epoch": 1.0418006430868167,
+      "grad_norm": 3.6438238620758057,
+      "learning_rate": 6.750000000000001e-07,
+      "loss": 0.1659,
+      "num_input_tokens_seen": 1331168,
+      "step": 81
+    },
+    {
+      "epoch": 1.0546623794212218,
+      "grad_norm": 3.6471991539001465,
+      "learning_rate": 6.833333333333334e-07,
+      "loss": 0.1619,
+      "num_input_tokens_seen": 1347968,
+      "step": 82
+    },
+    {
+      "epoch": 1.067524115755627,
+      "grad_norm": 4.136232376098633,
+      "learning_rate": 6.916666666666668e-07,
+      "loss": 0.168,
+      "num_input_tokens_seen": 1363872,
+      "step": 83
+    },
+    {
+      "epoch": 1.0803858520900322,
+      "grad_norm": 4.436996936798096,
+      "learning_rate": 7.000000000000001e-07,
+      "loss": 0.1627,
+      "num_input_tokens_seen": 1380896,
+      "step": 84
+    },
+    {
+      "epoch": 1.0932475884244373,
+      "grad_norm": 10.516170501708984,
+      "learning_rate": 7.083333333333334e-07,
+      "loss": 0.1421,
+      "num_input_tokens_seen": 1397792,
+      "step": 85
+    },
+    {
+      "epoch": 1.1061093247588425,
+      "grad_norm": 15.584237098693848,
+      "learning_rate": 7.166666666666668e-07,
+      "loss": 0.1517,
+      "num_input_tokens_seen": 1414272,
+      "step": 86
+    },
+    {
+      "epoch": 1.1189710610932475,
+      "grad_norm": 8.475227355957031,
+      "learning_rate": 7.25e-07,
+      "loss": 0.1519,
+      "num_input_tokens_seen": 1429696,
+      "step": 87
+    },
+    {
+      "epoch": 1.1318327974276527,
+      "grad_norm": 5.768808841705322,
+      "learning_rate": 7.333333333333334e-07,
+      "loss": 0.128,
+      "num_input_tokens_seen": 1446720,
+      "step": 88
+    },
+    {
+      "epoch": 1.144694533762058,
+      "grad_norm": 15.064901351928711,
+      "learning_rate": 7.416666666666668e-07,
+      "loss": 0.1481,
+      "num_input_tokens_seen": 1463008,
+      "step": 89
+    },
+    {
+      "epoch": 1.157556270096463,
+      "grad_norm": 17.70471954345703,
+      "learning_rate": 7.5e-07,
+      "loss": 0.1636,
+      "num_input_tokens_seen": 1480096,
+      "step": 90
+    },
+    {
+      "epoch": 1.1704180064308682,
+      "grad_norm": 5.037917613983154,
+      "learning_rate": 7.583333333333334e-07,
+      "loss": 0.1443,
+      "num_input_tokens_seen": 1497024,
+      "step": 91
+    },
+    {
+      "epoch": 1.1832797427652733,
+      "grad_norm": 17.397666931152344,
+      "learning_rate": 7.666666666666667e-07,
+      "loss": 0.1592,
+      "num_input_tokens_seen": 1513792,
+      "step": 92
+    },
+    {
+      "epoch": 1.1961414790996785,
+      "grad_norm": 18.808683395385742,
+      "learning_rate": 7.750000000000001e-07,
+      "loss": 0.1744,
+      "num_input_tokens_seen": 1529696,
+      "step": 93
+    },
+    {
+      "epoch": 1.2090032154340835,
+      "grad_norm": 15.173974990844727,
+      "learning_rate": 7.833333333333335e-07,
+      "loss": 0.1615,
+      "num_input_tokens_seen": 1546400,
+      "step": 94
+    },
+    {
+      "epoch": 1.2218649517684887,
+      "grad_norm": 3.6459403038024902,
+      "learning_rate": 7.916666666666667e-07,
+      "loss": 0.1466,
+      "num_input_tokens_seen": 1564448,
+      "step": 95
+    },
+    {
+      "epoch": 1.234726688102894,
+      "grad_norm": 4.910373210906982,
+      "learning_rate": 8.000000000000001e-07,
+      "loss": 0.1333,
+      "num_input_tokens_seen": 1580736,
+      "step": 96
+    },
+    {
+      "epoch": 1.247588424437299,
+      "grad_norm": 11.399758338928223,
+      "learning_rate": 8.083333333333334e-07,
+      "loss": 0.1475,
+      "num_input_tokens_seen": 1596928,
+      "step": 97
+    },
+    {
+      "epoch": 1.2604501607717042,
+      "grad_norm": 4.701439380645752,
+      "learning_rate": 8.166666666666668e-07,
+      "loss": 0.142,
+      "num_input_tokens_seen": 1613728,
+      "step": 98
+    },
+    {
+      "epoch": 1.2733118971061093,
+      "grad_norm": 7.250323295593262,
+      "learning_rate": 8.250000000000001e-07,
+      "loss": 0.148,
+      "num_input_tokens_seen": 1630112,
+      "step": 99
+    },
+    {
+      "epoch": 1.2861736334405145,
+      "grad_norm": 17.24268341064453,
+      "learning_rate": 8.333333333333333e-07,
+      "loss": 0.1539,
+      "num_input_tokens_seen": 1645248,
+      "step": 100
+    },
+    {
+      "epoch": 1.2990353697749195,
+      "grad_norm": 4.017528057098389,
+      "learning_rate": 8.416666666666667e-07,
+      "loss": 0.1496,
+      "num_input_tokens_seen": 1662400,
+      "step": 101
+    },
+    {
+      "epoch": 1.3118971061093248,
+      "grad_norm": 10.226727485656738,
+      "learning_rate": 8.500000000000001e-07,
+      "loss": 0.1563,
+      "num_input_tokens_seen": 1678528,
+      "step": 102
+    },
+    {
+      "epoch": 1.32475884244373,
+      "grad_norm": 4.085242748260498,
+      "learning_rate": 8.583333333333334e-07,
+      "loss": 0.1295,
+      "num_input_tokens_seen": 1694432,
+      "step": 103
+    },
+    {
+      "epoch": 1.337620578778135,
+      "grad_norm": 9.983370780944824,
+      "learning_rate": 8.666666666666668e-07,
+      "loss": 0.1353,
+      "num_input_tokens_seen": 1710624,
+      "step": 104
+    },
+    {
+      "epoch": 1.3504823151125402,
+      "grad_norm": 6.53292179107666,
+      "learning_rate": 8.75e-07,
+      "loss": 0.1376,
+      "num_input_tokens_seen": 1726560,
+      "step": 105
+    },
+    {
+      "epoch": 1.3633440514469453,
+      "grad_norm": 4.412125587463379,
+      "learning_rate": 8.833333333333334e-07,
+      "loss": 0.1313,
+      "num_input_tokens_seen": 1742912,
+      "step": 106
+    },
+    {
+      "epoch": 1.3762057877813505,
+      "grad_norm": 13.620604515075684,
+      "learning_rate": 8.916666666666668e-07,
+      "loss": 0.1367,
+      "num_input_tokens_seen": 1758688,
+      "step": 107
+    },
+    {
+      "epoch": 1.3890675241157555,
+      "grad_norm": 6.362453937530518,
+      "learning_rate": 9.000000000000001e-07,
+      "loss": 0.135,
+      "num_input_tokens_seen": 1775264,
+      "step": 108
+    },
+    {
+      "epoch": 1.4019292604501608,
+      "grad_norm": 3.8915514945983887,
+      "learning_rate": 9.083333333333335e-07,
+      "loss": 0.1212,
+      "num_input_tokens_seen": 1791904,
+      "step": 109
+    },
+    {
+      "epoch": 1.414790996784566,
+      "grad_norm": 9.137332916259766,
+      "learning_rate": 9.166666666666666e-07,
+      "loss": 0.1355,
+      "num_input_tokens_seen": 1808736,
+      "step": 110
+    },
+    {
+      "epoch": 1.427652733118971,
+      "grad_norm": 4.745510578155518,
+      "learning_rate": 9.25e-07,
+      "loss": 0.1439,
+      "num_input_tokens_seen": 1825504,
+      "step": 111
+    },
+    {
+      "epoch": 1.4405144694533762,
+      "grad_norm": 10.531625747680664,
+      "learning_rate": 9.333333333333334e-07,
+      "loss": 0.1323,
+      "num_input_tokens_seen": 1842080,
+      "step": 112
+    },
+    {
+      "epoch": 1.4533762057877815,
+      "grad_norm": 11.189078330993652,
+      "learning_rate": 9.416666666666667e-07,
+      "loss": 0.1376,
+      "num_input_tokens_seen": 1858432,
+      "step": 113
+    },
+    {
+      "epoch": 1.4662379421221865,
+      "grad_norm": 5.619433879852295,
+      "learning_rate": 9.500000000000001e-07,
+      "loss": 0.1191,
+      "num_input_tokens_seen": 1874304,
+      "step": 114
+    },
+    {
+      "epoch": 1.4790996784565915,
+      "grad_norm": 4.590907096862793,
+      "learning_rate": 9.583333333333334e-07,
+      "loss": 0.1045,
+      "num_input_tokens_seen": 1890144,
+      "step": 115
+    },
+    {
+      "epoch": 1.4919614147909968,
+      "grad_norm": 4.900210380554199,
+      "learning_rate": 9.666666666666668e-07,
+      "loss": 0.1467,
+      "num_input_tokens_seen": 1905664,
+      "step": 116
+    },
+    {
+      "epoch": 1.504823151125402,
+      "grad_norm": 3.0244414806365967,
+      "learning_rate": 9.750000000000002e-07,
+      "loss": 0.1142,
+      "num_input_tokens_seen": 1922240,
+      "step": 117
+    },
+    {
+      "epoch": 1.517684887459807,
+      "grad_norm": 2.886939287185669,
+      "learning_rate": 9.833333333333334e-07,
+      "loss": 0.1107,
+      "num_input_tokens_seen": 1938240,
+      "step": 118
+    },
+    {
+      "epoch": 1.5305466237942122,
+      "grad_norm": 6.869494438171387,
+      "learning_rate": 9.916666666666668e-07,
+      "loss": 0.146,
+      "num_input_tokens_seen": 1955008,
+      "step": 119
+    },
+    {
+      "epoch": 1.5434083601286175,
+      "grad_norm": 16.34566879272461,
+      "learning_rate": 1.0000000000000002e-06,
+      "loss": 0.1533,
+      "num_input_tokens_seen": 1971456,
+      "step": 120
+    },
+    {
+      "epoch": 1.5562700964630225,
+      "grad_norm": 15.347967147827148,
+      "learning_rate": 1.0083333333333333e-06,
+      "loss": 0.1315,
+      "num_input_tokens_seen": 1987552,
+      "step": 121
+    },
+    {
+      "epoch": 1.5691318327974275,
+      "grad_norm": 4.213266372680664,
+      "learning_rate": 1.0166666666666667e-06,
+      "loss": 0.1197,
+      "num_input_tokens_seen": 2004192,
+      "step": 122
+    },
+    {
+      "epoch": 1.5819935691318328,
+      "grad_norm": 14.208905220031738,
+      "learning_rate": 1.025e-06,
+      "loss": 0.1228,
+      "num_input_tokens_seen": 2020832,
+      "step": 123
+    },
+    {
+      "epoch": 1.594855305466238,
+      "grad_norm": 17.388845443725586,
+      "learning_rate": 1.0333333333333333e-06,
+      "loss": 0.1273,
+      "num_input_tokens_seen": 2037312,
+      "step": 124
+    },
+    {
+      "epoch": 1.607717041800643,
+      "grad_norm": 14.383207321166992,
+      "learning_rate": 1.0416666666666667e-06,
+      "loss": 0.141,
+      "num_input_tokens_seen": 2054304,
+      "step": 125
+    },
+    {
+      "epoch": 1.6205787781350482,
+      "grad_norm": 6.2650370597839355,
+      "learning_rate": 1.0500000000000001e-06,
+      "loss": 0.1315,
+      "num_input_tokens_seen": 2071136,
+      "step": 126
+    },
+    {
+      "epoch": 1.6334405144694535,
+      "grad_norm": 8.396490097045898,
+      "learning_rate": 1.0583333333333335e-06,
+      "loss": 0.1136,
+      "num_input_tokens_seen": 2088064,
+      "step": 127
+    },
+    {
+      "epoch": 1.6463022508038585,
+      "grad_norm": 4.598783016204834,
+      "learning_rate": 1.066666666666667e-06,
+      "loss": 0.1013,
+      "num_input_tokens_seen": 2105120,
+      "step": 128
+    },
+    {
+      "epoch": 1.6591639871382635,
+      "grad_norm": 9.912994384765625,
+      "learning_rate": 1.075e-06,
+      "loss": 0.1056,
+      "num_input_tokens_seen": 2120992,
+      "step": 129
+    },
+    {
+      "epoch": 1.6720257234726688,
+      "grad_norm": 7.70595645904541,
+      "learning_rate": 1.0833333333333335e-06,
+      "loss": 0.1071,
+      "num_input_tokens_seen": 2137984,
+      "step": 130
+    },
+    {
+      "epoch": 1.684887459807074,
+      "grad_norm": 6.183793544769287,
+      "learning_rate": 1.0916666666666667e-06,
+      "loss": 0.1357,
+      "num_input_tokens_seen": 2154240,
+      "step": 131
+    },
+    {
+      "epoch": 1.697749196141479,
+      "grad_norm": 6.863802909851074,
+      "learning_rate": 1.1e-06,
+      "loss": 0.1181,
+      "num_input_tokens_seen": 2170464,
+      "step": 132
+    },
+    {
+      "epoch": 1.7106109324758842,
+      "grad_norm": 2.7540361881256104,
+      "learning_rate": 1.1083333333333335e-06,
+      "loss": 0.0826,
+      "num_input_tokens_seen": 2187008,
+      "step": 133
+    },
+    {
+      "epoch": 1.7234726688102895,
+      "grad_norm": 12.97872257232666,
+      "learning_rate": 1.1166666666666666e-06,
+      "loss": 0.1221,
+      "num_input_tokens_seen": 2202880,
+      "step": 134
+    },
+    {
+      "epoch": 1.7363344051446945,
+      "grad_norm": 4.774652004241943,
+      "learning_rate": 1.125e-06,
+      "loss": 0.1021,
+      "num_input_tokens_seen": 2219744,
+      "step": 135
+    },
+    {
+      "epoch": 1.7491961414790995,
+      "grad_norm": 3.889441967010498,
+      "learning_rate": 1.1333333333333334e-06,
+      "loss": 0.098,
+      "num_input_tokens_seen": 2236064,
+      "step": 136
+    },
+    {
+      "epoch": 1.762057877813505,
+      "grad_norm": 5.095253944396973,
+      "learning_rate": 1.1416666666666668e-06,
+      "loss": 0.1085,
+      "num_input_tokens_seen": 2252832,
+      "step": 137
+    },
+    {
+      "epoch": 1.77491961414791,
+      "grad_norm": 5.206950664520264,
+      "learning_rate": 1.1500000000000002e-06,
+      "loss": 0.1038,
+      "num_input_tokens_seen": 2268704,
+      "step": 138
+    },
+    {
+      "epoch": 1.787781350482315,
+      "grad_norm": 10.339069366455078,
+      "learning_rate": 1.1583333333333334e-06,
+      "loss": 0.1081,
+      "num_input_tokens_seen": 2285280,
+      "step": 139
+    },
+    {
+      "epoch": 1.8006430868167203,
+      "grad_norm": 8.227212905883789,
+      "learning_rate": 1.1666666666666668e-06,
+      "loss": 0.1287,
+      "num_input_tokens_seen": 2301600,
+      "step": 140
+    },
+    {
+      "epoch": 1.8135048231511255,
+      "grad_norm": 6.72472620010376,
+      "learning_rate": 1.175e-06,
+      "loss": 0.1068,
+      "num_input_tokens_seen": 2318048,
+      "step": 141
+    },
+    {
+      "epoch": 1.8263665594855305,
+      "grad_norm": 10.02261734008789,
+      "learning_rate": 1.1833333333333334e-06,
+      "loss": 0.1202,
+      "num_input_tokens_seen": 2333728,
+      "step": 142
+    },
+    {
+      "epoch": 1.8392282958199357,
+      "grad_norm": 10.770837783813477,
+      "learning_rate": 1.1916666666666668e-06,
+      "loss": 0.119,
+      "num_input_tokens_seen": 2349760,
+      "step": 143
+    },
+    {
+      "epoch": 1.852090032154341,
+      "grad_norm": 6.685535907745361,
+      "learning_rate": 1.2000000000000002e-06,
+      "loss": 0.1273,
+      "num_input_tokens_seen": 2366336,
+      "step": 144
+    },
+    {
+      "epoch": 1.864951768488746,
+      "grad_norm": 4.437434673309326,
+      "learning_rate": 1.2083333333333333e-06,
+      "loss": 0.1024,
+      "num_input_tokens_seen": 2382016,
+      "step": 145
+    },
+    {
+      "epoch": 1.877813504823151,
+      "grad_norm": 11.942359924316406,
+      "learning_rate": 1.2166666666666667e-06,
+      "loss": 0.1143,
+      "num_input_tokens_seen": 2398176,
+      "step": 146
+    },
+    {
+      "epoch": 1.8906752411575563,
+      "grad_norm": 6.428237438201904,
+      "learning_rate": 1.2250000000000001e-06,
+      "loss": 0.1229,
+      "num_input_tokens_seen": 2414656,
+      "step": 147
+    },
+    {
+      "epoch": 1.9035369774919615,
+      "grad_norm": 7.896732807159424,
+      "learning_rate": 1.2333333333333335e-06,
+      "loss": 0.0964,
+      "num_input_tokens_seen": 2430400,
+      "step": 148
+    },
+    {
+      "epoch": 1.9163987138263665,
+      "grad_norm": 12.14167594909668,
+      "learning_rate": 1.2416666666666667e-06,
+      "loss": 0.1132,
+      "num_input_tokens_seen": 2447520,
+      "step": 149
+    },
+    {
+      "epoch": 1.9292604501607717,
+      "grad_norm": 3.0627174377441406,
+      "learning_rate": 1.25e-06,
+      "loss": 0.0734,
+      "num_input_tokens_seen": 2463872,
+      "step": 150
+    },
+    {
+      "epoch": 1.942122186495177,
+      "grad_norm": 3.2333552837371826,
+      "learning_rate": 1.2583333333333333e-06,
+      "loss": 0.0939,
+      "num_input_tokens_seen": 2481344,
+      "step": 151
+    },
+    {
+      "epoch": 1.954983922829582,
+      "grad_norm": 14.160690307617188,
+      "learning_rate": 1.2666666666666669e-06,
+      "loss": 0.1143,
+      "num_input_tokens_seen": 2497632,
+      "step": 152
+    },
+    {
+      "epoch": 1.967845659163987,
+      "grad_norm": 5.38826322555542,
+      "learning_rate": 1.275e-06,
+      "loss": 0.1114,
+      "num_input_tokens_seen": 2513024,
+      "step": 153
+    },
+    {
+      "epoch": 1.9807073954983923,
+      "grad_norm": 3.4523892402648926,
+      "learning_rate": 1.2833333333333335e-06,
+      "loss": 0.0948,
+      "num_input_tokens_seen": 2528800,
+      "step": 154
+    },
+    {
+      "epoch": 1.9935691318327975,
+      "grad_norm": 6.9574127197265625,
+      "learning_rate": 1.2916666666666669e-06,
+      "loss": 0.0805,
+      "num_input_tokens_seen": 2545696,
+      "step": 155
+    },
+    {
+      "epoch": 2.0064308681672025,
+      "grad_norm": 7.220541954040527,
+      "learning_rate": 1.3e-06,
+      "loss": 0.1001,
+      "num_input_tokens_seen": 2561728,
+      "step": 156
+    },
+    {
+      "epoch": 2.0192926045016075,
+      "grad_norm": 6.95176887512207,
+      "learning_rate": 1.3083333333333334e-06,
+      "loss": 0.1002,
+      "num_input_tokens_seen": 2577856,
+      "step": 157
+    },
+    {
+      "epoch": 2.032154340836013,
+      "grad_norm": 10.752556800842285,
+      "learning_rate": 1.3166666666666666e-06,
+      "loss": 0.0847,
+      "num_input_tokens_seen": 2593984,
+      "step": 158
+    },
+    {
+      "epoch": 2.045016077170418,
+      "grad_norm": 6.510659217834473,
+      "learning_rate": 1.3250000000000002e-06,
+      "loss": 0.071,
+      "num_input_tokens_seen": 2610720,
+      "step": 159
+    },
+    {
+      "epoch": 2.057877813504823,
+      "grad_norm": 3.897312641143799,
+      "learning_rate": 1.3333333333333334e-06,
+      "loss": 0.0791,
+      "num_input_tokens_seen": 2626400,
+      "step": 160
+    },
+    {
+      "epoch": 2.0707395498392285,
+      "grad_norm": 2.8594729900360107,
+      "learning_rate": 1.3416666666666666e-06,
+      "loss": 0.0769,
+      "num_input_tokens_seen": 2642496,
+      "step": 161
+    },
+    {
+      "epoch": 2.0836012861736335,
+      "grad_norm": 8.970246315002441,
+      "learning_rate": 1.3500000000000002e-06,
+      "loss": 0.0916,
+      "num_input_tokens_seen": 2658144,
+      "step": 162
+    },
+    {
+      "epoch": 2.0964630225080385,
+      "grad_norm": 2.965878963470459,
+      "learning_rate": 1.3583333333333334e-06,
+      "loss": 0.0584,
+      "num_input_tokens_seen": 2674560,
+      "step": 163
+    },
+    {
+      "epoch": 2.1093247588424435,
+      "grad_norm": 6.79748010635376,
+      "learning_rate": 1.3666666666666668e-06,
+      "loss": 0.0811,
+      "num_input_tokens_seen": 2690688,
+      "step": 164
+    },
+    {
+      "epoch": 2.122186495176849,
+      "grad_norm": 10.366084098815918,
+      "learning_rate": 1.3750000000000002e-06,
+      "loss": 0.0663,
+      "num_input_tokens_seen": 2706720,
+      "step": 165
+    },
+    {
+      "epoch": 2.135048231511254,
+      "grad_norm": 8.038703918457031,
+      "learning_rate": 1.3833333333333336e-06,
+      "loss": 0.0802,
+      "num_input_tokens_seen": 2723008,
+      "step": 166
+    },
+    {
+      "epoch": 2.147909967845659,
+      "grad_norm": 6.01867151260376,
+      "learning_rate": 1.3916666666666668e-06,
+      "loss": 0.0587,
+      "num_input_tokens_seen": 2738784,
+      "step": 167
+    },
+    {
+      "epoch": 2.1607717041800645,
+      "grad_norm": 5.053437232971191,
+      "learning_rate": 1.4000000000000001e-06,
+      "loss": 0.0953,
+      "num_input_tokens_seen": 2755072,
+      "step": 168
+    },
+    {
+      "epoch": 2.1736334405144695,
+      "grad_norm": 7.119932174682617,
+      "learning_rate": 1.4083333333333335e-06,
+      "loss": 0.0795,
+      "num_input_tokens_seen": 2771296,
+      "step": 169
+    },
+    {
+      "epoch": 2.1864951768488745,
+      "grad_norm": 4.602327346801758,
+      "learning_rate": 1.4166666666666667e-06,
+      "loss": 0.1015,
+      "num_input_tokens_seen": 2787712,
+      "step": 170
+    },
+    {
+      "epoch": 2.19935691318328,
+      "grad_norm": 2.7282161712646484,
+      "learning_rate": 1.425e-06,
+      "loss": 0.0614,
+      "num_input_tokens_seen": 2803552,
+      "step": 171
+    },
+    {
+      "epoch": 2.212218649517685,
+      "grad_norm": 4.694391250610352,
+      "learning_rate": 1.4333333333333335e-06,
+      "loss": 0.0753,
+      "num_input_tokens_seen": 2820352,
+      "step": 172
+    },
+    {
+      "epoch": 2.22508038585209,
+      "grad_norm": 3.231022357940674,
+      "learning_rate": 1.4416666666666667e-06,
+      "loss": 0.08,
+      "num_input_tokens_seen": 2837696,
+      "step": 173
+    },
+    {
+      "epoch": 2.237942122186495,
+      "grad_norm": 4.522435188293457,
+      "learning_rate": 1.45e-06,
+      "loss": 0.0603,
+      "num_input_tokens_seen": 2854272,
+      "step": 174
+    },
+    {
+      "epoch": 2.2508038585209005,
+      "grad_norm": 8.292889595031738,
+      "learning_rate": 1.4583333333333335e-06,
+      "loss": 0.0874,
+      "num_input_tokens_seen": 2870464,
+      "step": 175
+    },
+    {
+      "epoch": 2.2636655948553055,
+      "grad_norm": 3.635068655014038,
+      "learning_rate": 1.4666666666666669e-06,
+      "loss": 0.0833,
+      "num_input_tokens_seen": 2886880,
+      "step": 176
+    },
+    {
+      "epoch": 2.2765273311897105,
+      "grad_norm": 3.8000781536102295,
+      "learning_rate": 1.475e-06,
+      "loss": 0.0676,
+      "num_input_tokens_seen": 2902880,
+      "step": 177
+    },
+    {
+      "epoch": 2.289389067524116,
+      "grad_norm": 9.920318603515625,
+      "learning_rate": 1.4833333333333337e-06,
+      "loss": 0.086,
+      "num_input_tokens_seen": 2919584,
+      "step": 178
+    },
+    {
+      "epoch": 2.302250803858521,
+      "grad_norm": 3.8999791145324707,
+      "learning_rate": 1.4916666666666669e-06,
+      "loss": 0.0662,
+      "num_input_tokens_seen": 2935136,
+      "step": 179
+    },
+    {
+      "epoch": 2.315112540192926,
+      "grad_norm": 6.011571407318115,
+      "learning_rate": 1.5e-06,
+      "loss": 0.1028,
+      "num_input_tokens_seen": 2951680,
+      "step": 180
+    },
+    {
+      "epoch": 2.327974276527331,
+      "grad_norm": 5.94368839263916,
+      "learning_rate": 1.5083333333333336e-06,
+      "loss": 0.0882,
+      "num_input_tokens_seen": 2967712,
+      "step": 181
+    },
+    {
+      "epoch": 2.3408360128617365,
+      "grad_norm": 3.9432876110076904,
+      "learning_rate": 1.5166666666666668e-06,
+      "loss": 0.0806,
+      "num_input_tokens_seen": 2984224,
+      "step": 182
+    },
+    {
+      "epoch": 2.3536977491961415,
+      "grad_norm": 18.06218147277832,
+      "learning_rate": 1.525e-06,
+      "loss": 0.1108,
+      "num_input_tokens_seen": 2999904,
+      "step": 183
+    },
+    {
+      "epoch": 2.3665594855305465,
+      "grad_norm": 5.799235820770264,
+      "learning_rate": 1.5333333333333334e-06,
+      "loss": 0.0687,
+      "num_input_tokens_seen": 3017376,
+      "step": 184
+    },
+    {
+      "epoch": 2.379421221864952,
+      "grad_norm": 4.252455711364746,
+      "learning_rate": 1.5416666666666668e-06,
+      "loss": 0.0736,
+      "num_input_tokens_seen": 3033728,
+      "step": 185
+    },
+    {
+      "epoch": 2.392282958199357,
+      "grad_norm": 6.050833702087402,
+      "learning_rate": 1.5500000000000002e-06,
+      "loss": 0.0779,
+      "num_input_tokens_seen": 3050176,
+      "step": 186
+    },
+    {
+      "epoch": 2.405144694533762,
+      "grad_norm": 8.076692581176758,
+      "learning_rate": 1.5583333333333334e-06,
+      "loss": 0.0709,
+      "num_input_tokens_seen": 3066400,
+      "step": 187
+    },
+    {
+      "epoch": 2.418006430868167,
+      "grad_norm": 7.372990608215332,
+      "learning_rate": 1.566666666666667e-06,
+      "loss": 0.0652,
+      "num_input_tokens_seen": 3083104,
+      "step": 188
+    },
+    {
+      "epoch": 2.4308681672025725,
+      "grad_norm": 14.172348022460938,
+      "learning_rate": 1.5750000000000002e-06,
+      "loss": 0.1095,
+      "num_input_tokens_seen": 3100000,
+      "step": 189
+    },
+    {
+      "epoch": 2.4437299035369775,
+      "grad_norm": 4.726587772369385,
+      "learning_rate": 1.5833333333333333e-06,
+      "loss": 0.0618,
+      "num_input_tokens_seen": 3116800,
+      "step": 190
+    },
+    {
+      "epoch": 2.4565916398713825,
+      "grad_norm": 10.408024787902832,
+      "learning_rate": 1.591666666666667e-06,
+      "loss": 0.0666,
+      "num_input_tokens_seen": 3133568,
+      "step": 191
+    },
+    {
+      "epoch": 2.469453376205788,
+      "grad_norm": 7.295907497406006,
+      "learning_rate": 1.6000000000000001e-06,
+      "loss": 0.0573,
+      "num_input_tokens_seen": 3150048,
+      "step": 192
+    },
+    {
+      "epoch": 2.482315112540193,
+      "grad_norm": 6.490230083465576,
+      "learning_rate": 1.6083333333333333e-06,
+      "loss": 0.0577,
+      "num_input_tokens_seen": 3166688,
+      "step": 193
+    },
+    {
+      "epoch": 2.495176848874598,
+      "grad_norm": 6.940796375274658,
+      "learning_rate": 1.6166666666666667e-06,
+      "loss": 0.0813,
+      "num_input_tokens_seen": 3182752,
+      "step": 194
+    },
+    {
+      "epoch": 2.508038585209003,
+      "grad_norm": 8.307454109191895,
+      "learning_rate": 1.6250000000000001e-06,
+      "loss": 0.066,
+      "num_input_tokens_seen": 3198944,
+      "step": 195
+    },
+    {
+      "epoch": 2.5209003215434085,
+      "grad_norm": 4.472862243652344,
+      "learning_rate": 1.6333333333333335e-06,
+      "loss": 0.0622,
+      "num_input_tokens_seen": 3215552,
+      "step": 196
+    },
+    {
+      "epoch": 2.5337620578778135,
+      "grad_norm": 3.5001590251922607,
+      "learning_rate": 1.6416666666666667e-06,
+      "loss": 0.0616,
+      "num_input_tokens_seen": 3231488,
+      "step": 197
+    },
+    {
+      "epoch": 2.5466237942122185,
+      "grad_norm": 13.425641059875488,
+      "learning_rate": 1.6500000000000003e-06,
+      "loss": 0.0993,
+      "num_input_tokens_seen": 3247840,
+      "step": 198
+    },
+    {
+      "epoch": 2.559485530546624,
+      "grad_norm": 12.016642570495605,
+      "learning_rate": 1.6583333333333335e-06,
+      "loss": 0.0702,
+      "num_input_tokens_seen": 3264128,
+      "step": 199
+    },
+    {
+      "epoch": 2.572347266881029,
+      "grad_norm": 10.653433799743652,
+      "learning_rate": 1.6666666666666667e-06,
+      "loss": 0.0743,
+      "num_input_tokens_seen": 3280192,
+      "step": 200
+    },
+    {
+      "epoch": 2.585209003215434,
+      "grad_norm": 3.59945011138916,
+      "learning_rate": 1.6750000000000003e-06,
+      "loss": 0.0647,
+      "num_input_tokens_seen": 3296480,
+      "step": 201
+    },
+    {
+      "epoch": 2.598070739549839,
+      "grad_norm": 11.130800247192383,
+      "learning_rate": 1.6833333333333335e-06,
+      "loss": 0.0814,
+      "num_input_tokens_seen": 3313120,
+      "step": 202
+    },
+    {
+      "epoch": 2.6109324758842445,
+      "grad_norm": 11.733818054199219,
+      "learning_rate": 1.6916666666666666e-06,
+      "loss": 0.0861,
+      "num_input_tokens_seen": 3329184,
+      "step": 203
+    },
+    {
+      "epoch": 2.6237942122186495,
+      "grad_norm": 6.696333885192871,
+      "learning_rate": 1.7000000000000002e-06,
+      "loss": 0.0769,
+      "num_input_tokens_seen": 3346208,
+      "step": 204
+    },
+    {
+      "epoch": 2.6366559485530545,
+      "grad_norm": 8.523835182189941,
+      "learning_rate": 1.7083333333333334e-06,
+      "loss": 0.0888,
+      "num_input_tokens_seen": 3363008,
+      "step": 205
+    },
+    {
+      "epoch": 2.64951768488746,
+      "grad_norm": 12.503429412841797,
+      "learning_rate": 1.7166666666666668e-06,
+      "loss": 0.1017,
+      "num_input_tokens_seen": 3379488,
+      "step": 206
+    },
+    {
+      "epoch": 2.662379421221865,
+      "grad_norm": 7.644631385803223,
+      "learning_rate": 1.725e-06,
+      "loss": 0.0677,
+      "num_input_tokens_seen": 3395648,
+      "step": 207
+    },
+    {
+      "epoch": 2.67524115755627,
+      "grad_norm": 7.251969814300537,
+      "learning_rate": 1.7333333333333336e-06,
+      "loss": 0.0861,
+      "num_input_tokens_seen": 3412736,
+      "step": 208
+    },
+    {
+      "epoch": 2.688102893890675,
+      "grad_norm": 3.763118267059326,
+      "learning_rate": 1.7416666666666668e-06,
+      "loss": 0.0562,
+      "num_input_tokens_seen": 3429472,
+      "step": 209
+    },
+    {
+      "epoch": 2.7009646302250805,
+      "grad_norm": 8.583102226257324,
+      "learning_rate": 1.75e-06,
+      "loss": 0.0641,
+      "num_input_tokens_seen": 3445632,
+      "step": 210
+    },
+    {
+      "epoch": 2.7138263665594855,
+      "grad_norm": 9.621906280517578,
+      "learning_rate": 1.7583333333333336e-06,
+      "loss": 0.0829,
+      "num_input_tokens_seen": 3462720,
+      "step": 211
+    },
+    {
+      "epoch": 2.7266881028938905,
+      "grad_norm": 6.067561626434326,
+      "learning_rate": 1.7666666666666668e-06,
+      "loss": 0.0632,
+      "num_input_tokens_seen": 3478720,
+      "step": 212
+    },
+    {
+      "epoch": 2.739549839228296,
+      "grad_norm": 4.912470817565918,
+      "learning_rate": 1.7750000000000002e-06,
+      "loss": 0.062,
+      "num_input_tokens_seen": 3494304,
+      "step": 213
+    },
+    {
+      "epoch": 2.752411575562701,
+      "grad_norm": 8.044519424438477,
+      "learning_rate": 1.7833333333333336e-06,
+      "loss": 0.0816,
+      "num_input_tokens_seen": 3510176,
+      "step": 214
+    },
+    {
+      "epoch": 2.765273311897106,
+      "grad_norm": 8.589887619018555,
+      "learning_rate": 1.7916666666666667e-06,
+      "loss": 0.096,
+      "num_input_tokens_seen": 3526848,
+      "step": 215
+    },
+    {
+      "epoch": 2.778135048231511,
+      "grad_norm": 3.3357672691345215,
+      "learning_rate": 1.8000000000000001e-06,
+      "loss": 0.0639,
+      "num_input_tokens_seen": 3543136,
+      "step": 216
+    },
+    {
+      "epoch": 2.7909967845659165,
+      "grad_norm": 5.992812156677246,
+      "learning_rate": 1.8083333333333335e-06,
+      "loss": 0.0915,
+      "num_input_tokens_seen": 3558240,
+      "step": 217
+    },
+    {
+      "epoch": 2.8038585209003215,
+      "grad_norm": 4.8690924644470215,
+      "learning_rate": 1.816666666666667e-06,
+      "loss": 0.061,
+      "num_input_tokens_seen": 3574240,
+      "step": 218
+    },
+    {
+      "epoch": 2.816720257234727,
+      "grad_norm": 2.928898811340332,
+      "learning_rate": 1.825e-06,
+      "loss": 0.0572,
+      "num_input_tokens_seen": 3590240,
+      "step": 219
+    },
+    {
+      "epoch": 2.829581993569132,
+      "grad_norm": 2.8405849933624268,
+      "learning_rate": 1.8333333333333333e-06,
+      "loss": 0.0497,
+      "num_input_tokens_seen": 3606912,
+      "step": 220
+    },
+    {
+      "epoch": 2.842443729903537,
+      "grad_norm": 5.940427303314209,
+      "learning_rate": 1.8416666666666669e-06,
+      "loss": 0.0672,
+      "num_input_tokens_seen": 3623008,
+      "step": 221
+    },
+    {
+      "epoch": 2.855305466237942,
+      "grad_norm": 4.370597839355469,
+      "learning_rate": 1.85e-06,
+      "loss": 0.0563,
+      "num_input_tokens_seen": 3639456,
+      "step": 222
+    },
+    {
+      "epoch": 2.868167202572347,
+      "grad_norm": 3.9959638118743896,
+      "learning_rate": 1.8583333333333335e-06,
+      "loss": 0.069,
+      "num_input_tokens_seen": 3656288,
+      "step": 223
+    },
+    {
+      "epoch": 2.8810289389067525,
+      "grad_norm": 4.756357192993164,
+      "learning_rate": 1.8666666666666669e-06,
+      "loss": 0.0824,
+      "num_input_tokens_seen": 3672896,
+      "step": 224
+    },
+    {
+      "epoch": 2.8938906752411575,
+      "grad_norm": 3.312553644180298,
+      "learning_rate": 1.8750000000000003e-06,
+      "loss": 0.057,
+      "num_input_tokens_seen": 3689632,
+      "step": 225
+    },
+    {
+      "epoch": 2.906752411575563,
+      "grad_norm": 3.71880841255188,
+      "learning_rate": 1.8833333333333334e-06,
+      "loss": 0.0549,
+      "num_input_tokens_seen": 3706944,
+      "step": 226
+    },
+    {
+      "epoch": 2.919614147909968,
+      "grad_norm": 4.111685276031494,
+      "learning_rate": 1.8916666666666668e-06,
+      "loss": 0.0652,
+      "num_input_tokens_seen": 3723072,
+      "step": 227
+    },
+    {
+      "epoch": 2.932475884244373,
+      "grad_norm": 4.362407684326172,
+      "learning_rate": 1.9000000000000002e-06,
+      "loss": 0.0743,
+      "num_input_tokens_seen": 3740064,
+      "step": 228
+    },
+    {
+      "epoch": 2.945337620578778,
+      "grad_norm": 2.441969156265259,
+      "learning_rate": 1.9083333333333334e-06,
+      "loss": 0.0416,
+      "num_input_tokens_seen": 3756864,
+      "step": 229
+    },
+    {
+      "epoch": 2.958199356913183,
+      "grad_norm": 3.4988579750061035,
+      "learning_rate": 1.916666666666667e-06,
+      "loss": 0.0668,
+      "num_input_tokens_seen": 3774336,
+      "step": 230
+    },
+    {
+      "epoch": 2.9710610932475885,
+      "grad_norm": 3.7807693481445312,
+      "learning_rate": 1.925e-06,
+      "loss": 0.0603,
+      "num_input_tokens_seen": 3790336,
+      "step": 231
+    },
+    {
+      "epoch": 2.9839228295819935,
+      "grad_norm": 3.471942901611328,
+      "learning_rate": 1.9333333333333336e-06,
+      "loss": 0.066,
+      "num_input_tokens_seen": 3807520,
+      "step": 232
+    },
+    {
+      "epoch": 2.996784565916399,
+      "grad_norm": 3.4720284938812256,
+      "learning_rate": 1.9416666666666666e-06,
+      "loss": 0.0692,
+      "num_input_tokens_seen": 3823872,
+      "step": 233
+    },
+    {
+      "epoch": 3.009646302250804,
+      "grad_norm": 1.9317958354949951,
+      "learning_rate": 1.9500000000000004e-06,
+      "loss": 0.0323,
+      "num_input_tokens_seen": 3840320,
+      "step": 234
+    },
+    {
+      "epoch": 3.022508038585209,
+      "grad_norm": 4.849405288696289,
+      "learning_rate": 1.9583333333333334e-06,
+      "loss": 0.0403,
+      "num_input_tokens_seen": 3856256,
+      "step": 235
+    },
+    {
+      "epoch": 3.035369774919614,
+      "grad_norm": 2.371988296508789,
+      "learning_rate": 1.9666666666666668e-06,
+      "loss": 0.0477,
+      "num_input_tokens_seen": 3872512,
+      "step": 236
+    },
+    {
+      "epoch": 3.0482315112540195,
+      "grad_norm": 3.362964153289795,
+      "learning_rate": 1.975e-06,
+      "loss": 0.0356,
+      "num_input_tokens_seen": 3888256,
+      "step": 237
+    },
+    {
+      "epoch": 3.0610932475884245,
+      "grad_norm": 2.618176221847534,
+      "learning_rate": 1.9833333333333335e-06,
+      "loss": 0.0324,
+      "num_input_tokens_seen": 3905024,
+      "step": 238
+    },
+    {
+      "epoch": 3.0739549839228295,
+      "grad_norm": 2.5595703125,
+      "learning_rate": 1.991666666666667e-06,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 3922016,
+      "step": 239
+    },
+    {
+      "epoch": 3.0868167202572345,
+      "grad_norm": 2.788818597793579,
+      "learning_rate": 2.0000000000000003e-06,
+      "loss": 0.0315,
+      "num_input_tokens_seen": 3938400,
+      "step": 240
+    },
+    {
+      "epoch": 3.09967845659164,
+      "grad_norm": 3.9531607627868652,
+      "learning_rate": 2.0083333333333337e-06,
+      "loss": 0.0366,
+      "num_input_tokens_seen": 3954912,
+      "step": 241
+    },
+    {
+      "epoch": 3.112540192926045,
+      "grad_norm": 3.2901012897491455,
+      "learning_rate": 2.0166666666666667e-06,
+      "loss": 0.0307,
+      "num_input_tokens_seen": 3970400,
+      "step": 242
+    },
+    {
+      "epoch": 3.12540192926045,
+      "grad_norm": 4.283529281616211,
+      "learning_rate": 2.025e-06,
+      "loss": 0.0338,
+      "num_input_tokens_seen": 3986944,
+      "step": 243
+    },
+    {
+      "epoch": 3.1382636655948555,
+      "grad_norm": 4.467242240905762,
+      "learning_rate": 2.0333333333333335e-06,
+      "loss": 0.0348,
+      "num_input_tokens_seen": 4003648,
+      "step": 244
+    },
+    {
+      "epoch": 3.1511254019292605,
+      "grad_norm": 3.3191885948181152,
+      "learning_rate": 2.041666666666667e-06,
+      "loss": 0.0355,
+      "num_input_tokens_seen": 4020384,
+      "step": 245
+    },
+    {
+      "epoch": 3.1639871382636655,
+      "grad_norm": 4.061459064483643,
+      "learning_rate": 2.05e-06,
+      "loss": 0.0272,
+      "num_input_tokens_seen": 4037088,
+      "step": 246
+    },
+    {
+      "epoch": 3.176848874598071,
+      "grad_norm": 3.0776522159576416,
+      "learning_rate": 2.0583333333333337e-06,
+      "loss": 0.0464,
+      "num_input_tokens_seen": 4053472,
+      "step": 247
+    },
+    {
+      "epoch": 3.189710610932476,
+      "grad_norm": 2.13104510307312,
+      "learning_rate": 2.0666666666666666e-06,
+      "loss": 0.0268,
+      "num_input_tokens_seen": 4069248,
+      "step": 248
+    },
+    {
+      "epoch": 3.202572347266881,
+      "grad_norm": 2.1540310382843018,
+      "learning_rate": 2.075e-06,
+      "loss": 0.0229,
+      "num_input_tokens_seen": 4085024,
+      "step": 249
+    },
+    {
+      "epoch": 3.215434083601286,
+      "grad_norm": 3.068967342376709,
+      "learning_rate": 2.0833333333333334e-06,
+      "loss": 0.0373,
+      "num_input_tokens_seen": 4102432,
+      "step": 250
+    },
+    {
+      "epoch": 3.2282958199356915,
+      "grad_norm": 2.7403666973114014,
+      "learning_rate": 2.091666666666667e-06,
+      "loss": 0.0361,
+      "num_input_tokens_seen": 4118368,
+      "step": 251
+    },
+    {
+      "epoch": 3.2411575562700965,
+      "grad_norm": 4.994419574737549,
+      "learning_rate": 2.1000000000000002e-06,
+      "loss": 0.0401,
+      "num_input_tokens_seen": 4134784,
+      "step": 252
+    },
+    {
+      "epoch": 3.2540192926045015,
+      "grad_norm": 2.287346839904785,
+      "learning_rate": 2.1083333333333336e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4150528,
+      "step": 253
+    },
+    {
+      "epoch": 3.266881028938907,
+      "grad_norm": 3.649418354034424,
+      "learning_rate": 2.116666666666667e-06,
+      "loss": 0.0481,
+      "num_input_tokens_seen": 4166816,
+      "step": 254
+    },
+    {
+      "epoch": 3.279742765273312,
+      "grad_norm": 2.9041836261749268,
+      "learning_rate": 2.125e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 4182880,
+      "step": 255
+    },
+    {
+      "epoch": 3.292604501607717,
+      "grad_norm": 3.602240800857544,
+      "learning_rate": 2.133333333333334e-06,
+      "loss": 0.0457,
+      "num_input_tokens_seen": 4200128,
+      "step": 256
+    },
+    {
+      "epoch": 3.305466237942122,
+      "grad_norm": 1.6382755041122437,
+      "learning_rate": 2.1416666666666668e-06,
+      "loss": 0.0175,
+      "num_input_tokens_seen": 4215680,
+      "step": 257
+    },
+    {
+      "epoch": 3.3183279742765275,
+      "grad_norm": 2.4267561435699463,
+      "learning_rate": 2.15e-06,
+      "loss": 0.0371,
+      "num_input_tokens_seen": 4232736,
+      "step": 258
+    },
+    {
+      "epoch": 3.3311897106109325,
+      "grad_norm": 2.553440570831299,
+      "learning_rate": 2.1583333333333336e-06,
+      "loss": 0.0268,
+      "num_input_tokens_seen": 4249664,
+      "step": 259
+    },
+    {
+      "epoch": 3.3440514469453375,
+      "grad_norm": 2.951179027557373,
+      "learning_rate": 2.166666666666667e-06,
+      "loss": 0.0417,
+      "num_input_tokens_seen": 4266016,
+      "step": 260
+    },
+    {
+      "epoch": 3.356913183279743,
+      "grad_norm": 2.710747003555298,
+      "learning_rate": 2.1750000000000004e-06,
+      "loss": 0.0264,
+      "num_input_tokens_seen": 4281664,
+      "step": 261
+    },
+    {
+      "epoch": 3.369774919614148,
+      "grad_norm": 3.465902090072632,
+      "learning_rate": 2.1833333333333333e-06,
+      "loss": 0.0336,
+      "num_input_tokens_seen": 4298368,
+      "step": 262
+    },
+    {
+      "epoch": 3.382636655948553,
+      "grad_norm": 4.009627819061279,
+      "learning_rate": 2.191666666666667e-06,
+      "loss": 0.0434,
+      "num_input_tokens_seen": 4314336,
+      "step": 263
+    },
+    {
+      "epoch": 3.395498392282958,
+      "grad_norm": 4.071226596832275,
+      "learning_rate": 2.2e-06,
+      "loss": 0.0389,
+      "num_input_tokens_seen": 4330144,
+      "step": 264
+    },
+    {
+      "epoch": 3.4083601286173635,
+      "grad_norm": 4.341132164001465,
+      "learning_rate": 2.2083333333333335e-06,
+      "loss": 0.045,
+      "num_input_tokens_seen": 4345856,
+      "step": 265
+    },
+    {
+      "epoch": 3.4212218649517685,
+      "grad_norm": 4.663236141204834,
+      "learning_rate": 2.216666666666667e-06,
+      "loss": 0.0331,
+      "num_input_tokens_seen": 4361696,
+      "step": 266
+    },
+    {
+      "epoch": 3.4340836012861735,
+      "grad_norm": 2.17475962638855,
+      "learning_rate": 2.2250000000000003e-06,
+      "loss": 0.0228,
+      "num_input_tokens_seen": 4378304,
+      "step": 267
+    },
+    {
+      "epoch": 3.446945337620579,
+      "grad_norm": 5.099360942840576,
+      "learning_rate": 2.2333333333333333e-06,
+      "loss": 0.0307,
+      "num_input_tokens_seen": 4395232,
+      "step": 268
+    },
+    {
+      "epoch": 3.459807073954984,
+      "grad_norm": 4.4752020835876465,
+      "learning_rate": 2.2416666666666667e-06,
+      "loss": 0.0332,
+      "num_input_tokens_seen": 4411232,
+      "step": 269
+    },
+    {
+      "epoch": 3.472668810289389,
+      "grad_norm": 4.047648906707764,
+      "learning_rate": 2.25e-06,
+      "loss": 0.0662,
+      "num_input_tokens_seen": 4427520,
+      "step": 270
+    },
+    {
+      "epoch": 3.485530546623794,
+      "grad_norm": 5.564074516296387,
+      "learning_rate": 2.2583333333333335e-06,
+      "loss": 0.0431,
+      "num_input_tokens_seen": 4443904,
+      "step": 271
+    },
+    {
+      "epoch": 3.4983922829581995,
+      "grad_norm": 3.9845523834228516,
+      "learning_rate": 2.266666666666667e-06,
+      "loss": 0.0423,
+      "num_input_tokens_seen": 4460608,
+      "step": 272
+    },
+    {
+      "epoch": 3.5112540192926045,
+      "grad_norm": 3.6314871311187744,
+      "learning_rate": 2.2750000000000002e-06,
+      "loss": 0.0447,
+      "num_input_tokens_seen": 4477120,
+      "step": 273
+    },
+    {
+      "epoch": 3.5241157556270095,
+      "grad_norm": 3.1981313228607178,
+      "learning_rate": 2.2833333333333336e-06,
+      "loss": 0.0337,
+      "num_input_tokens_seen": 4493152,
+      "step": 274
+    },
+    {
+      "epoch": 3.536977491961415,
+      "grad_norm": 2.1383423805236816,
+      "learning_rate": 2.2916666666666666e-06,
+      "loss": 0.0319,
+      "num_input_tokens_seen": 4509120,
+      "step": 275
+    },
+    {
+      "epoch": 3.54983922829582,
+      "grad_norm": 2.778956890106201,
+      "learning_rate": 2.3000000000000004e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 4525984,
+      "step": 276
+    },
+    {
+      "epoch": 3.562700964630225,
+      "grad_norm": 2.518002510070801,
+      "learning_rate": 2.3083333333333334e-06,
+      "loss": 0.0244,
+      "num_input_tokens_seen": 4542816,
+      "step": 277
+    },
+    {
+      "epoch": 3.57556270096463,
+      "grad_norm": 3.8519890308380127,
+      "learning_rate": 2.316666666666667e-06,
+      "loss": 0.0377,
+      "num_input_tokens_seen": 4559872,
+      "step": 278
+    },
+    {
+      "epoch": 3.5884244372990355,
+      "grad_norm": 5.541743278503418,
+      "learning_rate": 2.325e-06,
+      "loss": 0.0477,
+      "num_input_tokens_seen": 4576288,
+      "step": 279
+    },
+    {
+      "epoch": 3.6012861736334405,
+      "grad_norm": 4.099382400512695,
+      "learning_rate": 2.3333333333333336e-06,
+      "loss": 0.0332,
+      "num_input_tokens_seen": 4591456,
+      "step": 280
+    },
+    {
+      "epoch": 3.6141479099678455,
+      "grad_norm": 2.509037733078003,
+      "learning_rate": 2.341666666666667e-06,
+      "loss": 0.0285,
+      "num_input_tokens_seen": 4608608,
+      "step": 281
+    },
+    {
+      "epoch": 3.627009646302251,
+      "grad_norm": 6.764764308929443,
+      "learning_rate": 2.35e-06,
+      "loss": 0.0468,
+      "num_input_tokens_seen": 4625120,
+      "step": 282
+    },
+    {
+      "epoch": 3.639871382636656,
+      "grad_norm": 6.219926834106445,
+      "learning_rate": 2.3583333333333338e-06,
+      "loss": 0.037,
+      "num_input_tokens_seen": 4641952,
+      "step": 283
+    },
+    {
+      "epoch": 3.652733118971061,
+      "grad_norm": 3.869555950164795,
+      "learning_rate": 2.3666666666666667e-06,
+      "loss": 0.041,
+      "num_input_tokens_seen": 4657920,
+      "step": 284
+    },
+    {
+      "epoch": 3.665594855305466,
+      "grad_norm": 4.1329264640808105,
+      "learning_rate": 2.375e-06,
+      "loss": 0.0461,
+      "num_input_tokens_seen": 4674688,
+      "step": 285
+    },
+    {
+      "epoch": 3.6784565916398715,
+      "grad_norm": 6.7732648849487305,
+      "learning_rate": 2.3833333333333335e-06,
+      "loss": 0.0594,
+      "num_input_tokens_seen": 4690592,
+      "step": 286
+    },
+    {
+      "epoch": 3.6913183279742765,
+      "grad_norm": 8.550586700439453,
+      "learning_rate": 2.391666666666667e-06,
+      "loss": 0.0728,
+      "num_input_tokens_seen": 4707360,
+      "step": 287
+    },
+    {
+      "epoch": 3.7041800643086815,
+      "grad_norm": 4.587222099304199,
+      "learning_rate": 2.4000000000000003e-06,
+      "loss": 0.029,
+      "num_input_tokens_seen": 4724192,
+      "step": 288
+    },
+    {
+      "epoch": 3.717041800643087,
+      "grad_norm": 3.3474302291870117,
+      "learning_rate": 2.4083333333333337e-06,
+      "loss": 0.0412,
+      "num_input_tokens_seen": 4741344,
+      "step": 289
+    },
+    {
+      "epoch": 3.729903536977492,
+      "grad_norm": 4.359026908874512,
+      "learning_rate": 2.4166666666666667e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 4756768,
+      "step": 290
+    },
+    {
+      "epoch": 3.742765273311897,
+      "grad_norm": 8.029643058776855,
+      "learning_rate": 2.425e-06,
+      "loss": 0.0796,
+      "num_input_tokens_seen": 4772608,
+      "step": 291
+    },
+    {
+      "epoch": 3.755627009646302,
+      "grad_norm": 3.9188246726989746,
+      "learning_rate": 2.4333333333333335e-06,
+      "loss": 0.0447,
+      "num_input_tokens_seen": 4788928,
+      "step": 292
+    },
+    {
+      "epoch": 3.7684887459807075,
+      "grad_norm": 2.2623226642608643,
+      "learning_rate": 2.441666666666667e-06,
+      "loss": 0.0252,
+      "num_input_tokens_seen": 4805408,
+      "step": 293
+    },
+    {
+      "epoch": 3.7813504823151125,
+      "grad_norm": 5.41758394241333,
+      "learning_rate": 2.4500000000000003e-06,
+      "loss": 0.0458,
+      "num_input_tokens_seen": 4821248,
+      "step": 294
+    },
+    {
+      "epoch": 3.7942122186495175,
+      "grad_norm": 3.402024984359741,
+      "learning_rate": 2.4583333333333332e-06,
+      "loss": 0.0431,
+      "num_input_tokens_seen": 4837024,
+      "step": 295
+    },
+    {
+      "epoch": 3.807073954983923,
+      "grad_norm": 2.2912893295288086,
+      "learning_rate": 2.466666666666667e-06,
+      "loss": 0.0422,
+      "num_input_tokens_seen": 4853760,
+      "step": 296
+    },
+    {
+      "epoch": 3.819935691318328,
+      "grad_norm": 2.5446107387542725,
+      "learning_rate": 2.475e-06,
+      "loss": 0.0428,
+      "num_input_tokens_seen": 4870560,
+      "step": 297
+    },
+    {
+      "epoch": 3.832797427652733,
+      "grad_norm": 4.001276969909668,
+      "learning_rate": 2.4833333333333334e-06,
+      "loss": 0.0473,
+      "num_input_tokens_seen": 4887040,
+      "step": 298
+    },
+    {
+      "epoch": 3.845659163987138,
+      "grad_norm": 2.1214165687561035,
+      "learning_rate": 2.491666666666667e-06,
+      "loss": 0.0223,
+      "num_input_tokens_seen": 4903232,
+      "step": 299
+    },
+    {
+      "epoch": 3.8585209003215435,
+      "grad_norm": 2.0410876274108887,
+      "learning_rate": 2.5e-06,
+      "loss": 0.0274,
+      "num_input_tokens_seen": 4919392,
+      "step": 300
+    },
+    {
+      "epoch": 3.8713826366559485,
+      "grad_norm": 4.287325382232666,
+      "learning_rate": 2.5083333333333336e-06,
+      "loss": 0.0454,
+      "num_input_tokens_seen": 4935648,
+      "step": 301
+    },
+    {
+      "epoch": 3.884244372990354,
+      "grad_norm": 4.437230110168457,
+      "learning_rate": 2.5166666666666666e-06,
+      "loss": 0.022,
+      "num_input_tokens_seen": 4952768,
+      "step": 302
+    },
+    {
+      "epoch": 3.897106109324759,
+      "grad_norm": 3.8610730171203613,
+      "learning_rate": 2.5250000000000004e-06,
+      "loss": 0.0357,
+      "num_input_tokens_seen": 4969408,
+      "step": 303
+    },
+    {
+      "epoch": 3.909967845659164,
+      "grad_norm": 2.636066436767578,
+      "learning_rate": 2.5333333333333338e-06,
+      "loss": 0.0458,
+      "num_input_tokens_seen": 4986592,
+      "step": 304
+    },
+    {
+      "epoch": 3.922829581993569,
+      "grad_norm": 4.873857021331787,
+      "learning_rate": 2.5416666666666668e-06,
+      "loss": 0.0454,
+      "num_input_tokens_seen": 5002720,
+      "step": 305
+    },
+    {
+      "epoch": 3.935691318327974,
+      "grad_norm": 2.460498571395874,
+      "learning_rate": 2.55e-06,
+      "loss": 0.0262,
+      "num_input_tokens_seen": 5018848,
+      "step": 306
+    },
+    {
+      "epoch": 3.9485530546623795,
+      "grad_norm": 1.9787613153457642,
+      "learning_rate": 2.558333333333334e-06,
+      "loss": 0.0234,
+      "num_input_tokens_seen": 5034528,
+      "step": 307
+    },
+    {
+      "epoch": 3.9614147909967845,
+      "grad_norm": 2.9524741172790527,
+      "learning_rate": 2.566666666666667e-06,
+      "loss": 0.0366,
+      "num_input_tokens_seen": 5051328,
+      "step": 308
+    },
+    {
+      "epoch": 3.97427652733119,
+      "grad_norm": 2.858250379562378,
+      "learning_rate": 2.5750000000000003e-06,
+      "loss": 0.0236,
+      "num_input_tokens_seen": 5067712,
+      "step": 309
+    },
+    {
+      "epoch": 3.987138263665595,
+      "grad_norm": 2.985665798187256,
+      "learning_rate": 2.5833333333333337e-06,
+      "loss": 0.0403,
+      "num_input_tokens_seen": 5085088,
+      "step": 310
+    },
+    {
+      "epoch": 4.0,
+      "grad_norm": 2.72078013420105,
+      "learning_rate": 2.5916666666666667e-06,
+      "loss": 0.0395,
+      "num_input_tokens_seen": 5101536,
+      "step": 311
+    },
+    {
+      "epoch": 4.012861736334405,
+      "grad_norm": 1.2907602787017822,
+      "learning_rate": 2.6e-06,
+      "loss": 0.0163,
+      "num_input_tokens_seen": 5117696,
+      "step": 312
+    },
+    {
+      "epoch": 4.02572347266881,
+      "grad_norm": 1.910504937171936,
+      "learning_rate": 2.608333333333333e-06,
+      "loss": 0.0181,
+      "num_input_tokens_seen": 5133408,
+      "step": 313
+    },
+    {
+      "epoch": 4.038585209003215,
+      "grad_norm": 1.1473162174224854,
+      "learning_rate": 2.616666666666667e-06,
+      "loss": 0.013,
+      "num_input_tokens_seen": 5149664,
+      "step": 314
+    },
+    {
+      "epoch": 4.051446945337621,
+      "grad_norm": 2.9885878562927246,
+      "learning_rate": 2.6250000000000003e-06,
+      "loss": 0.0178,
+      "num_input_tokens_seen": 5165344,
+      "step": 315
+    },
+    {
+      "epoch": 4.064308681672026,
+      "grad_norm": 2.308147430419922,
+      "learning_rate": 2.6333333333333332e-06,
+      "loss": 0.0153,
+      "num_input_tokens_seen": 5183264,
+      "step": 316
+    },
+    {
+      "epoch": 4.077170418006431,
+      "grad_norm": 3.1830930709838867,
+      "learning_rate": 2.6416666666666666e-06,
+      "loss": 0.0205,
+      "num_input_tokens_seen": 5199296,
+      "step": 317
+    },
+    {
+      "epoch": 4.090032154340836,
+      "grad_norm": 0.674608051776886,
+      "learning_rate": 2.6500000000000005e-06,
+      "loss": 0.0021,
+      "num_input_tokens_seen": 5216032,
+      "step": 318
+    },
+    {
+      "epoch": 4.102893890675241,
+      "grad_norm": 3.075059175491333,
+      "learning_rate": 2.6583333333333334e-06,
+      "loss": 0.022,
+      "num_input_tokens_seen": 5232544,
+      "step": 319
+    },
+    {
+      "epoch": 4.115755627009646,
+      "grad_norm": 1.1589558124542236,
+      "learning_rate": 2.666666666666667e-06,
+      "loss": 0.0134,
+      "num_input_tokens_seen": 5248704,
+      "step": 320
+    },
+    {
+      "epoch": 4.128617363344051,
+      "grad_norm": 4.190515518188477,
+      "learning_rate": 2.6750000000000002e-06,
+      "loss": 0.0175,
+      "num_input_tokens_seen": 5264384,
+      "step": 321
+    },
+    {
+      "epoch": 4.141479099678457,
+      "grad_norm": 3.435070276260376,
+      "learning_rate": 2.683333333333333e-06,
+      "loss": 0.023,
+      "num_input_tokens_seen": 5280928,
+      "step": 322
+    },
+    {
+      "epoch": 4.154340836012862,
+      "grad_norm": 3.3521199226379395,
+      "learning_rate": 2.691666666666667e-06,
+      "loss": 0.0303,
+      "num_input_tokens_seen": 5296512,
+      "step": 323
+    },
+    {
+      "epoch": 4.167202572347267,
+      "grad_norm": 2.0815391540527344,
+      "learning_rate": 2.7000000000000004e-06,
+      "loss": 0.0187,
+      "num_input_tokens_seen": 5313440,
+      "step": 324
+    },
+    {
+      "epoch": 4.180064308681672,
+      "grad_norm": 1.968308687210083,
+      "learning_rate": 2.7083333333333334e-06,
+      "loss": 0.0126,
+      "num_input_tokens_seen": 5329088,
+      "step": 325
+    },
+    {
+      "epoch": 4.192926045016077,
+      "grad_norm": 2.134216070175171,
+      "learning_rate": 2.7166666666666668e-06,
+      "loss": 0.0203,
+      "num_input_tokens_seen": 5345376,
+      "step": 326
+    },
+    {
+      "epoch": 4.205787781350482,
+      "grad_norm": 1.2245476245880127,
+      "learning_rate": 2.7250000000000006e-06,
+      "loss": 0.0078,
+      "num_input_tokens_seen": 5361568,
+      "step": 327
+    },
+    {
+      "epoch": 4.218649517684887,
+      "grad_norm": 1.908057451248169,
+      "learning_rate": 2.7333333333333336e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 5377536,
+      "step": 328
+    },
+    {
+      "epoch": 4.231511254019293,
+      "grad_norm": 1.6413862705230713,
+      "learning_rate": 2.741666666666667e-06,
+      "loss": 0.0113,
+      "num_input_tokens_seen": 5395040,
+      "step": 329
+    },
+    {
+      "epoch": 4.244372990353698,
+      "grad_norm": 1.442226529121399,
+      "learning_rate": 2.7500000000000004e-06,
+      "loss": 0.0058,
+      "num_input_tokens_seen": 5411488,
+      "step": 330
+    },
+    {
+      "epoch": 4.257234726688103,
+      "grad_norm": 0.9877079129219055,
+      "learning_rate": 2.7583333333333333e-06,
+      "loss": 0.007,
+      "num_input_tokens_seen": 5427328,
+      "step": 331
+    },
+    {
+      "epoch": 4.270096463022508,
+      "grad_norm": 2.9424357414245605,
+      "learning_rate": 2.766666666666667e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 5444352,
+      "step": 332
+    },
+    {
+      "epoch": 4.282958199356913,
+      "grad_norm": 3.5407063961029053,
+      "learning_rate": 2.7750000000000005e-06,
+      "loss": 0.0276,
+      "num_input_tokens_seen": 5459840,
+      "step": 333
+    },
+    {
+      "epoch": 4.295819935691318,
+      "grad_norm": 4.147865295410156,
+      "learning_rate": 2.7833333333333335e-06,
+      "loss": 0.0367,
+      "num_input_tokens_seen": 5476000,
+      "step": 334
+    },
+    {
+      "epoch": 4.308681672025724,
+      "grad_norm": 2.4164767265319824,
+      "learning_rate": 2.791666666666667e-06,
+      "loss": 0.0161,
+      "num_input_tokens_seen": 5491968,
+      "step": 335
+    },
+    {
+      "epoch": 4.321543408360129,
+      "grad_norm": 2.553860902786255,
+      "learning_rate": 2.8000000000000003e-06,
+      "loss": 0.018,
+      "num_input_tokens_seen": 5508864,
+      "step": 336
+    },
+    {
+      "epoch": 4.334405144694534,
+      "grad_norm": 1.050309419631958,
+      "learning_rate": 2.8083333333333333e-06,
+      "loss": 0.0044,
+      "num_input_tokens_seen": 5525536,
+      "step": 337
+    },
+    {
+      "epoch": 4.347266881028939,
+      "grad_norm": 2.375540018081665,
+      "learning_rate": 2.816666666666667e-06,
+      "loss": 0.0109,
+      "num_input_tokens_seen": 5542368,
+      "step": 338
+    },
+    {
+      "epoch": 4.360128617363344,
+      "grad_norm": 3.041912794113159,
+      "learning_rate": 2.825e-06,
+      "loss": 0.0173,
+      "num_input_tokens_seen": 5558400,
+      "step": 339
+    },
+    {
+      "epoch": 4.372990353697749,
+      "grad_norm": 2.464390754699707,
+      "learning_rate": 2.8333333333333335e-06,
+      "loss": 0.0107,
+      "num_input_tokens_seen": 5575008,
+      "step": 340
+    },
+    {
+      "epoch": 4.385852090032154,
+      "grad_norm": 1.3086371421813965,
+      "learning_rate": 2.841666666666667e-06,
+      "loss": 0.0116,
+      "num_input_tokens_seen": 5590784,
+      "step": 341
+    },
+    {
+      "epoch": 4.39871382636656,
+      "grad_norm": 3.057859420776367,
+      "learning_rate": 2.85e-06,
+      "loss": 0.0281,
+      "num_input_tokens_seen": 5608032,
+      "step": 342
+    },
+    {
+      "epoch": 4.411575562700965,
+      "grad_norm": 2.7371633052825928,
+      "learning_rate": 2.8583333333333336e-06,
+      "loss": 0.0246,
+      "num_input_tokens_seen": 5624704,
+      "step": 343
+    },
+    {
+      "epoch": 4.42443729903537,
+      "grad_norm": 1.7932931184768677,
+      "learning_rate": 2.866666666666667e-06,
+      "loss": 0.0146,
+      "num_input_tokens_seen": 5641056,
+      "step": 344
+    },
+    {
+      "epoch": 4.437299035369775,
+      "grad_norm": 3.9446775913238525,
+      "learning_rate": 2.875e-06,
+      "loss": 0.0439,
+      "num_input_tokens_seen": 5656608,
+      "step": 345
+    },
+    {
+      "epoch": 4.45016077170418,
+      "grad_norm": 3.7706024646759033,
+      "learning_rate": 2.8833333333333334e-06,
+      "loss": 0.0279,
+      "num_input_tokens_seen": 5672928,
+      "step": 346
+    },
+    {
+      "epoch": 4.463022508038585,
+      "grad_norm": 2.38531756401062,
+      "learning_rate": 2.8916666666666672e-06,
+      "loss": 0.0276,
+      "num_input_tokens_seen": 5689728,
+      "step": 347
+    },
+    {
+      "epoch": 4.47588424437299,
+      "grad_norm": 2.534675121307373,
+      "learning_rate": 2.9e-06,
+      "loss": 0.0167,
+      "num_input_tokens_seen": 5705824,
+      "step": 348
+    },
+    {
+      "epoch": 4.488745980707396,
+      "grad_norm": 3.28116774559021,
+      "learning_rate": 2.9083333333333336e-06,
+      "loss": 0.0258,
+      "num_input_tokens_seen": 5723008,
+      "step": 349
+    },
+    {
+      "epoch": 4.501607717041801,
+      "grad_norm": 3.5873191356658936,
+      "learning_rate": 2.916666666666667e-06,
+      "loss": 0.0306,
+      "num_input_tokens_seen": 5739680,
+      "step": 350
+    },
+    {
+      "epoch": 4.514469453376206,
+      "grad_norm": 4.889535427093506,
+      "learning_rate": 2.925e-06,
+      "loss": 0.0395,
+      "num_input_tokens_seen": 5756480,
+      "step": 351
+    },
+    {
+      "epoch": 4.527331189710611,
+      "grad_norm": 1.8129762411117554,
+      "learning_rate": 2.9333333333333338e-06,
+      "loss": 0.0203,
+      "num_input_tokens_seen": 5773088,
+      "step": 352
+    },
+    {
+      "epoch": 4.540192926045016,
+      "grad_norm": 1.5045830011367798,
+      "learning_rate": 2.941666666666667e-06,
+      "loss": 0.0241,
+      "num_input_tokens_seen": 5789312,
+      "step": 353
+    },
+    {
+      "epoch": 4.553054662379421,
+      "grad_norm": 1.4582304954528809,
+      "learning_rate": 2.95e-06,
+      "loss": 0.0151,
+      "num_input_tokens_seen": 5804352,
+      "step": 354
+    },
+    {
+      "epoch": 4.565916398713826,
+      "grad_norm": 1.6875511407852173,
+      "learning_rate": 2.9583333333333335e-06,
+      "loss": 0.0355,
+      "num_input_tokens_seen": 5820384,
+      "step": 355
+    },
+    {
+      "epoch": 4.578778135048232,
+      "grad_norm": 1.0483993291854858,
+      "learning_rate": 2.9666666666666673e-06,
+      "loss": 0.0093,
+      "num_input_tokens_seen": 5837280,
+      "step": 356
+    },
+    {
+      "epoch": 4.591639871382637,
+      "grad_norm": 2.731086254119873,
+      "learning_rate": 2.9750000000000003e-06,
+      "loss": 0.0314,
+      "num_input_tokens_seen": 5853760,
+      "step": 357
+    },
+    {
+      "epoch": 4.604501607717042,
+      "grad_norm": 1.5985758304595947,
+      "learning_rate": 2.9833333333333337e-06,
+      "loss": 0.0155,
+      "num_input_tokens_seen": 5870240,
+      "step": 358
+    },
+    {
+      "epoch": 4.617363344051447,
+      "grad_norm": 1.3119585514068604,
+      "learning_rate": 2.991666666666667e-06,
+      "loss": 0.0138,
+      "num_input_tokens_seen": 5887296,
+      "step": 359
+    },
+    {
+      "epoch": 4.630225080385852,
+      "grad_norm": 2.3107335567474365,
+      "learning_rate": 3e-06,
+      "loss": 0.0212,
+      "num_input_tokens_seen": 5904384,
+      "step": 360
+    },
+    {
+      "epoch": 4.643086816720257,
+      "grad_norm": 1.8473104238510132,
+      "learning_rate": 3.0083333333333335e-06,
+      "loss": 0.0255,
+      "num_input_tokens_seen": 5920832,
+      "step": 361
+    },
+    {
+      "epoch": 4.655948553054662,
+      "grad_norm": 1.6255360841751099,
+      "learning_rate": 3.0166666666666673e-06,
+      "loss": 0.0231,
+      "num_input_tokens_seen": 5936736,
+      "step": 362
+    },
+    {
+      "epoch": 4.668810289389068,
+      "grad_norm": 1.843116044998169,
+      "learning_rate": 3.0250000000000003e-06,
+      "loss": 0.0083,
+      "num_input_tokens_seen": 5953120,
+      "step": 363
+    },
+    {
+      "epoch": 4.681672025723473,
+      "grad_norm": 1.7974934577941895,
+      "learning_rate": 3.0333333333333337e-06,
+      "loss": 0.0122,
+      "num_input_tokens_seen": 5970336,
+      "step": 364
+    },
+    {
+      "epoch": 4.694533762057878,
+      "grad_norm": 1.799026370048523,
+      "learning_rate": 3.0416666666666666e-06,
+      "loss": 0.0142,
+      "num_input_tokens_seen": 5987488,
+      "step": 365
+    },
+    {
+      "epoch": 4.707395498392283,
+      "grad_norm": 2.38661789894104,
+      "learning_rate": 3.05e-06,
+      "loss": 0.0293,
+      "num_input_tokens_seen": 6003296,
+      "step": 366
+    },
+    {
+      "epoch": 4.720257234726688,
+      "grad_norm": 2.2720234394073486,
+      "learning_rate": 3.058333333333334e-06,
+      "loss": 0.0292,
+      "num_input_tokens_seen": 6020768,
+      "step": 367
+    },
+    {
+      "epoch": 4.733118971061093,
+      "grad_norm": 3.6864068508148193,
+      "learning_rate": 3.066666666666667e-06,
+      "loss": 0.032,
+      "num_input_tokens_seen": 6036992,
+      "step": 368
+    },
+    {
+      "epoch": 4.745980707395498,
+      "grad_norm": 2.1829123497009277,
+      "learning_rate": 3.075e-06,
+      "loss": 0.0189,
+      "num_input_tokens_seen": 6053568,
+      "step": 369
+    },
+    {
+      "epoch": 4.758842443729904,
+      "grad_norm": 2.033625602722168,
+      "learning_rate": 3.0833333333333336e-06,
+      "loss": 0.022,
+      "num_input_tokens_seen": 6070336,
+      "step": 370
+    },
+    {
+      "epoch": 4.771704180064309,
+      "grad_norm": 2.665611982345581,
+      "learning_rate": 3.0916666666666666e-06,
+      "loss": 0.0242,
+      "num_input_tokens_seen": 6087200,
+      "step": 371
+    },
+    {
+      "epoch": 4.784565916398714,
+      "grad_norm": 1.981733798980713,
+      "learning_rate": 3.1000000000000004e-06,
+      "loss": 0.015,
+      "num_input_tokens_seen": 6104224,
+      "step": 372
+    },
+    {
+      "epoch": 4.797427652733119,
+      "grad_norm": 1.30072820186615,
+      "learning_rate": 3.1083333333333338e-06,
+      "loss": 0.0154,
+      "num_input_tokens_seen": 6120576,
+      "step": 373
+    },
+    {
+      "epoch": 4.810289389067524,
+      "grad_norm": 2.196270704269409,
+      "learning_rate": 3.1166666666666668e-06,
+      "loss": 0.0195,
+      "num_input_tokens_seen": 6137440,
+      "step": 374
+    },
+    {
+      "epoch": 4.823151125401929,
+      "grad_norm": 2.597658157348633,
+      "learning_rate": 3.125e-06,
+      "loss": 0.031,
+      "num_input_tokens_seen": 6154976,
+      "step": 375
+    },
+    {
+      "epoch": 4.836012861736334,
+      "grad_norm": 3.6647465229034424,
+      "learning_rate": 3.133333333333334e-06,
+      "loss": 0.0376,
+      "num_input_tokens_seen": 6170912,
+      "step": 376
+    },
+    {
+      "epoch": 4.84887459807074,
+      "grad_norm": 1.159162998199463,
+      "learning_rate": 3.141666666666667e-06,
+      "loss": 0.0093,
+      "num_input_tokens_seen": 6187968,
+      "step": 377
+    },
+    {
+      "epoch": 4.861736334405145,
+      "grad_norm": 1.5810602903366089,
+      "learning_rate": 3.1500000000000003e-06,
+      "loss": 0.027,
+      "num_input_tokens_seen": 6204320,
+      "step": 378
+    },
+    {
+      "epoch": 4.87459807073955,
+      "grad_norm": 2.0108237266540527,
+      "learning_rate": 3.1583333333333337e-06,
+      "loss": 0.0191,
+      "num_input_tokens_seen": 6220512,
+      "step": 379
+    },
+    {
+      "epoch": 4.887459807073955,
+      "grad_norm": 1.5167394876480103,
+      "learning_rate": 3.1666666666666667e-06,
+      "loss": 0.0132,
+      "num_input_tokens_seen": 6236512,
+      "step": 380
+    },
+    {
+      "epoch": 4.90032154340836,
+      "grad_norm": 2.1819732189178467,
+      "learning_rate": 3.175e-06,
+      "loss": 0.0111,
+      "num_input_tokens_seen": 6252416,
+      "step": 381
+    },
+    {
+      "epoch": 4.913183279742765,
+      "grad_norm": 1.414007306098938,
+      "learning_rate": 3.183333333333334e-06,
+      "loss": 0.0145,
+      "num_input_tokens_seen": 6268992,
+      "step": 382
+    },
+    {
+      "epoch": 4.92604501607717,
+      "grad_norm": 1.6091843843460083,
+      "learning_rate": 3.191666666666667e-06,
+      "loss": 0.0082,
+      "num_input_tokens_seen": 6285312,
+      "step": 383
+    },
+    {
+      "epoch": 4.938906752411576,
+      "grad_norm": 1.0037862062454224,
+      "learning_rate": 3.2000000000000003e-06,
+      "loss": 0.0165,
+      "num_input_tokens_seen": 6300992,
+      "step": 384
+    },
+    {
+      "epoch": 4.951768488745981,
+      "grad_norm": 2.4763152599334717,
+      "learning_rate": 3.2083333333333337e-06,
+      "loss": 0.0243,
+      "num_input_tokens_seen": 6318560,
+      "step": 385
+    },
+    {
+      "epoch": 4.951768488745981,
+      "num_input_tokens_seen": 6318560,
+      "step": 385,
+      "total_flos": 2.5049338265940787e+17,
+      "train_loss": 0.9364173337708336,
+      "train_runtime": 4331.1035,
+      "train_samples_per_second": 22.95,
+      "train_steps_per_second": 0.089
+    }
+  ],
+  "logging_steps": 1,
+  "max_steps": 385,
+  "num_input_tokens_seen": 6318560,
+  "num_train_epochs": 5,
+  "save_steps": 980,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 2.5049338265940787e+17,
+  "train_batch_size": 4,
+  "trial_name": null,
+  "trial_params": null
+}

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:76f84697cc5f35a19aa098445f2ef3e5650b3d161fb9f59aeb9c53bd327abfd6
+size 6584