llama3 results

llama-factory/config/glm-4-9b_lora_sft_bf16-p1.yaml

@@ -13,7 +13,7 @@ upcast_layernorm: true
 
 ### dataset
 dataset: alpaca_mgtv_p1
-template: llama3
+template: glm4
 cutoff_len: 4096
 max_samples: 25000
 overwrite_cache: true

llama-factory/config/glm-4-9b_lora_sft_bf16-p2.yaml

@@ -12,15 +12,15 @@ loraplus_lr_ratio: 16.0                 # use LoRA+ with lambda=16.0
 upcast_layernorm: true
 
 ### dataset
-dataset: alpaca_mgtv_p1
-template: llama3
+dataset: alpaca_mgtv_p2
+template: glm4
 cutoff_len: 4096
 max_samples: 25000
 overwrite_cache: true
 preprocessing_num_workers: 16
 
 ### output
-output_dir: saves/glm-4-9b/lora/sft_bf16_p1_full
+output_dir: saves/glm-4-9b/lora/sft_bf16_p2_full
 logging_steps: 10
 save_steps: 175
 plot_loss: true
@@ -43,4 +43,4 @@ eval_strategy: steps
 eval_steps: 175
 
 report_to: wandb
-run_name: glm-4-9b_p1_full # optional
+run_name: glm-4-9b_p2_full # optional

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: THUDM/glm-4-9b-chat-1m
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/adapter_config.json

@@ -0,0 +1,31 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "THUDM/glm-4-9b-chat-1m",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "dense_h_to_4h",
+    "query_key_value",
+    "dense_4h_to_h",
+    "dense"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53bff5afc22b8b0cb765a3f584f54c10fa235e1941f0077848d142bf671afd16
+size 85409560

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/added_tokens.json

@@ -0,0 +1,16 @@
+{
+  "<eop>": 151334,
+  "<sop>": 151333,
+  "<|assistant|>": 151337,
+  "<|begin_of_image|>": 151339,
+  "<|begin_of_video|>": 151341,
+  "<|end_of_image|>": 151340,
+  "<|end_of_video|>": 151342,
+  "<|endoftext|>": 151329,
+  "<|observation|>": 151338,
+  "<|system|>": 151335,
+  "<|user|>": 151336,
+  "[MASK]": 151330,
+  "[gMASK]": 151331,
+  "[sMASK]": 151332
+}

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3922cca390d6be2fa1661bf96614e3b664825c1923aee428500bb9efe011a24f
+size 170990330

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1ff264f99d31b522cc7e2a4eac9d38606d0c58a34c0adc74d71e0ca8b371dc36
+size 14244

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:33b7a8d09c2546f7106398b095bc2a462fc9b7d28f34ff33c2a19f3162ac9490
+size 1064

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/special_tokens_map.json

@@ -0,0 +1,32 @@
+{
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<sop>",
+    "<eop>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|begin_of_image|>",
+    "<|end_of_image|>",
+    "<|begin_of_video|>",
+    "<|end_of_video|>"
+  ],
+  "eos_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|endoftext|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/tokenization_chatglm.py

@@ -0,0 +1,323 @@
+import regex as re
+import base64
+import os
+import json
+import tiktoken
+from torch import TensorType
+from typing import List, Optional, Union, Dict, Any
+from transformers import PreTrainedTokenizer
+from transformers.utils import logging, PaddingStrategy
+from transformers.tokenization_utils_base import EncodedInput, BatchEncoding
+
+
+class ChatGLM4Tokenizer(PreTrainedTokenizer):
+    vocab_files_names = {"vocab_file": "tokenizer.model"}
+    model_input_names = ["input_ids", "attention_mask", "position_ids"]
+
+    def __init__(
+            self,
+            vocab_file,
+            padding_side="left",
+            clean_up_tokenization_spaces=False,
+            encode_special_tokens=False,
+            **kwargs
+    ):
+        self.name = "GLM4Tokenizer"
+        self.vocab_file = vocab_file
+        pat_str = "(?i:'s|'t|'re|'ve|'m|'ll|'d)|[^\\r\\n\\p{L}\\p{N}]?\\p{L}+|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+        self.pat_str = re.compile(pat_str)
+        self.encode_special_tokens = encode_special_tokens
+
+        mergeable_ranks = {}
+        with open(vocab_file) as f:
+            for line in f:
+                token, rank = line.strip().split()
+                rank = int(rank)
+                token = base64.b64decode(token)
+                mergeable_ranks[token] = rank
+
+        self.mergeable_ranks = mergeable_ranks
+
+        self.tokenizer = tiktoken.Encoding(
+            name="my_tokenizer",
+            pat_str=pat_str,
+            mergeable_ranks=mergeable_ranks,
+            special_tokens={}
+        )
+        self.decoder = {rank: token for token, rank in mergeable_ranks.items()}
+        self.n_words = len(self.decoder)
+
+        super().__init__(
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs
+        )
+
+    @property
+    def vocab_size(self):
+        return self.n_words
+
+    def get_vocab(self):
+        """ Returns vocab as a dict """
+        vocab = {self._convert_id_to_token(i): i for i in range(self.vocab_size)}
+        vocab.update(self.added_tokens_encoder)
+        return vocab
+
+    def convert_tokens_to_string(self, tokens: List[Union[bytes, str, int]]) -> str:
+        """
+        Converts a sequence of tokens in a single string.
+        """
+        text = ""
+        temp = b""
+        for t in tokens:
+            if isinstance(t, int):
+                t = chr(t)
+            if isinstance(t, str):
+                if temp:
+                    text += temp.decode("utf-8", errors="replace")
+            elif isinstance(t, bytes):
+                temp += t
+            else:
+                raise TypeError("token should only be of type int, bytes or str")
+        if temp:
+            text += temp.decode("utf-8", errors="replace")
+        return text
+
+    def _tokenize(self, text, **kwargs):
+        tokens = []
+        ids = self.tokenizer.encode(text)
+        for t in ids:
+            tokens.append(self.decoder[t])
+        return tokens
+
+    def _convert_token_to_id(self, token):
+        """ Converts a token (str) in an id using the vocab. """
+        return self.mergeable_ranks[token]
+
+    def _convert_id_to_token(self, index):
+        """Converts an index (integer) in a token (str) using the vocab."""
+        return self.decoder.get(index, "")
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        """
+        Save the vocabulary and special tokens file to a directory.
+
+        Args:
+            save_directory (`str`):
+                The directory in which to save the vocabulary.
+            filename_prefix (`str`, *optional*):
+                An optional prefix to add to the named of the saved files.
+
+        Returns:
+            `Tuple(str)`: Paths to the files saved.
+        """
+        if os.path.isdir(save_directory):
+            vocab_file = os.path.join(
+                save_directory, self.vocab_files_names["vocab_file"]
+            )
+        else:
+            vocab_file = save_directory
+
+        with open(self.vocab_file, 'rb') as fin:
+            proto_str = fin.read()
+
+        with open(vocab_file, "wb") as writer:
+            writer.write(proto_str)
+
+        return (vocab_file,)
+
+    def get_prefix_tokens(self):
+        prefix_tokens = [self.convert_tokens_to_ids("[gMASK]"), self.convert_tokens_to_ids("<sop>")]
+        return prefix_tokens
+
+    def build_single_message(self, role, metadata, message, tokenize=True):
+        assert role in ["system", "user", "assistant", "observation"], role
+        if tokenize:
+            role_tokens = [self.convert_tokens_to_ids(f"<|{role}|>")] + self.tokenizer.encode(f"{metadata}\n",
+                                                                                              disallowed_special=())
+            message_tokens = self.tokenizer.encode(message, disallowed_special=())
+            tokens = role_tokens + message_tokens
+            return tokens
+        else:
+            return str(f"<|{role}|>{metadata}\n{message}")
+
+    # Use Jinja Template in tokenizer_config.json
+    # def apply_chat_template(
+    #         self,
+    #         conversation: Union[List[Dict[str, str]], List[List[Dict[str, str]]], "Conversation"],
+    #         add_generation_prompt: bool = False,
+    #         tokenize: bool = True,
+    #         padding: bool = False,
+    #         truncation: bool = False,
+    #         max_length: Optional[int] = None,
+    #         return_tensors: Optional[Union[str, TensorType]] = None,
+    #         return_dict: bool = False,
+    #         tokenizer_kwargs: Optional[Dict[str, Any]] = None,
+    #         add_special_tokens: bool = True,
+    #         **kwargs,
+    # ) -> Union[str, List[int], List[str], List[List[int]], BatchEncoding]:
+    #
+    #     if return_dict and not tokenize:
+    #         raise ValueError(
+    #             "`return_dict=True` is incompatible with `tokenize=False`, because there is no dict "
+    #             "of tokenizer outputs to return."
+    #         )
+    #
+    #     def handle_single_conversation(conversation):
+    #         input_ids = self.get_prefix_tokens() if add_special_tokens else []
+    #         input_message = "[gMASK]<sop>" if add_special_tokens else ""
+    #         for item in conversation:
+    #             if item.get("tools"):
+    #                 tools = item["tools"]
+    #                 content = "你是一个名为 GhatGLM 的人工智能助手。你是基于智谱AI训练的语言模型 GLM-4 模型开发的，你的任务是针对用户的问题和要求提供适当的答复和支持。"
+    #                 content += "\n\n# 可用工具"
+    #                 for tool in tools:
+    #                     if tool["type"] == "function":
+    #                         function = tool["function"]
+    #                         content += f"\n\n## {function['name']}\n\n{json.dumps(function, ensure_ascii=False, indent=4)}"
+    #                         content += "\n在调用上述函数时，请使用 Json 格式表示调用的参数。"
+    #                     elif tool["type"] == "python":
+    #                         content += "\n\n## python\n\n当你向 `python` 发送包含 Python 代码的消息时，该代码将会在一个有状态的 Jupyter notebook 环境中执行。\n`python` 返回代码执行的输出，或在执行 60 秒后返回超时。\n`/mnt/data` 将会持久化存储你的文件。在此会话中，`python` 无法访问互联网。不要使用 `python` 进行任何网络请求或者在线 API 调用，这些在线内容的访问将不会成功。"
+    #                     elif tool["type"] == "simple_browser":
+    #                         content += "\n\n## simple_browser\n\n你可以使用 `simple_browser` 工具。该工具支持以下函数：\n`search(query: str, recency_days: int)`：使用搜索引擎进行查询并显示结果，可以使用 `recency_days` 参数控制搜索内容的时效性。\n`mclick(ids: list[int])`：获取一系列指定 id 的页面内容。每次调用时，须选择3-10个页面。选择多个角度的页面，同时尽可能选择可信任的信息来源。考虑到部分页面是无法加载的，你也可以多打开一些可能有用的页面而不用担心内容过多。\n`open_url(url: str)`：打开指定的 URL。\n\n使用 `【{引用 id}†{引用文本}】` 来引用内容。\n\n操作步骤：1. 使用 `search` 来获得信息列表; 2. 使用 `mclick` 来获取指定 ID 页面的内容; 3. 根据获得的内容进行回复。在回复中应当引用信息来源。\n 如果用户提供了 URL，也可以用 `open_url` 直接打开页面。\n如果初次搜索结果没有找到合适的信息，也可以再次使用 `search` 进行搜索。"
+    #                     elif tool["type"] == "cogview":
+    #                         content += "\n\n## cogview\n\n如果用户的请求中包含了对图像的描述，你可以使用 `cogview` 来生成图像并展示给用户。你需要向 `cogview` 发送图像描述，规则：\n- 发送给 `cogview` 的消息必须使用英语。用户的中文描述必须完全翻译为英语。\n- 应当尽可能详细地描述图像生成的需求，需求描述约 100 英文单词。\n- 保持用户原始描述的意图。不要虚构内容或者没见过的人物。\n- 如无特殊说明，所在地为中国，持有中国立场并遵循中国社会主义价值观。"
+    #                     else:
+    #                         raise NotImplementedError(f"Unknown tool type {tool['type']}")
+    #                 input = self.build_single_message("system", "", content, tokenize=tokenize)
+    #                 if tokenize:
+    #                     input_ids.extend(input)
+    #                 else:
+    #                     input_message += input
+    #             if item["content"]:
+    #                 input = self.build_single_message(
+    #                     item["role"],
+    #                     item.get("metadata", ""),
+    #                     item["content"],
+    #                     tokenize=tokenize
+    #                 )
+    #                 if tokenize:
+    #                     input_ids.extend(input)
+    #                 else:
+    #                     input_message += input
+    #         if add_generation_prompt:
+    #             if tokenize:
+    #                 input_ids.extend([self.convert_tokens_to_ids("<|assistant|>")])
+    #             else:
+    #                 input_message += "<|assistant|>"
+    #         return input_ids if tokenize else input_message
+    #
+    #     # Main logic to handle different conversation formats
+    #     if isinstance(conversation, list) and all(isinstance(i, dict) for i in conversation):
+    #         result = handle_single_conversation(conversation)
+    #     elif isinstance(conversation, list) and all(isinstance(i, list) for i in conversation):
+    #         result = [handle_single_conversation(c) for c in conversation]
+    #     elif hasattr(conversation, "messages"):
+    #         result = handle_single_conversation(conversation.messages)
+    #     else:
+    #         raise ValueError("Invalid conversation format")
+    #
+    #     if tokenize:
+    #         output = self.batch_encode_plus(
+    #             [result] if isinstance(result[0], int) else result,
+    #             padding=padding,
+    #             truncation=truncation,
+    #             max_length=max_length,
+    #             return_tensors=return_tensors,
+    #             is_split_into_words=True,
+    #             add_special_tokens=False
+    #         )
+    #         if return_dict:
+    #             return output
+    #         else:
+    #             return output["input_ids"]
+    #     else:
+    #         return result
+
+    def build_inputs_with_special_tokens(
+            self, token_ids_0: List[int], token_ids_1: Optional[List[int]] = None
+    ) -> List[int]:
+        """
+        Build model inputs from a sequence or a pair of sequence for sequence classification tasks by concatenating and
+        adding special tokens. A BERT sequence has the following format:
+
+        - single sequence: `[CLS] X [SEP]`
+        - pair of sequences: `[CLS] A [SEP] B [SEP]`
+
+        Args:
+            token_ids_0 (`List[int]`):
+                List of IDs to which the special tokens will be added.
+            token_ids_1 (`List[int]`, *optional*):
+                Optional second list of IDs for sequence pairs.
+
+        Returns:
+            `List[int]`: List of [input IDs](../glossary#input-ids) with the appropriate special tokens.
+        """
+        prefix_tokens = self.get_prefix_tokens()
+        token_ids_0 = prefix_tokens + token_ids_0
+        if token_ids_1 is not None:
+            token_ids_0 = token_ids_0 + token_ids_1 + [self.convert_tokens_to_ids("<eos>")]
+        return token_ids_0
+
+    def _pad(
+            self,
+            encoded_inputs: Union[Dict[str, EncodedInput], BatchEncoding],
+            max_length: Optional[int] = None,
+            padding_strategy: PaddingStrategy = PaddingStrategy.DO_NOT_PAD,
+            pad_to_multiple_of: Optional[int] = None,
+            return_attention_mask: Optional[bool] = None,
+    ) -> dict:
+        """
+        Pad encoded inputs (on left/right and up to predefined length or max length in the batch)
+
+        Args:
+            encoded_inputs:
+                Dictionary of tokenized inputs (`List[int]`) or batch of tokenized inputs (`List[List[int]]`).
+            max_length: maximum length of the returned list and optionally padding length (see below).
+                Will truncate by taking into account the special tokens.
+            padding_strategy: PaddingStrategy to use for padding.
+
+                - PaddingStrategy.LONGEST Pad to the longest sequence in the batch
+                - PaddingStrategy.MAX_LENGTH: Pad to the max length (default)
+                - PaddingStrategy.DO_NOT_PAD: Do not pad
+                The tokenizer padding sides are defined in self.padding_side:
+
+                    - 'left': pads on the left of the sequences
+                    - 'right': pads on the right of the sequences
+            pad_to_multiple_of: (optional) Integer if set will pad the sequence to a multiple of the provided value.
+                This is especially useful to enable the use of Tensor Core on NVIDIA hardware with compute capability
+                `>= 7.5` (Volta).
+            return_attention_mask:
+                (optional) Set to False to avoid returning attention mask (default: set to model specifics)
+        """
+        # Load from model defaults
+        assert self.padding_side == "left"
+
+        required_input = encoded_inputs[self.model_input_names[0]]
+        seq_length = len(required_input)
+
+        if padding_strategy == PaddingStrategy.LONGEST:
+            max_length = len(required_input)
+
+        if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
+            max_length = ((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of
+
+        needs_to_be_padded = padding_strategy != PaddingStrategy.DO_NOT_PAD and len(required_input) != max_length
+
+        # Initialize attention mask if not present.
+        if "attention_mask" not in encoded_inputs:
+            encoded_inputs["attention_mask"] = [1] * seq_length
+
+        if "position_ids" not in encoded_inputs:
+            encoded_inputs["position_ids"] = list(range(seq_length))
+
+        if needs_to_be_padded:
+            difference = max_length - len(required_input)
+
+            if "attention_mask" in encoded_inputs:
+                encoded_inputs["attention_mask"] = [0] * difference + encoded_inputs["attention_mask"]
+            if "position_ids" in encoded_inputs:
+                encoded_inputs["position_ids"] = [0] * difference + encoded_inputs["position_ids"]
+            encoded_inputs[self.model_input_names[0]] = [self.pad_token_id] * difference + required_input
+
+        return encoded_inputs

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5a493598071550244b2ee7f26118f3edec2150b9dfa967929a99052ac83fe716
+size 2623634

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/tokenizer_config.json

@@ -0,0 +1,148 @@
+{
+  "added_tokens_decoder": {
+    "151329": {
+      "content": "<|endoftext|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151330": {
+      "content": "[MASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151331": {
+      "content": "[gMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151332": {
+      "content": "[sMASK]",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151333": {
+      "content": "<sop>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151334": {
+      "content": "<eop>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151335": {
+      "content": "<|system|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151336": {
+      "content": "<|user|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151337": {
+      "content": "<|assistant|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151338": {
+      "content": "<|observation|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151339": {
+      "content": "<|begin_of_image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151340": {
+      "content": "<|end_of_image|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151341": {
+      "content": "<|begin_of_video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "151342": {
+      "content": "<|end_of_video|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "additional_special_tokens": [
+    "<|endoftext|>",
+    "[MASK]",
+    "[gMASK]",
+    "[sMASK]",
+    "<sop>",
+    "<eop>",
+    "<|system|>",
+    "<|user|>",
+    "<|assistant|>",
+    "<|observation|>",
+    "<|begin_of_image|>",
+    "<|end_of_image|>",
+    "<|begin_of_video|>",
+    "<|end_of_video|>"
+  ],
+  "auto_map": {
+    "AutoTokenizer": [
+      "tokenization_chatglm.ChatGLM4Tokenizer",
+      null
+    ]
+  },
+  "chat_template": "{{ '[gMASK]<sop>' }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|system|>\n' + system_message }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|user|>\n' + content + '<|assistant|>' }}{% elif message['role'] == 'assistant' %}{{ '\n' + content }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": false,
+  "do_lower_case": false,
+  "eos_token": "<|endoftext|>",
+  "model_max_length": 1024000,
+  "pad_token": "<|endoftext|>",
+  "padding_side": "right",
+  "remove_space": false,
+  "split_special_tokens": false,
+  "tokenizer_class": "ChatGLM4Tokenizer"
+}

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/trainer_state.json

@@ -0,0 +1,161 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 0.9950248756218906,
+  "eval_steps": 175,
+  "global_step": 175,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05685856432125089,
+      "grad_norm": 2.919694662094116,
+      "learning_rate": 1.4285714285714285e-05,
+      "loss": 3.8009,
+      "step": 10
+    },
+    {
+      "epoch": 0.11371712864250177,
+      "grad_norm": 3.130059003829956,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 0.3289,
+      "step": 20
+    },
+    {
+      "epoch": 0.17057569296375266,
+      "grad_norm": 1.6621949672698975,
+      "learning_rate": 4.2857142857142856e-05,
+      "loss": 0.2598,
+      "step": 30
+    },
+    {
+      "epoch": 0.22743425728500355,
+      "grad_norm": 2.371370792388916,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 0.2401,
+      "step": 40
+    },
+    {
+      "epoch": 0.28429282160625446,
+      "grad_norm": 0.8625539541244507,
+      "learning_rate": 7.142857142857143e-05,
+      "loss": 0.2306,
+      "step": 50
+    },
+    {
+      "epoch": 0.3411513859275053,
+      "grad_norm": 5.281027793884277,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.2463,
+      "step": 60
+    },
+    {
+      "epoch": 0.39800995024875624,
+      "grad_norm": 1.4231479167938232,
+      "learning_rate": 0.0001,
+      "loss": 0.2291,
+      "step": 70
+    },
+    {
+      "epoch": 0.4548685145700071,
+      "grad_norm": 1.9341018199920654,
+      "learning_rate": 9.993784606094612e-05,
+      "loss": 0.2301,
+      "step": 80
+    },
+    {
+      "epoch": 0.511727078891258,
+      "grad_norm": 2.0011138916015625,
+      "learning_rate": 9.975153876827008e-05,
+      "loss": 0.2207,
+      "step": 90
+    },
+    {
+      "epoch": 0.5685856432125089,
+      "grad_norm": 0.6652698516845703,
+      "learning_rate": 9.944154131125642e-05,
+      "loss": 0.2271,
+      "step": 100
+    },
+    {
+      "epoch": 0.6254442075337597,
+      "grad_norm": 1.3393596410751343,
+      "learning_rate": 9.900862439242719e-05,
+      "loss": 0.216,
+      "step": 110
+    },
+    {
+      "epoch": 0.6823027718550106,
+      "grad_norm": 5.120615482330322,
+      "learning_rate": 9.84538643114539e-05,
+      "loss": 0.2311,
+      "step": 120
+    },
+    {
+      "epoch": 0.7391613361762616,
+      "grad_norm": 1.72005033493042,
+      "learning_rate": 9.777864028930705e-05,
+      "loss": 0.212,
+      "step": 130
+    },
+    {
+      "epoch": 0.7960199004975125,
+      "grad_norm": 1.5774250030517578,
+      "learning_rate": 9.698463103929542e-05,
+      "loss": 0.2261,
+      "step": 140
+    },
+    {
+      "epoch": 0.8528784648187633,
+      "grad_norm": 93.82937622070312,
+      "learning_rate": 9.607381059352038e-05,
+      "loss": 2.368,
+      "step": 150
+    },
+    {
+      "epoch": 0.9097370291400142,
+      "grad_norm": 11.905146598815918,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 2.4337,
+      "step": 160
+    },
+    {
+      "epoch": 0.9665955934612651,
+      "grad_norm": 4.584591388702393,
+      "learning_rate": 9.391107866851143e-05,
+      "loss": 0.8958,
+      "step": 170
+    },
+    {
+      "epoch": 0.9950248756218906,
+      "eval_accuracy": 0.7612800000000001,
+      "eval_loss": 0.4472738206386566,
+      "eval_runtime": 136.3095,
+      "eval_samples_per_second": 18.341,
+      "eval_steps_per_second": 18.341,
+      "step": 175
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 700,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 4,
+  "save_steps": 175,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 4.358760815174615e+17,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/checkpoint-175/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9e6f3241dabbb8a63b52024fb0fb0d68c85f4b48b07e8579ebd8f41fe5fd662
+size 5304

llama-factory/saves/glm-4-9b/lora/sft_bf16_p1_full/trainer_log.jsonl

@@ -0,0 +1,31 @@
+{"current_steps": 10, "total_steps": 700, "loss": 3.8009, "learning_rate": 1.4285714285714285e-05, "epoch": 0.05685856432125089, "percentage": 1.43, "elapsed_time": "0:02:16", "remaining_time": "2:37:24", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 20, "total_steps": 700, "loss": 0.3289, "learning_rate": 2.857142857142857e-05, "epoch": 0.11371712864250177, "percentage": 2.86, "elapsed_time": "0:04:33", "remaining_time": "2:35:10", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 30, "total_steps": 700, "loss": 0.2598, "learning_rate": 4.2857142857142856e-05, "epoch": 0.17057569296375266, "percentage": 4.29, "elapsed_time": "0:06:51", "remaining_time": "2:32:59", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 40, "total_steps": 700, "loss": 0.2401, "learning_rate": 5.714285714285714e-05, "epoch": 0.22743425728500355, "percentage": 5.71, "elapsed_time": "0:09:08", "remaining_time": "2:30:47", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 50, "total_steps": 700, "loss": 0.2306, "learning_rate": 7.142857142857143e-05, "epoch": 0.28429282160625446, "percentage": 7.14, "elapsed_time": "0:11:25", "remaining_time": "2:28:26", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 60, "total_steps": 700, "loss": 0.2463, "learning_rate": 8.571428571428571e-05, "epoch": 0.3411513859275053, "percentage": 8.57, "elapsed_time": "0:13:42", "remaining_time": "2:26:12", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 70, "total_steps": 700, "loss": 0.2291, "learning_rate": 0.0001, "epoch": 0.39800995024875624, "percentage": 10.0, "elapsed_time": "0:15:58", "remaining_time": "2:23:49", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 80, "total_steps": 700, "loss": 0.2301, "learning_rate": 9.993784606094612e-05, "epoch": 0.4548685145700071, "percentage": 11.43, "elapsed_time": "0:18:16", "remaining_time": "2:21:40", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 90, "total_steps": 700, "loss": 0.2207, "learning_rate": 9.975153876827008e-05, "epoch": 0.511727078891258, "percentage": 12.86, "elapsed_time": "0:20:33", "remaining_time": "2:19:20", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 100, "total_steps": 700, "loss": 0.2271, "learning_rate": 9.944154131125642e-05, "epoch": 0.5685856432125089, "percentage": 14.29, "elapsed_time": "0:22:50", "remaining_time": "2:17:02", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 110, "total_steps": 700, "loss": 0.216, "learning_rate": 9.900862439242719e-05, "epoch": 0.6254442075337597, "percentage": 15.71, "elapsed_time": "0:25:07", "remaining_time": "2:14:43", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 120, "total_steps": 700, "loss": 0.2311, "learning_rate": 9.84538643114539e-05, "epoch": 0.6823027718550106, "percentage": 17.14, "elapsed_time": "0:27:24", "remaining_time": "2:12:27", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 130, "total_steps": 700, "loss": 0.212, "learning_rate": 9.777864028930705e-05, "epoch": 0.7391613361762616, "percentage": 18.57, "elapsed_time": "0:29:41", "remaining_time": "2:10:10", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 140, "total_steps": 700, "loss": 0.2261, "learning_rate": 9.698463103929542e-05, "epoch": 0.7960199004975125, "percentage": 20.0, "elapsed_time": "0:31:59", "remaining_time": "2:07:56", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 150, "total_steps": 700, "loss": 2.368, "learning_rate": 9.607381059352038e-05, "epoch": 0.8528784648187633, "percentage": 21.43, "elapsed_time": "0:34:15", "remaining_time": "2:05:37", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 160, "total_steps": 700, "loss": 2.4337, "learning_rate": 9.504844339512095e-05, "epoch": 0.9097370291400142, "percentage": 22.86, "elapsed_time": "0:36:32", "remaining_time": "2:03:21", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 170, "total_steps": 700, "loss": 0.8958, "learning_rate": 9.391107866851143e-05, "epoch": 0.9665955934612651, "percentage": 24.29, "elapsed_time": "0:38:49", "remaining_time": "2:01:02", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 175, "total_steps": 700, "eval_loss": 0.4472738206386566, "epoch": 0.9950248756218906, "percentage": 25.0, "elapsed_time": "0:42:14", "remaining_time": "2:06:43", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 180, "total_steps": 700, "loss": 0.4271, "learning_rate": 9.266454408160779e-05, "epoch": 1.023454157782516, "percentage": 25.71, "elapsed_time": "0:43:22", "remaining_time": "2:05:19", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 190, "total_steps": 700, "loss": 0.2324, "learning_rate": 9.131193871579975e-05, "epoch": 1.080312722103767, "percentage": 27.14, "elapsed_time": "0:45:39", "remaining_time": "2:02:34", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 200, "total_steps": 700, "loss": 0.1996, "learning_rate": 8.985662536114613e-05, "epoch": 1.1371712864250179, "percentage": 28.57, "elapsed_time": "0:47:56", "remaining_time": "1:59:52", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 210, "total_steps": 700, "loss": 0.1961, "learning_rate": 8.83022221559489e-05, "epoch": 1.1940298507462686, "percentage": 30.0, "elapsed_time": "0:50:13", "remaining_time": "1:57:12", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 220, "total_steps": 700, "loss": 0.1942, "learning_rate": 8.665259359149132e-05, "epoch": 1.2508884150675195, "percentage": 31.43, "elapsed_time": "0:52:30", "remaining_time": "1:54:34", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 230, "total_steps": 700, "loss": 0.2152, "learning_rate": 8.491184090430364e-05, "epoch": 1.3077469793887704, "percentage": 32.86, "elapsed_time": "0:54:47", "remaining_time": "1:51:58", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 240, "total_steps": 700, "loss": 0.1986, "learning_rate": 8.308429187984297e-05, "epoch": 1.3646055437100213, "percentage": 34.29, "elapsed_time": "0:57:04", "remaining_time": "1:49:24", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 250, "total_steps": 700, "loss": 0.209, "learning_rate": 8.117449009293668e-05, "epoch": 1.4214641080312722, "percentage": 35.71, "elapsed_time": "0:59:21", "remaining_time": "1:46:50", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 260, "total_steps": 700, "loss": 0.191, "learning_rate": 7.91871836117395e-05, "epoch": 1.4783226723525231, "percentage": 37.14, "elapsed_time": "1:01:38", "remaining_time": "1:44:18", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 270, "total_steps": 700, "loss": 0.2046, "learning_rate": 7.712731319328798e-05, "epoch": 1.535181236673774, "percentage": 38.57, "elapsed_time": "1:03:55", "remaining_time": "1:41:47", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 280, "total_steps": 700, "loss": 0.1939, "learning_rate": 7.500000000000001e-05, "epoch": 1.5920398009950247, "percentage": 40.0, "elapsed_time": "1:06:12", "remaining_time": "1:39:18", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 290, "total_steps": 700, "loss": 0.2072, "learning_rate": 7.281053286765815e-05, "epoch": 1.6488983653162759, "percentage": 41.43, "elapsed_time": "1:08:29", "remaining_time": "1:36:49", "throughput": "0.00", "total_tokens": 0}
+{"current_steps": 300, "total_steps": 700, "loss": 0.1934, "learning_rate": 7.056435515653059e-05, "epoch": 1.7057569296375266, "percentage": 42.86, "elapsed_time": "1:10:46", "remaining_time": "1:34:21", "throughput": "0.00", "total_tokens": 0}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/README.md

@@ -0,0 +1,76 @@
+---
+license: other
+library_name: peft
+tags:
+- llama-factory
+- lora
+- generated_from_trainer
+base_model: shenzhi-wang/Llama3-8B-Chinese-Chat
+metrics:
+- accuracy
+model-index:
+- name: sft_bf16_p1_full
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# sft_bf16_p1_full
+
+This model is a fine-tuned version of [shenzhi-wang/Llama3-8B-Chinese-Chat](https://huggingface.co/shenzhi-wang/Llama3-8B-Chinese-Chat) on the alpaca_mgtv_p1 dataset.
+It achieves the following results on the evaluation set:
+- Loss: 0.7541
+- Accuracy: 0.9024
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0001
+- train_batch_size: 16
+- eval_batch_size: 1
+- seed: 42
+- gradient_accumulation_steps: 8
+- total_train_batch_size: 128
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: cosine
+- lr_scheduler_warmup_ratio: 0.1
+- num_epochs: 10.0
+
+### Training results
+
+| Training Loss | Epoch  | Step | Accuracy | Validation Loss |
+|:-------------:|:------:|:----:|:--------:|:---------------:|
+| 0.258         | 0.9950 | 175  | 0.909    | 0.2465          |
+| 0.2297        | 1.9900 | 350  | 0.9031   | 0.2461          |
+| 0.1792        | 2.9851 | 525  | 0.9013   | 0.2521          |
+| 0.122         | 3.9801 | 700  | 0.9050   | 0.2748          |
+| 0.073         | 4.9751 | 875  | 0.9083   | 0.3309          |
+| 0.0484        | 5.9701 | 1050 | 0.9057   | 0.4314          |
+| 0.0691        | 6.9950 | 1225 | 0.4234   | 0.9058          |
+| 0.0466        | 7.9900 | 1400 | 0.4437   | 0.9055          |
+| 0.0365        | 8.9851 | 1575 | 0.6049   | 0.9038          |
+| 0.0338        | 9.9801 | 1750 | 0.7541   | 0.9024          |
+
+
+### Framework versions
+
+- PEFT 0.11.1
+- Transformers 4.41.2
+- Pytorch 2.2.1+cu121
+- Datasets 2.19.1
+- Tokenizers 0.19.1

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "shenzhi-wang/Llama3-8B-Chinese-Chat",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3168a79e537f2be379be61a9722ef961b4184da9da3e5ac8f4fcb00c47222f54
+size 83945296

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/all_results.json

@@ -0,0 +1,13 @@
+{
+    "epoch": 9.980099502487562,
+    "eval_accuracy": 0.9024,
+    "eval_loss": 0.7541078925132751,
+    "eval_runtime": 206.1105,
+    "eval_samples_per_second": 12.129,
+    "eval_steps_per_second": 12.129,
+    "total_flos": 5.085255522623422e+18,
+    "train_loss": 0.020103505866868154,
+    "train_runtime": 30005.6869,
+    "train_samples_per_second": 7.499,
+    "train_steps_per_second": 0.058
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: shenzhi-wang/Llama3-8B-Chinese-Chat
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "shenzhi-wang/Llama3-8B-Chinese-Chat",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "down_proj",
+    "up_proj",
+    "gate_proj",
+    "k_proj",
+    "v_proj",
+    "o_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:088013fcc28735c873f194b57f82fa95311fd81789a91355e5b1371417f66409
+size 83945296

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41fa18f01e112b2c99e1c9d683a05978b240458e14418e2f9e80146b15b583f3
+size 168149394

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3e5d946241df2516b06d7074d8779088eae7607173ad780df56583910a9589b
+size 14244

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3e62b5c4d714158b801cbca58b356b6362807d0824a52c3e4e5c6a9777589410
+size 1064

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/trainer_state.json

@@ -0,0 +1,822 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 5.970149253731344,
+  "eval_steps": 175,
+  "global_step": 1050,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05685856432125089,
+      "grad_norm": 1.478421688079834,
+      "learning_rate": 9.523809523809523e-06,
+      "loss": 0.9166,
+      "step": 10
+    },
+    {
+      "epoch": 0.11371712864250177,
+      "grad_norm": 0.9236305356025696,
+      "learning_rate": 1.9047619047619046e-05,
+      "loss": 0.3687,
+      "step": 20
+    },
+    {
+      "epoch": 0.17057569296375266,
+      "grad_norm": 0.6159961223602295,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 0.328,
+      "step": 30
+    },
+    {
+      "epoch": 0.22743425728500355,
+      "grad_norm": 1.8057916164398193,
+      "learning_rate": 3.809523809523809e-05,
+      "loss": 0.3049,
+      "step": 40
+    },
+    {
+      "epoch": 0.28429282160625446,
+      "grad_norm": 1.5811758041381836,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 0.2999,
+      "step": 50
+    },
+    {
+      "epoch": 0.3411513859275053,
+      "grad_norm": 1.9880269765853882,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 0.2951,
+      "step": 60
+    },
+    {
+      "epoch": 0.39800995024875624,
+      "grad_norm": 1.5736329555511475,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 0.282,
+      "step": 70
+    },
+    {
+      "epoch": 0.4548685145700071,
+      "grad_norm": 1.3553781509399414,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 0.2805,
+      "step": 80
+    },
+    {
+      "epoch": 0.511727078891258,
+      "grad_norm": 1.3975526094436646,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.2717,
+      "step": 90
+    },
+    {
+      "epoch": 0.5685856432125089,
+      "grad_norm": 1.0928113460540771,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 0.2659,
+      "step": 100
+    },
+    {
+      "epoch": 0.6254442075337597,
+      "grad_norm": 1.163079857826233,
+      "learning_rate": 9.999309273455528e-05,
+      "loss": 0.27,
+      "step": 110
+    },
+    {
+      "epoch": 0.6823027718550106,
+      "grad_norm": 0.849062979221344,
+      "learning_rate": 9.993784606094612e-05,
+      "loss": 0.2671,
+      "step": 120
+    },
+    {
+      "epoch": 0.7391613361762616,
+      "grad_norm": 0.8933653831481934,
+      "learning_rate": 9.982741376606078e-05,
+      "loss": 0.2869,
+      "step": 130
+    },
+    {
+      "epoch": 0.7960199004975125,
+      "grad_norm": 1.046842098236084,
+      "learning_rate": 9.966191788709716e-05,
+      "loss": 0.2997,
+      "step": 140
+    },
+    {
+      "epoch": 0.8528784648187633,
+      "grad_norm": 2.4692931175231934,
+      "learning_rate": 9.944154131125642e-05,
+      "loss": 0.2735,
+      "step": 150
+    },
+    {
+      "epoch": 0.9097370291400142,
+      "grad_norm": 1.4990626573562622,
+      "learning_rate": 9.916652757363698e-05,
+      "loss": 0.2905,
+      "step": 160
+    },
+    {
+      "epoch": 0.9665955934612651,
+      "grad_norm": 1.4853521585464478,
+      "learning_rate": 9.883718058810707e-05,
+      "loss": 0.258,
+      "step": 170
+    },
+    {
+      "epoch": 0.9950248756218906,
+      "eval_accuracy": 0.909,
+      "eval_loss": 0.24650488793849945,
+      "eval_runtime": 206.2691,
+      "eval_samples_per_second": 12.12,
+      "eval_steps_per_second": 12.12,
+      "step": 175
+    },
+    {
+      "epoch": 1.023454157782516,
+      "grad_norm": 0.8761231303215027,
+      "learning_rate": 9.84538643114539e-05,
+      "loss": 0.2471,
+      "step": 180
+    },
+    {
+      "epoch": 1.080312722103767,
+      "grad_norm": 1.037913203239441,
+      "learning_rate": 9.801700234117999e-05,
+      "loss": 0.2304,
+      "step": 190
+    },
+    {
+      "epoch": 1.1371712864250179,
+      "grad_norm": 0.5963209867477417,
+      "learning_rate": 9.752707744739145e-05,
+      "loss": 0.2414,
+      "step": 200
+    },
+    {
+      "epoch": 1.1940298507462686,
+      "grad_norm": 0.8125211596488953,
+      "learning_rate": 9.698463103929542e-05,
+      "loss": 0.2276,
+      "step": 210
+    },
+    {
+      "epoch": 1.2508884150675195,
+      "grad_norm": 2.1267828941345215,
+      "learning_rate": 9.639026256689628e-05,
+      "loss": 0.2137,
+      "step": 220
+    },
+    {
+      "epoch": 1.3077469793887704,
+      "grad_norm": 3.7273290157318115,
+      "learning_rate": 9.574462885855174e-05,
+      "loss": 0.2467,
+      "step": 230
+    },
+    {
+      "epoch": 1.3646055437100213,
+      "grad_norm": 4.215674877166748,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 0.2682,
+      "step": 240
+    },
+    {
+      "epoch": 1.4214641080312722,
+      "grad_norm": 1.7884002923965454,
+      "learning_rate": 9.430247552150673e-05,
+      "loss": 0.29,
+      "step": 250
+    },
+    {
+      "epoch": 1.4783226723525231,
+      "grad_norm": 1.312519907951355,
+      "learning_rate": 9.350754959646306e-05,
+      "loss": 0.2252,
+      "step": 260
+    },
+    {
+      "epoch": 1.535181236673774,
+      "grad_norm": 2.8921542167663574,
+      "learning_rate": 9.266454408160779e-05,
+      "loss": 0.2408,
+      "step": 270
+    },
+    {
+      "epoch": 1.5920398009950247,
+      "grad_norm": 2.1106109619140625,
+      "learning_rate": 9.177439057064683e-05,
+      "loss": 0.2171,
+      "step": 280
+    },
+    {
+      "epoch": 1.6488983653162759,
+      "grad_norm": 1.2954068183898926,
+      "learning_rate": 9.083807275988284e-05,
+      "loss": 0.2394,
+      "step": 290
+    },
+    {
+      "epoch": 1.7057569296375266,
+      "grad_norm": 3.02253794670105,
+      "learning_rate": 8.985662536114613e-05,
+      "loss": 0.2307,
+      "step": 300
+    },
+    {
+      "epoch": 1.7626154939587777,
+      "grad_norm": 1.063110589981079,
+      "learning_rate": 8.883113295834892e-05,
+      "loss": 0.2172,
+      "step": 310
+    },
+    {
+      "epoch": 1.8194740582800284,
+      "grad_norm": 1.7286932468414307,
+      "learning_rate": 8.776272880892675e-05,
+      "loss": 0.2251,
+      "step": 320
+    },
+    {
+      "epoch": 1.8763326226012793,
+      "grad_norm": 1.194043755531311,
+      "learning_rate": 8.665259359149132e-05,
+      "loss": 0.2435,
+      "step": 330
+    },
+    {
+      "epoch": 1.9331911869225302,
+      "grad_norm": 0.8665173649787903,
+      "learning_rate": 8.550195410107902e-05,
+      "loss": 0.2425,
+      "step": 340
+    },
+    {
+      "epoch": 1.9900497512437811,
+      "grad_norm": 1.0586192607879639,
+      "learning_rate": 8.43120818934367e-05,
+      "loss": 0.2297,
+      "step": 350
+    },
+    {
+      "epoch": 1.9900497512437811,
+      "eval_accuracy": 0.9031333333333333,
+      "eval_loss": 0.24609853327274323,
+      "eval_runtime": 206.1828,
+      "eval_samples_per_second": 12.125,
+      "eval_steps_per_second": 12.125,
+      "step": 350
+    },
+    {
+      "epoch": 2.046908315565032,
+      "grad_norm": 1.062510371208191,
+      "learning_rate": 8.308429187984297e-05,
+      "loss": 0.1928,
+      "step": 360
+    },
+    {
+      "epoch": 2.1037668798862827,
+      "grad_norm": 1.1897845268249512,
+      "learning_rate": 8.181994087401819e-05,
+      "loss": 0.194,
+      "step": 370
+    },
+    {
+      "epoch": 2.160625444207534,
+      "grad_norm": 1.3862345218658447,
+      "learning_rate": 8.052042609272817e-05,
+      "loss": 0.1699,
+      "step": 380
+    },
+    {
+      "epoch": 2.2174840085287846,
+      "grad_norm": 0.9099957346916199,
+      "learning_rate": 7.91871836117395e-05,
+      "loss": 0.186,
+      "step": 390
+    },
+    {
+      "epoch": 2.2743425728500357,
+      "grad_norm": 1.0390052795410156,
+      "learning_rate": 7.782168677883206e-05,
+      "loss": 0.1711,
+      "step": 400
+    },
+    {
+      "epoch": 2.3312011371712864,
+      "grad_norm": 2.652061700820923,
+      "learning_rate": 7.642544458562278e-05,
+      "loss": 0.2012,
+      "step": 410
+    },
+    {
+      "epoch": 2.388059701492537,
+      "grad_norm": 1.5696091651916504,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.1898,
+      "step": 420
+    },
+    {
+      "epoch": 2.4449182658137882,
+      "grad_norm": 0.8261299729347229,
+      "learning_rate": 7.354692826101102e-05,
+      "loss": 0.2032,
+      "step": 430
+    },
+    {
+      "epoch": 2.501776830135039,
+      "grad_norm": 1.252537727355957,
+      "learning_rate": 7.20678351380872e-05,
+      "loss": 0.1854,
+      "step": 440
+    },
+    {
+      "epoch": 2.55863539445629,
+      "grad_norm": 1.3897773027420044,
+      "learning_rate": 7.056435515653059e-05,
+      "loss": 0.202,
+      "step": 450
+    },
+    {
+      "epoch": 2.6154939587775408,
+      "grad_norm": 0.8310902714729309,
+      "learning_rate": 6.903814979122249e-05,
+      "loss": 0.1789,
+      "step": 460
+    },
+    {
+      "epoch": 2.672352523098792,
+      "grad_norm": 1.3531970977783203,
+      "learning_rate": 6.749090563055076e-05,
+      "loss": 0.1969,
+      "step": 470
+    },
+    {
+      "epoch": 2.7292110874200426,
+      "grad_norm": 1.0367170572280884,
+      "learning_rate": 6.592433251258423e-05,
+      "loss": 0.1967,
+      "step": 480
+    },
+    {
+      "epoch": 2.7860696517412933,
+      "grad_norm": 1.2085515260696411,
+      "learning_rate": 6.434016163555452e-05,
+      "loss": 0.1825,
+      "step": 490
+    },
+    {
+      "epoch": 2.8429282160625444,
+      "grad_norm": 1.2390882968902588,
+      "learning_rate": 6.274014364473274e-05,
+      "loss": 0.212,
+      "step": 500
+    },
+    {
+      "epoch": 2.8997867803837956,
+      "grad_norm": 1.4294568300247192,
+      "learning_rate": 6.112604669781572e-05,
+      "loss": 0.191,
+      "step": 510
+    },
+    {
+      "epoch": 2.9566453447050463,
+      "grad_norm": 1.522834300994873,
+      "learning_rate": 5.949965451095951e-05,
+      "loss": 0.1792,
+      "step": 520
+    },
+    {
+      "epoch": 2.9850746268656714,
+      "eval_accuracy": 0.9013333333333332,
+      "eval_loss": 0.25212058424949646,
+      "eval_runtime": 206.7127,
+      "eval_samples_per_second": 12.094,
+      "eval_steps_per_second": 12.094,
+      "step": 525
+    },
+    {
+      "epoch": 3.013503909026297,
+      "grad_norm": 0.655222475528717,
+      "learning_rate": 5.786276438761927e-05,
+      "loss": 0.1592,
+      "step": 530
+    },
+    {
+      "epoch": 3.070362473347548,
+      "grad_norm": 0.8747724294662476,
+      "learning_rate": 5.621718523237427e-05,
+      "loss": 0.1186,
+      "step": 540
+    },
+    {
+      "epoch": 3.1272210376687988,
+      "grad_norm": 1.3602584600448608,
+      "learning_rate": 5.456473555193242e-05,
+      "loss": 0.1195,
+      "step": 550
+    },
+    {
+      "epoch": 3.18407960199005,
+      "grad_norm": 1.0027141571044922,
+      "learning_rate": 5.290724144552379e-05,
+      "loss": 0.144,
+      "step": 560
+    },
+    {
+      "epoch": 3.2409381663113006,
+      "grad_norm": 2.7845115661621094,
+      "learning_rate": 5.124653458690365e-05,
+      "loss": 0.1508,
+      "step": 570
+    },
+    {
+      "epoch": 3.2977967306325517,
+      "grad_norm": 0.7762941718101501,
+      "learning_rate": 4.9584450200195156e-05,
+      "loss": 0.1399,
+      "step": 580
+    },
+    {
+      "epoch": 3.3546552949538024,
+      "grad_norm": 0.9395729303359985,
+      "learning_rate": 4.792282503180867e-05,
+      "loss": 0.1321,
+      "step": 590
+    },
+    {
+      "epoch": 3.411513859275053,
+      "grad_norm": 0.9491863250732422,
+      "learning_rate": 4.626349532067879e-05,
+      "loss": 0.1245,
+      "step": 600
+    },
+    {
+      "epoch": 3.4683724235963043,
+      "grad_norm": 1.0270098447799683,
+      "learning_rate": 4.4608294769062075e-05,
+      "loss": 0.1211,
+      "step": 610
+    },
+    {
+      "epoch": 3.525230987917555,
+      "grad_norm": 1.0683083534240723,
+      "learning_rate": 4.295905251613817e-05,
+      "loss": 0.1336,
+      "step": 620
+    },
+    {
+      "epoch": 3.582089552238806,
+      "grad_norm": 1.0028445720672607,
+      "learning_rate": 4.131759111665349e-05,
+      "loss": 0.1379,
+      "step": 630
+    },
+    {
+      "epoch": 3.638948116560057,
+      "grad_norm": 0.6078014969825745,
+      "learning_rate": 3.968572452684113e-05,
+      "loss": 0.1248,
+      "step": 640
+    },
+    {
+      "epoch": 3.695806680881308,
+      "grad_norm": 1.0846917629241943,
+      "learning_rate": 3.806525609984312e-05,
+      "loss": 0.1244,
+      "step": 650
+    },
+    {
+      "epoch": 3.7526652452025586,
+      "grad_norm": 1.1056978702545166,
+      "learning_rate": 3.6457976592849754e-05,
+      "loss": 0.1455,
+      "step": 660
+    },
+    {
+      "epoch": 3.8095238095238093,
+      "grad_norm": 0.8466194868087769,
+      "learning_rate": 3.486566218815871e-05,
+      "loss": 0.1381,
+      "step": 670
+    },
+    {
+      "epoch": 3.8663823738450604,
+      "grad_norm": 1.3114445209503174,
+      "learning_rate": 3.329007253034063e-05,
+      "loss": 0.1278,
+      "step": 680
+    },
+    {
+      "epoch": 3.923240938166311,
+      "grad_norm": 1.2287284135818481,
+      "learning_rate": 3.173294878168025e-05,
+      "loss": 0.1487,
+      "step": 690
+    },
+    {
+      "epoch": 3.9800995024875623,
+      "grad_norm": 0.7370882630348206,
+      "learning_rate": 3.019601169804216e-05,
+      "loss": 0.122,
+      "step": 700
+    },
+    {
+      "epoch": 3.9800995024875623,
+      "eval_accuracy": 0.9049666666666666,
+      "eval_loss": 0.27484622597694397,
+      "eval_runtime": 207.5459,
+      "eval_samples_per_second": 12.046,
+      "eval_steps_per_second": 12.046,
+      "step": 700
+    },
+    {
+      "epoch": 4.036958066808813,
+      "grad_norm": 0.7140550017356873,
+      "learning_rate": 2.8680959727287317e-05,
+      "loss": 0.1074,
+      "step": 710
+    },
+    {
+      "epoch": 4.093816631130064,
+      "grad_norm": 1.042784333229065,
+      "learning_rate": 2.718946713234185e-05,
+      "loss": 0.0856,
+      "step": 720
+    },
+    {
+      "epoch": 4.150675195451315,
+      "grad_norm": 0.7461434602737427,
+      "learning_rate": 2.5723182140992387e-05,
+      "loss": 0.0837,
+      "step": 730
+    },
+    {
+      "epoch": 4.2075337597725655,
+      "grad_norm": 1.004974603652954,
+      "learning_rate": 2.428372512445233e-05,
+      "loss": 0.0749,
+      "step": 740
+    },
+    {
+      "epoch": 4.264392324093817,
+      "grad_norm": 0.6601787805557251,
+      "learning_rate": 2.2872686806712035e-05,
+      "loss": 0.0835,
+      "step": 750
+    },
+    {
+      "epoch": 4.321250888415068,
+      "grad_norm": 0.6331136226654053,
+      "learning_rate": 2.1491626506651914e-05,
+      "loss": 0.0685,
+      "step": 760
+    },
+    {
+      "epoch": 4.378109452736318,
+      "grad_norm": 0.8726811408996582,
+      "learning_rate": 2.0142070414860704e-05,
+      "loss": 0.0787,
+      "step": 770
+    },
+    {
+      "epoch": 4.434968017057569,
+      "grad_norm": 0.8191830515861511,
+      "learning_rate": 1.8825509907063327e-05,
+      "loss": 0.0783,
+      "step": 780
+    },
+    {
+      "epoch": 4.49182658137882,
+      "grad_norm": 0.6537315249443054,
+      "learning_rate": 1.7543399896022405e-05,
+      "loss": 0.0759,
+      "step": 790
+    },
+    {
+      "epoch": 4.548685145700071,
+      "grad_norm": 1.3562214374542236,
+      "learning_rate": 1.629715722373423e-05,
+      "loss": 0.0952,
+      "step": 800
+    },
+    {
+      "epoch": 4.605543710021322,
+      "grad_norm": 1.0490374565124512,
+      "learning_rate": 1.5088159095696363e-05,
+      "loss": 0.0916,
+      "step": 810
+    },
+    {
+      "epoch": 4.662402274342573,
+      "grad_norm": 0.8466753363609314,
+      "learning_rate": 1.3917741558976894e-05,
+      "loss": 0.0874,
+      "step": 820
+    },
+    {
+      "epoch": 4.719260838663824,
+      "grad_norm": 0.9645017385482788,
+      "learning_rate": 1.2787198025767416e-05,
+      "loss": 0.0879,
+      "step": 830
+    },
+    {
+      "epoch": 4.776119402985074,
+      "grad_norm": 3.8121776580810547,
+      "learning_rate": 1.1697777844051105e-05,
+      "loss": 0.0697,
+      "step": 840
+    },
+    {
+      "epoch": 4.832977967306325,
+      "grad_norm": 1.1398985385894775,
+      "learning_rate": 1.0650684916965559e-05,
+      "loss": 0.0873,
+      "step": 850
+    },
+    {
+      "epoch": 4.8898365316275765,
+      "grad_norm": 1.19906485080719,
+      "learning_rate": 9.647076372386194e-06,
+      "loss": 0.0826,
+      "step": 860
+    },
+    {
+      "epoch": 4.946695095948828,
+      "grad_norm": 0.7644526958465576,
+      "learning_rate": 8.688061284200266e-06,
+      "loss": 0.073,
+      "step": 870
+    },
+    {
+      "epoch": 4.975124378109452,
+      "eval_accuracy": 0.9083,
+      "eval_loss": 0.33086976408958435,
+      "eval_runtime": 206.4018,
+      "eval_samples_per_second": 12.112,
+      "eval_steps_per_second": 12.112,
+      "step": 875
+    },
+    {
+      "epoch": 5.003553660270078,
+      "grad_norm": 0.686549723148346,
+      "learning_rate": 7.774699446684608e-06,
+      "loss": 0.082,
+      "step": 880
+    },
+    {
+      "epoch": 5.060412224591329,
+      "grad_norm": 0.5072228312492371,
+      "learning_rate": 6.908000203341802e-06,
+      "loss": 0.0516,
+      "step": 890
+    },
+    {
+      "epoch": 5.11727078891258,
+      "grad_norm": 0.5079275965690613,
+      "learning_rate": 6.088921331488568e-06,
+      "loss": 0.0607,
+      "step": 900
+    },
+    {
+      "epoch": 5.174129353233831,
+      "grad_norm": 0.8684201240539551,
+      "learning_rate": 5.318367983829392e-06,
+      "loss": 0.0451,
+      "step": 910
+    },
+    {
+      "epoch": 5.2309879175550815,
+      "grad_norm": 0.822227954864502,
+      "learning_rate": 4.597191688184754e-06,
+      "loss": 0.0516,
+      "step": 920
+    },
+    {
+      "epoch": 5.287846481876333,
+      "grad_norm": 0.5135931968688965,
+      "learning_rate": 3.9261894064796135e-06,
+      "loss": 0.0479,
+      "step": 930
+    },
+    {
+      "epoch": 5.344705046197584,
+      "grad_norm": 0.5866013765335083,
+      "learning_rate": 3.306102654031823e-06,
+      "loss": 0.0547,
+      "step": 940
+    },
+    {
+      "epoch": 5.401563610518834,
+      "grad_norm": 1.5747359991073608,
+      "learning_rate": 2.737616680113758e-06,
+      "loss": 0.0555,
+      "step": 950
+    },
+    {
+      "epoch": 5.458422174840085,
+      "grad_norm": 2.3203916549682617,
+      "learning_rate": 2.221359710692961e-06,
+      "loss": 0.0495,
+      "step": 960
+    },
+    {
+      "epoch": 5.515280739161336,
+      "grad_norm": 0.6227638721466064,
+      "learning_rate": 1.757902254188254e-06,
+      "loss": 0.0522,
+      "step": 970
+    },
+    {
+      "epoch": 5.572139303482587,
+      "grad_norm": 1.4704910516738892,
+      "learning_rate": 1.3477564710088098e-06,
+      "loss": 0.0536,
+      "step": 980
+    },
+    {
+      "epoch": 5.628997867803838,
+      "grad_norm": 0.4436222314834595,
+      "learning_rate": 9.913756075728087e-07,
+      "loss": 0.0518,
+      "step": 990
+    },
+    {
+      "epoch": 5.685856432125089,
+      "grad_norm": 0.6575887799263,
+      "learning_rate": 6.891534954310885e-07,
+      "loss": 0.0491,
+      "step": 1000
+    },
+    {
+      "epoch": 5.74271499644634,
+      "grad_norm": 0.5118672251701355,
+      "learning_rate": 4.4142411604936597e-07,
+      "loss": 0.0355,
+      "step": 1010
+    },
+    {
+      "epoch": 5.79957356076759,
+      "grad_norm": 0.8050460815429688,
+      "learning_rate": 2.4846123172992954e-07,
+      "loss": 0.0425,
+      "step": 1020
+    },
+    {
+      "epoch": 5.856432125088841,
+      "grad_norm": 0.6702028512954712,
+      "learning_rate": 1.1047808308075058e-07,
+      "loss": 0.0553,
+      "step": 1030
+    },
+    {
+      "epoch": 5.9132906894100925,
+      "grad_norm": 0.4666765630245209,
+      "learning_rate": 2.7627153366222013e-08,
+      "loss": 0.061,
+      "step": 1040
+    },
+    {
+      "epoch": 5.970149253731344,
+      "grad_norm": 0.5390927195549011,
+      "learning_rate": 0.0,
+      "loss": 0.0484,
+      "step": 1050
+    },
+    {
+      "epoch": 5.970149253731344,
+      "eval_accuracy": 0.9057333333333334,
+      "eval_loss": 0.43139833211898804,
+      "eval_runtime": 206.2708,
+      "eval_samples_per_second": 12.12,
+      "eval_steps_per_second": 12.12,
+      "step": 1050
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1050,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 6,
+  "save_steps": 175,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": true
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.0511582264790876e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1050/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63f73e90c9f7ea09d67451fc2a878fff87fc5533353fc1d1514ce97ce70cb436
+size 5304

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: shenzhi-wang/Llama3-8B-Chinese-Chat
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "shenzhi-wang/Llama3-8B-Chinese-Chat",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aba120128a8f19dd591b355f8778dfcc44f93d783f4d936f0dc10bae147900ef
+size 83945296

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb36f97bed4e500a58f4956dacbdc1dd1a6023889bcdbf01677465b362ca7a79
+size 168149394

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8d3b7102895eb0637b0cab516bd672f216b2bf79078a83eb301011a90444f44c
+size 14244

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:576afbb73e5a87f7092900be7c7e883ad3337a235de39853353bfeba08bc5108
+size 1064

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/tokenizer_config.json

@@ -0,0 +1,2065 @@
+{
+  "added_tokens_decoder": {
+    "128000": {
+      "content": "<|begin_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128001": {
+      "content": "<|end_of_text|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128002": {
+      "content": "<|reserved_special_token_0|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128003": {
+      "content": "<|reserved_special_token_1|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128004": {
+      "content": "<|reserved_special_token_2|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128005": {
+      "content": "<|reserved_special_token_3|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128006": {
+      "content": "<|start_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128007": {
+      "content": "<|end_header_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128008": {
+      "content": "<|reserved_special_token_4|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128009": {
+      "content": "<|eot_id|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128010": {
+      "content": "<|reserved_special_token_5|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128011": {
+      "content": "<|reserved_special_token_6|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128012": {
+      "content": "<|reserved_special_token_7|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128013": {
+      "content": "<|reserved_special_token_8|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128014": {
+      "content": "<|reserved_special_token_9|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128015": {
+      "content": "<|reserved_special_token_10|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128016": {
+      "content": "<|reserved_special_token_11|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128017": {
+      "content": "<|reserved_special_token_12|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128018": {
+      "content": "<|reserved_special_token_13|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128019": {
+      "content": "<|reserved_special_token_14|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128020": {
+      "content": "<|reserved_special_token_15|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128021": {
+      "content": "<|reserved_special_token_16|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128022": {
+      "content": "<|reserved_special_token_17|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128023": {
+      "content": "<|reserved_special_token_18|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128024": {
+      "content": "<|reserved_special_token_19|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128025": {
+      "content": "<|reserved_special_token_20|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128026": {
+      "content": "<|reserved_special_token_21|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128027": {
+      "content": "<|reserved_special_token_22|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128028": {
+      "content": "<|reserved_special_token_23|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128029": {
+      "content": "<|reserved_special_token_24|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128030": {
+      "content": "<|reserved_special_token_25|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128031": {
+      "content": "<|reserved_special_token_26|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128032": {
+      "content": "<|reserved_special_token_27|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128033": {
+      "content": "<|reserved_special_token_28|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128034": {
+      "content": "<|reserved_special_token_29|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128035": {
+      "content": "<|reserved_special_token_30|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128036": {
+      "content": "<|reserved_special_token_31|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128037": {
+      "content": "<|reserved_special_token_32|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128038": {
+      "content": "<|reserved_special_token_33|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128039": {
+      "content": "<|reserved_special_token_34|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128040": {
+      "content": "<|reserved_special_token_35|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128041": {
+      "content": "<|reserved_special_token_36|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128042": {
+      "content": "<|reserved_special_token_37|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128043": {
+      "content": "<|reserved_special_token_38|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128044": {
+      "content": "<|reserved_special_token_39|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128045": {
+      "content": "<|reserved_special_token_40|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128046": {
+      "content": "<|reserved_special_token_41|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128047": {
+      "content": "<|reserved_special_token_42|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128048": {
+      "content": "<|reserved_special_token_43|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128049": {
+      "content": "<|reserved_special_token_44|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128050": {
+      "content": "<|reserved_special_token_45|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128051": {
+      "content": "<|reserved_special_token_46|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128052": {
+      "content": "<|reserved_special_token_47|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128053": {
+      "content": "<|reserved_special_token_48|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128054": {
+      "content": "<|reserved_special_token_49|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128055": {
+      "content": "<|reserved_special_token_50|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128056": {
+      "content": "<|reserved_special_token_51|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128057": {
+      "content": "<|reserved_special_token_52|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128058": {
+      "content": "<|reserved_special_token_53|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128059": {
+      "content": "<|reserved_special_token_54|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128060": {
+      "content": "<|reserved_special_token_55|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128061": {
+      "content": "<|reserved_special_token_56|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128062": {
+      "content": "<|reserved_special_token_57|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128063": {
+      "content": "<|reserved_special_token_58|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128064": {
+      "content": "<|reserved_special_token_59|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128065": {
+      "content": "<|reserved_special_token_60|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128066": {
+      "content": "<|reserved_special_token_61|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128067": {
+      "content": "<|reserved_special_token_62|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128068": {
+      "content": "<|reserved_special_token_63|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128069": {
+      "content": "<|reserved_special_token_64|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128070": {
+      "content": "<|reserved_special_token_65|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128071": {
+      "content": "<|reserved_special_token_66|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128072": {
+      "content": "<|reserved_special_token_67|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128073": {
+      "content": "<|reserved_special_token_68|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128074": {
+      "content": "<|reserved_special_token_69|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128075": {
+      "content": "<|reserved_special_token_70|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128076": {
+      "content": "<|reserved_special_token_71|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128077": {
+      "content": "<|reserved_special_token_72|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128078": {
+      "content": "<|reserved_special_token_73|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128079": {
+      "content": "<|reserved_special_token_74|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128080": {
+      "content": "<|reserved_special_token_75|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128081": {
+      "content": "<|reserved_special_token_76|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128082": {
+      "content": "<|reserved_special_token_77|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128083": {
+      "content": "<|reserved_special_token_78|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128084": {
+      "content": "<|reserved_special_token_79|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128085": {
+      "content": "<|reserved_special_token_80|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128086": {
+      "content": "<|reserved_special_token_81|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128087": {
+      "content": "<|reserved_special_token_82|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128088": {
+      "content": "<|reserved_special_token_83|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128089": {
+      "content": "<|reserved_special_token_84|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128090": {
+      "content": "<|reserved_special_token_85|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128091": {
+      "content": "<|reserved_special_token_86|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128092": {
+      "content": "<|reserved_special_token_87|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128093": {
+      "content": "<|reserved_special_token_88|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128094": {
+      "content": "<|reserved_special_token_89|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128095": {
+      "content": "<|reserved_special_token_90|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128096": {
+      "content": "<|reserved_special_token_91|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128097": {
+      "content": "<|reserved_special_token_92|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128098": {
+      "content": "<|reserved_special_token_93|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128099": {
+      "content": "<|reserved_special_token_94|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128100": {
+      "content": "<|reserved_special_token_95|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128101": {
+      "content": "<|reserved_special_token_96|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128102": {
+      "content": "<|reserved_special_token_97|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128103": {
+      "content": "<|reserved_special_token_98|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128104": {
+      "content": "<|reserved_special_token_99|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128105": {
+      "content": "<|reserved_special_token_100|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128106": {
+      "content": "<|reserved_special_token_101|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128107": {
+      "content": "<|reserved_special_token_102|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128108": {
+      "content": "<|reserved_special_token_103|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128109": {
+      "content": "<|reserved_special_token_104|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128110": {
+      "content": "<|reserved_special_token_105|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128111": {
+      "content": "<|reserved_special_token_106|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128112": {
+      "content": "<|reserved_special_token_107|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128113": {
+      "content": "<|reserved_special_token_108|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128114": {
+      "content": "<|reserved_special_token_109|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128115": {
+      "content": "<|reserved_special_token_110|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128116": {
+      "content": "<|reserved_special_token_111|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128117": {
+      "content": "<|reserved_special_token_112|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128118": {
+      "content": "<|reserved_special_token_113|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128119": {
+      "content": "<|reserved_special_token_114|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128120": {
+      "content": "<|reserved_special_token_115|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128121": {
+      "content": "<|reserved_special_token_116|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128122": {
+      "content": "<|reserved_special_token_117|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128123": {
+      "content": "<|reserved_special_token_118|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128124": {
+      "content": "<|reserved_special_token_119|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128125": {
+      "content": "<|reserved_special_token_120|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128126": {
+      "content": "<|reserved_special_token_121|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128127": {
+      "content": "<|reserved_special_token_122|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128128": {
+      "content": "<|reserved_special_token_123|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128129": {
+      "content": "<|reserved_special_token_124|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128130": {
+      "content": "<|reserved_special_token_125|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128131": {
+      "content": "<|reserved_special_token_126|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128132": {
+      "content": "<|reserved_special_token_127|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128133": {
+      "content": "<|reserved_special_token_128|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128134": {
+      "content": "<|reserved_special_token_129|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128135": {
+      "content": "<|reserved_special_token_130|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128136": {
+      "content": "<|reserved_special_token_131|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128137": {
+      "content": "<|reserved_special_token_132|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128138": {
+      "content": "<|reserved_special_token_133|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128139": {
+      "content": "<|reserved_special_token_134|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128140": {
+      "content": "<|reserved_special_token_135|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128141": {
+      "content": "<|reserved_special_token_136|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128142": {
+      "content": "<|reserved_special_token_137|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128143": {
+      "content": "<|reserved_special_token_138|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128144": {
+      "content": "<|reserved_special_token_139|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128145": {
+      "content": "<|reserved_special_token_140|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128146": {
+      "content": "<|reserved_special_token_141|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128147": {
+      "content": "<|reserved_special_token_142|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128148": {
+      "content": "<|reserved_special_token_143|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128149": {
+      "content": "<|reserved_special_token_144|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128150": {
+      "content": "<|reserved_special_token_145|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128151": {
+      "content": "<|reserved_special_token_146|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128152": {
+      "content": "<|reserved_special_token_147|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128153": {
+      "content": "<|reserved_special_token_148|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128154": {
+      "content": "<|reserved_special_token_149|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128155": {
+      "content": "<|reserved_special_token_150|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128156": {
+      "content": "<|reserved_special_token_151|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128157": {
+      "content": "<|reserved_special_token_152|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128158": {
+      "content": "<|reserved_special_token_153|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128159": {
+      "content": "<|reserved_special_token_154|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128160": {
+      "content": "<|reserved_special_token_155|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128161": {
+      "content": "<|reserved_special_token_156|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128162": {
+      "content": "<|reserved_special_token_157|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128163": {
+      "content": "<|reserved_special_token_158|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128164": {
+      "content": "<|reserved_special_token_159|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128165": {
+      "content": "<|reserved_special_token_160|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128166": {
+      "content": "<|reserved_special_token_161|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128167": {
+      "content": "<|reserved_special_token_162|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128168": {
+      "content": "<|reserved_special_token_163|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128169": {
+      "content": "<|reserved_special_token_164|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128170": {
+      "content": "<|reserved_special_token_165|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128171": {
+      "content": "<|reserved_special_token_166|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128172": {
+      "content": "<|reserved_special_token_167|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128173": {
+      "content": "<|reserved_special_token_168|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128174": {
+      "content": "<|reserved_special_token_169|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128175": {
+      "content": "<|reserved_special_token_170|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128176": {
+      "content": "<|reserved_special_token_171|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128177": {
+      "content": "<|reserved_special_token_172|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128178": {
+      "content": "<|reserved_special_token_173|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128179": {
+      "content": "<|reserved_special_token_174|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128180": {
+      "content": "<|reserved_special_token_175|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128181": {
+      "content": "<|reserved_special_token_176|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128182": {
+      "content": "<|reserved_special_token_177|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128183": {
+      "content": "<|reserved_special_token_178|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128184": {
+      "content": "<|reserved_special_token_179|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128185": {
+      "content": "<|reserved_special_token_180|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128186": {
+      "content": "<|reserved_special_token_181|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128187": {
+      "content": "<|reserved_special_token_182|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128188": {
+      "content": "<|reserved_special_token_183|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128189": {
+      "content": "<|reserved_special_token_184|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128190": {
+      "content": "<|reserved_special_token_185|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128191": {
+      "content": "<|reserved_special_token_186|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128192": {
+      "content": "<|reserved_special_token_187|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128193": {
+      "content": "<|reserved_special_token_188|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128194": {
+      "content": "<|reserved_special_token_189|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128195": {
+      "content": "<|reserved_special_token_190|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128196": {
+      "content": "<|reserved_special_token_191|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128197": {
+      "content": "<|reserved_special_token_192|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128198": {
+      "content": "<|reserved_special_token_193|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128199": {
+      "content": "<|reserved_special_token_194|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128200": {
+      "content": "<|reserved_special_token_195|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128201": {
+      "content": "<|reserved_special_token_196|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128202": {
+      "content": "<|reserved_special_token_197|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128203": {
+      "content": "<|reserved_special_token_198|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128204": {
+      "content": "<|reserved_special_token_199|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128205": {
+      "content": "<|reserved_special_token_200|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128206": {
+      "content": "<|reserved_special_token_201|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128207": {
+      "content": "<|reserved_special_token_202|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128208": {
+      "content": "<|reserved_special_token_203|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128209": {
+      "content": "<|reserved_special_token_204|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128210": {
+      "content": "<|reserved_special_token_205|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128211": {
+      "content": "<|reserved_special_token_206|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128212": {
+      "content": "<|reserved_special_token_207|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128213": {
+      "content": "<|reserved_special_token_208|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128214": {
+      "content": "<|reserved_special_token_209|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128215": {
+      "content": "<|reserved_special_token_210|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128216": {
+      "content": "<|reserved_special_token_211|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128217": {
+      "content": "<|reserved_special_token_212|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128218": {
+      "content": "<|reserved_special_token_213|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128219": {
+      "content": "<|reserved_special_token_214|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128220": {
+      "content": "<|reserved_special_token_215|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128221": {
+      "content": "<|reserved_special_token_216|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128222": {
+      "content": "<|reserved_special_token_217|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128223": {
+      "content": "<|reserved_special_token_218|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128224": {
+      "content": "<|reserved_special_token_219|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128225": {
+      "content": "<|reserved_special_token_220|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128226": {
+      "content": "<|reserved_special_token_221|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128227": {
+      "content": "<|reserved_special_token_222|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128228": {
+      "content": "<|reserved_special_token_223|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128229": {
+      "content": "<|reserved_special_token_224|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128230": {
+      "content": "<|reserved_special_token_225|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128231": {
+      "content": "<|reserved_special_token_226|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128232": {
+      "content": "<|reserved_special_token_227|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128233": {
+      "content": "<|reserved_special_token_228|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128234": {
+      "content": "<|reserved_special_token_229|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128235": {
+      "content": "<|reserved_special_token_230|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128236": {
+      "content": "<|reserved_special_token_231|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128237": {
+      "content": "<|reserved_special_token_232|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128238": {
+      "content": "<|reserved_special_token_233|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128239": {
+      "content": "<|reserved_special_token_234|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128240": {
+      "content": "<|reserved_special_token_235|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128241": {
+      "content": "<|reserved_special_token_236|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128242": {
+      "content": "<|reserved_special_token_237|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128243": {
+      "content": "<|reserved_special_token_238|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128244": {
+      "content": "<|reserved_special_token_239|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128245": {
+      "content": "<|reserved_special_token_240|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128246": {
+      "content": "<|reserved_special_token_241|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128247": {
+      "content": "<|reserved_special_token_242|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128248": {
+      "content": "<|reserved_special_token_243|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128249": {
+      "content": "<|reserved_special_token_244|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128250": {
+      "content": "<|reserved_special_token_245|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128251": {
+      "content": "<|reserved_special_token_246|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128252": {
+      "content": "<|reserved_special_token_247|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128253": {
+      "content": "<|reserved_special_token_248|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128254": {
+      "content": "<|reserved_special_token_249|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    },
+    "128255": {
+      "content": "<|reserved_special_token_250|>",
+      "lstrip": false,
+      "normalized": false,
+      "rstrip": false,
+      "single_word": false,
+      "special": true
+    }
+  },
+  "bos_token": "<|begin_of_text|>",
+  "chat_template": "{{ '<|begin_of_text|>' }}{% if messages[0]['role'] == 'system' %}{% set system_message = messages[0]['content'] %}{% endif %}{% if system_message is defined %}{{ '<|start_header_id|>system<|end_header_id|>\n\n' + system_message + '<|eot_id|>' }}{% endif %}{% for message in messages %}{% set content = message['content'] %}{% if message['role'] == 'user' %}{{ '<|start_header_id|>user<|end_header_id|>\n\n' + content + '<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n' }}{% elif message['role'] == 'assistant' %}{{ content + '<|eot_id|>' }}{% endif %}{% endfor %}",
+  "clean_up_tokenization_spaces": true,
+  "eos_token": "<|eot_id|>",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<|eot_id|>",
+  "padding_side": "right",
+  "split_special_tokens": false,
+  "tokenizer_class": "PreTrainedTokenizerFast"
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/trainer_state.json

@@ -0,0 +1,950 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 6.9950248756218905,
+  "eval_steps": 175,
+  "global_step": 1225,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 0.05685856432125089,
+      "grad_norm": 1.478421688079834,
+      "learning_rate": 9.523809523809523e-06,
+      "loss": 0.9166,
+      "step": 10
+    },
+    {
+      "epoch": 0.11371712864250177,
+      "grad_norm": 0.9236305356025696,
+      "learning_rate": 1.9047619047619046e-05,
+      "loss": 0.3687,
+      "step": 20
+    },
+    {
+      "epoch": 0.17057569296375266,
+      "grad_norm": 0.6159961223602295,
+      "learning_rate": 2.857142857142857e-05,
+      "loss": 0.328,
+      "step": 30
+    },
+    {
+      "epoch": 0.22743425728500355,
+      "grad_norm": 1.8057916164398193,
+      "learning_rate": 3.809523809523809e-05,
+      "loss": 0.3049,
+      "step": 40
+    },
+    {
+      "epoch": 0.28429282160625446,
+      "grad_norm": 1.5811758041381836,
+      "learning_rate": 4.761904761904762e-05,
+      "loss": 0.2999,
+      "step": 50
+    },
+    {
+      "epoch": 0.3411513859275053,
+      "grad_norm": 1.9880269765853882,
+      "learning_rate": 5.714285714285714e-05,
+      "loss": 0.2951,
+      "step": 60
+    },
+    {
+      "epoch": 0.39800995024875624,
+      "grad_norm": 1.5736329555511475,
+      "learning_rate": 6.666666666666667e-05,
+      "loss": 0.282,
+      "step": 70
+    },
+    {
+      "epoch": 0.4548685145700071,
+      "grad_norm": 1.3553781509399414,
+      "learning_rate": 7.619047619047618e-05,
+      "loss": 0.2805,
+      "step": 80
+    },
+    {
+      "epoch": 0.511727078891258,
+      "grad_norm": 1.3975526094436646,
+      "learning_rate": 8.571428571428571e-05,
+      "loss": 0.2717,
+      "step": 90
+    },
+    {
+      "epoch": 0.5685856432125089,
+      "grad_norm": 1.0928113460540771,
+      "learning_rate": 9.523809523809524e-05,
+      "loss": 0.2659,
+      "step": 100
+    },
+    {
+      "epoch": 0.6254442075337597,
+      "grad_norm": 1.163079857826233,
+      "learning_rate": 9.999309273455528e-05,
+      "loss": 0.27,
+      "step": 110
+    },
+    {
+      "epoch": 0.6823027718550106,
+      "grad_norm": 0.849062979221344,
+      "learning_rate": 9.993784606094612e-05,
+      "loss": 0.2671,
+      "step": 120
+    },
+    {
+      "epoch": 0.7391613361762616,
+      "grad_norm": 0.8933653831481934,
+      "learning_rate": 9.982741376606078e-05,
+      "loss": 0.2869,
+      "step": 130
+    },
+    {
+      "epoch": 0.7960199004975125,
+      "grad_norm": 1.046842098236084,
+      "learning_rate": 9.966191788709716e-05,
+      "loss": 0.2997,
+      "step": 140
+    },
+    {
+      "epoch": 0.8528784648187633,
+      "grad_norm": 2.4692931175231934,
+      "learning_rate": 9.944154131125642e-05,
+      "loss": 0.2735,
+      "step": 150
+    },
+    {
+      "epoch": 0.9097370291400142,
+      "grad_norm": 1.4990626573562622,
+      "learning_rate": 9.916652757363698e-05,
+      "loss": 0.2905,
+      "step": 160
+    },
+    {
+      "epoch": 0.9665955934612651,
+      "grad_norm": 1.4853521585464478,
+      "learning_rate": 9.883718058810707e-05,
+      "loss": 0.258,
+      "step": 170
+    },
+    {
+      "epoch": 0.9950248756218906,
+      "eval_accuracy": 0.909,
+      "eval_loss": 0.24650488793849945,
+      "eval_runtime": 206.2691,
+      "eval_samples_per_second": 12.12,
+      "eval_steps_per_second": 12.12,
+      "step": 175
+    },
+    {
+      "epoch": 1.023454157782516,
+      "grad_norm": 0.8761231303215027,
+      "learning_rate": 9.84538643114539e-05,
+      "loss": 0.2471,
+      "step": 180
+    },
+    {
+      "epoch": 1.080312722103767,
+      "grad_norm": 1.037913203239441,
+      "learning_rate": 9.801700234117999e-05,
+      "loss": 0.2304,
+      "step": 190
+    },
+    {
+      "epoch": 1.1371712864250179,
+      "grad_norm": 0.5963209867477417,
+      "learning_rate": 9.752707744739145e-05,
+      "loss": 0.2414,
+      "step": 200
+    },
+    {
+      "epoch": 1.1940298507462686,
+      "grad_norm": 0.8125211596488953,
+      "learning_rate": 9.698463103929542e-05,
+      "loss": 0.2276,
+      "step": 210
+    },
+    {
+      "epoch": 1.2508884150675195,
+      "grad_norm": 2.1267828941345215,
+      "learning_rate": 9.639026256689628e-05,
+      "loss": 0.2137,
+      "step": 220
+    },
+    {
+      "epoch": 1.3077469793887704,
+      "grad_norm": 3.7273290157318115,
+      "learning_rate": 9.574462885855174e-05,
+      "loss": 0.2467,
+      "step": 230
+    },
+    {
+      "epoch": 1.3646055437100213,
+      "grad_norm": 4.215674877166748,
+      "learning_rate": 9.504844339512095e-05,
+      "loss": 0.2682,
+      "step": 240
+    },
+    {
+      "epoch": 1.4214641080312722,
+      "grad_norm": 1.7884002923965454,
+      "learning_rate": 9.430247552150673e-05,
+      "loss": 0.29,
+      "step": 250
+    },
+    {
+      "epoch": 1.4783226723525231,
+      "grad_norm": 1.312519907951355,
+      "learning_rate": 9.350754959646306e-05,
+      "loss": 0.2252,
+      "step": 260
+    },
+    {
+      "epoch": 1.535181236673774,
+      "grad_norm": 2.8921542167663574,
+      "learning_rate": 9.266454408160779e-05,
+      "loss": 0.2408,
+      "step": 270
+    },
+    {
+      "epoch": 1.5920398009950247,
+      "grad_norm": 2.1106109619140625,
+      "learning_rate": 9.177439057064683e-05,
+      "loss": 0.2171,
+      "step": 280
+    },
+    {
+      "epoch": 1.6488983653162759,
+      "grad_norm": 1.2954068183898926,
+      "learning_rate": 9.083807275988284e-05,
+      "loss": 0.2394,
+      "step": 290
+    },
+    {
+      "epoch": 1.7057569296375266,
+      "grad_norm": 3.02253794670105,
+      "learning_rate": 8.985662536114613e-05,
+      "loss": 0.2307,
+      "step": 300
+    },
+    {
+      "epoch": 1.7626154939587777,
+      "grad_norm": 1.063110589981079,
+      "learning_rate": 8.883113295834892e-05,
+      "loss": 0.2172,
+      "step": 310
+    },
+    {
+      "epoch": 1.8194740582800284,
+      "grad_norm": 1.7286932468414307,
+      "learning_rate": 8.776272880892675e-05,
+      "loss": 0.2251,
+      "step": 320
+    },
+    {
+      "epoch": 1.8763326226012793,
+      "grad_norm": 1.194043755531311,
+      "learning_rate": 8.665259359149132e-05,
+      "loss": 0.2435,
+      "step": 330
+    },
+    {
+      "epoch": 1.9331911869225302,
+      "grad_norm": 0.8665173649787903,
+      "learning_rate": 8.550195410107902e-05,
+      "loss": 0.2425,
+      "step": 340
+    },
+    {
+      "epoch": 1.9900497512437811,
+      "grad_norm": 1.0586192607879639,
+      "learning_rate": 8.43120818934367e-05,
+      "loss": 0.2297,
+      "step": 350
+    },
+    {
+      "epoch": 1.9900497512437811,
+      "eval_accuracy": 0.9031333333333333,
+      "eval_loss": 0.24609853327274323,
+      "eval_runtime": 206.1828,
+      "eval_samples_per_second": 12.125,
+      "eval_steps_per_second": 12.125,
+      "step": 350
+    },
+    {
+      "epoch": 2.046908315565032,
+      "grad_norm": 1.062510371208191,
+      "learning_rate": 8.308429187984297e-05,
+      "loss": 0.1928,
+      "step": 360
+    },
+    {
+      "epoch": 2.1037668798862827,
+      "grad_norm": 1.1897845268249512,
+      "learning_rate": 8.181994087401819e-05,
+      "loss": 0.194,
+      "step": 370
+    },
+    {
+      "epoch": 2.160625444207534,
+      "grad_norm": 1.3862345218658447,
+      "learning_rate": 8.052042609272817e-05,
+      "loss": 0.1699,
+      "step": 380
+    },
+    {
+      "epoch": 2.2174840085287846,
+      "grad_norm": 0.9099957346916199,
+      "learning_rate": 7.91871836117395e-05,
+      "loss": 0.186,
+      "step": 390
+    },
+    {
+      "epoch": 2.2743425728500357,
+      "grad_norm": 1.0390052795410156,
+      "learning_rate": 7.782168677883206e-05,
+      "loss": 0.1711,
+      "step": 400
+    },
+    {
+      "epoch": 2.3312011371712864,
+      "grad_norm": 2.652061700820923,
+      "learning_rate": 7.642544458562278e-05,
+      "loss": 0.2012,
+      "step": 410
+    },
+    {
+      "epoch": 2.388059701492537,
+      "grad_norm": 1.5696091651916504,
+      "learning_rate": 7.500000000000001e-05,
+      "loss": 0.1898,
+      "step": 420
+    },
+    {
+      "epoch": 2.4449182658137882,
+      "grad_norm": 0.8261299729347229,
+      "learning_rate": 7.354692826101102e-05,
+      "loss": 0.2032,
+      "step": 430
+    },
+    {
+      "epoch": 2.501776830135039,
+      "grad_norm": 1.252537727355957,
+      "learning_rate": 7.20678351380872e-05,
+      "loss": 0.1854,
+      "step": 440
+    },
+    {
+      "epoch": 2.55863539445629,
+      "grad_norm": 1.3897773027420044,
+      "learning_rate": 7.056435515653059e-05,
+      "loss": 0.202,
+      "step": 450
+    },
+    {
+      "epoch": 2.6154939587775408,
+      "grad_norm": 0.8310902714729309,
+      "learning_rate": 6.903814979122249e-05,
+      "loss": 0.1789,
+      "step": 460
+    },
+    {
+      "epoch": 2.672352523098792,
+      "grad_norm": 1.3531970977783203,
+      "learning_rate": 6.749090563055076e-05,
+      "loss": 0.1969,
+      "step": 470
+    },
+    {
+      "epoch": 2.7292110874200426,
+      "grad_norm": 1.0367170572280884,
+      "learning_rate": 6.592433251258423e-05,
+      "loss": 0.1967,
+      "step": 480
+    },
+    {
+      "epoch": 2.7860696517412933,
+      "grad_norm": 1.2085515260696411,
+      "learning_rate": 6.434016163555452e-05,
+      "loss": 0.1825,
+      "step": 490
+    },
+    {
+      "epoch": 2.8429282160625444,
+      "grad_norm": 1.2390882968902588,
+      "learning_rate": 6.274014364473274e-05,
+      "loss": 0.212,
+      "step": 500
+    },
+    {
+      "epoch": 2.8997867803837956,
+      "grad_norm": 1.4294568300247192,
+      "learning_rate": 6.112604669781572e-05,
+      "loss": 0.191,
+      "step": 510
+    },
+    {
+      "epoch": 2.9566453447050463,
+      "grad_norm": 1.522834300994873,
+      "learning_rate": 5.949965451095951e-05,
+      "loss": 0.1792,
+      "step": 520
+    },
+    {
+      "epoch": 2.9850746268656714,
+      "eval_accuracy": 0.9013333333333332,
+      "eval_loss": 0.25212058424949646,
+      "eval_runtime": 206.7127,
+      "eval_samples_per_second": 12.094,
+      "eval_steps_per_second": 12.094,
+      "step": 525
+    },
+    {
+      "epoch": 3.013503909026297,
+      "grad_norm": 0.655222475528717,
+      "learning_rate": 5.786276438761927e-05,
+      "loss": 0.1592,
+      "step": 530
+    },
+    {
+      "epoch": 3.070362473347548,
+      "grad_norm": 0.8747724294662476,
+      "learning_rate": 5.621718523237427e-05,
+      "loss": 0.1186,
+      "step": 540
+    },
+    {
+      "epoch": 3.1272210376687988,
+      "grad_norm": 1.3602584600448608,
+      "learning_rate": 5.456473555193242e-05,
+      "loss": 0.1195,
+      "step": 550
+    },
+    {
+      "epoch": 3.18407960199005,
+      "grad_norm": 1.0027141571044922,
+      "learning_rate": 5.290724144552379e-05,
+      "loss": 0.144,
+      "step": 560
+    },
+    {
+      "epoch": 3.2409381663113006,
+      "grad_norm": 2.7845115661621094,
+      "learning_rate": 5.124653458690365e-05,
+      "loss": 0.1508,
+      "step": 570
+    },
+    {
+      "epoch": 3.2977967306325517,
+      "grad_norm": 0.7762941718101501,
+      "learning_rate": 4.9584450200195156e-05,
+      "loss": 0.1399,
+      "step": 580
+    },
+    {
+      "epoch": 3.3546552949538024,
+      "grad_norm": 0.9395729303359985,
+      "learning_rate": 4.792282503180867e-05,
+      "loss": 0.1321,
+      "step": 590
+    },
+    {
+      "epoch": 3.411513859275053,
+      "grad_norm": 0.9491863250732422,
+      "learning_rate": 4.626349532067879e-05,
+      "loss": 0.1245,
+      "step": 600
+    },
+    {
+      "epoch": 3.4683724235963043,
+      "grad_norm": 1.0270098447799683,
+      "learning_rate": 4.4608294769062075e-05,
+      "loss": 0.1211,
+      "step": 610
+    },
+    {
+      "epoch": 3.525230987917555,
+      "grad_norm": 1.0683083534240723,
+      "learning_rate": 4.295905251613817e-05,
+      "loss": 0.1336,
+      "step": 620
+    },
+    {
+      "epoch": 3.582089552238806,
+      "grad_norm": 1.0028445720672607,
+      "learning_rate": 4.131759111665349e-05,
+      "loss": 0.1379,
+      "step": 630
+    },
+    {
+      "epoch": 3.638948116560057,
+      "grad_norm": 0.6078014969825745,
+      "learning_rate": 3.968572452684113e-05,
+      "loss": 0.1248,
+      "step": 640
+    },
+    {
+      "epoch": 3.695806680881308,
+      "grad_norm": 1.0846917629241943,
+      "learning_rate": 3.806525609984312e-05,
+      "loss": 0.1244,
+      "step": 650
+    },
+    {
+      "epoch": 3.7526652452025586,
+      "grad_norm": 1.1056978702545166,
+      "learning_rate": 3.6457976592849754e-05,
+      "loss": 0.1455,
+      "step": 660
+    },
+    {
+      "epoch": 3.8095238095238093,
+      "grad_norm": 0.8466194868087769,
+      "learning_rate": 3.486566218815871e-05,
+      "loss": 0.1381,
+      "step": 670
+    },
+    {
+      "epoch": 3.8663823738450604,
+      "grad_norm": 1.3114445209503174,
+      "learning_rate": 3.329007253034063e-05,
+      "loss": 0.1278,
+      "step": 680
+    },
+    {
+      "epoch": 3.923240938166311,
+      "grad_norm": 1.2287284135818481,
+      "learning_rate": 3.173294878168025e-05,
+      "loss": 0.1487,
+      "step": 690
+    },
+    {
+      "epoch": 3.9800995024875623,
+      "grad_norm": 0.7370882630348206,
+      "learning_rate": 3.019601169804216e-05,
+      "loss": 0.122,
+      "step": 700
+    },
+    {
+      "epoch": 3.9800995024875623,
+      "eval_accuracy": 0.9049666666666666,
+      "eval_loss": 0.27484622597694397,
+      "eval_runtime": 207.5459,
+      "eval_samples_per_second": 12.046,
+      "eval_steps_per_second": 12.046,
+      "step": 700
+    },
+    {
+      "epoch": 4.036958066808813,
+      "grad_norm": 0.7140550017356873,
+      "learning_rate": 2.8680959727287317e-05,
+      "loss": 0.1074,
+      "step": 710
+    },
+    {
+      "epoch": 4.093816631130064,
+      "grad_norm": 1.042784333229065,
+      "learning_rate": 2.718946713234185e-05,
+      "loss": 0.0856,
+      "step": 720
+    },
+    {
+      "epoch": 4.150675195451315,
+      "grad_norm": 0.7461434602737427,
+      "learning_rate": 2.5723182140992387e-05,
+      "loss": 0.0837,
+      "step": 730
+    },
+    {
+      "epoch": 4.2075337597725655,
+      "grad_norm": 1.004974603652954,
+      "learning_rate": 2.428372512445233e-05,
+      "loss": 0.0749,
+      "step": 740
+    },
+    {
+      "epoch": 4.264392324093817,
+      "grad_norm": 0.6601787805557251,
+      "learning_rate": 2.2872686806712035e-05,
+      "loss": 0.0835,
+      "step": 750
+    },
+    {
+      "epoch": 4.321250888415068,
+      "grad_norm": 0.6331136226654053,
+      "learning_rate": 2.1491626506651914e-05,
+      "loss": 0.0685,
+      "step": 760
+    },
+    {
+      "epoch": 4.378109452736318,
+      "grad_norm": 0.8726811408996582,
+      "learning_rate": 2.0142070414860704e-05,
+      "loss": 0.0787,
+      "step": 770
+    },
+    {
+      "epoch": 4.434968017057569,
+      "grad_norm": 0.8191830515861511,
+      "learning_rate": 1.8825509907063327e-05,
+      "loss": 0.0783,
+      "step": 780
+    },
+    {
+      "epoch": 4.49182658137882,
+      "grad_norm": 0.6537315249443054,
+      "learning_rate": 1.7543399896022405e-05,
+      "loss": 0.0759,
+      "step": 790
+    },
+    {
+      "epoch": 4.548685145700071,
+      "grad_norm": 1.3562214374542236,
+      "learning_rate": 1.629715722373423e-05,
+      "loss": 0.0952,
+      "step": 800
+    },
+    {
+      "epoch": 4.605543710021322,
+      "grad_norm": 1.0490374565124512,
+      "learning_rate": 1.5088159095696363e-05,
+      "loss": 0.0916,
+      "step": 810
+    },
+    {
+      "epoch": 4.662402274342573,
+      "grad_norm": 0.8466753363609314,
+      "learning_rate": 1.3917741558976894e-05,
+      "loss": 0.0874,
+      "step": 820
+    },
+    {
+      "epoch": 4.719260838663824,
+      "grad_norm": 0.9645017385482788,
+      "learning_rate": 1.2787198025767416e-05,
+      "loss": 0.0879,
+      "step": 830
+    },
+    {
+      "epoch": 4.776119402985074,
+      "grad_norm": 3.8121776580810547,
+      "learning_rate": 1.1697777844051105e-05,
+      "loss": 0.0697,
+      "step": 840
+    },
+    {
+      "epoch": 4.832977967306325,
+      "grad_norm": 1.1398985385894775,
+      "learning_rate": 1.0650684916965559e-05,
+      "loss": 0.0873,
+      "step": 850
+    },
+    {
+      "epoch": 4.8898365316275765,
+      "grad_norm": 1.19906485080719,
+      "learning_rate": 9.647076372386194e-06,
+      "loss": 0.0826,
+      "step": 860
+    },
+    {
+      "epoch": 4.946695095948828,
+      "grad_norm": 0.7644526958465576,
+      "learning_rate": 8.688061284200266e-06,
+      "loss": 0.073,
+      "step": 870
+    },
+    {
+      "epoch": 4.975124378109452,
+      "eval_accuracy": 0.9083,
+      "eval_loss": 0.33086976408958435,
+      "eval_runtime": 206.4018,
+      "eval_samples_per_second": 12.112,
+      "eval_steps_per_second": 12.112,
+      "step": 875
+    },
+    {
+      "epoch": 5.003553660270078,
+      "grad_norm": 0.686549723148346,
+      "learning_rate": 7.774699446684608e-06,
+      "loss": 0.082,
+      "step": 880
+    },
+    {
+      "epoch": 5.060412224591329,
+      "grad_norm": 0.5072228312492371,
+      "learning_rate": 6.908000203341802e-06,
+      "loss": 0.0516,
+      "step": 890
+    },
+    {
+      "epoch": 5.11727078891258,
+      "grad_norm": 0.5079275965690613,
+      "learning_rate": 6.088921331488568e-06,
+      "loss": 0.0607,
+      "step": 900
+    },
+    {
+      "epoch": 5.174129353233831,
+      "grad_norm": 0.8684201240539551,
+      "learning_rate": 5.318367983829392e-06,
+      "loss": 0.0451,
+      "step": 910
+    },
+    {
+      "epoch": 5.2309879175550815,
+      "grad_norm": 0.822227954864502,
+      "learning_rate": 4.597191688184754e-06,
+      "loss": 0.0516,
+      "step": 920
+    },
+    {
+      "epoch": 5.287846481876333,
+      "grad_norm": 0.5135931968688965,
+      "learning_rate": 3.9261894064796135e-06,
+      "loss": 0.0479,
+      "step": 930
+    },
+    {
+      "epoch": 5.344705046197584,
+      "grad_norm": 0.5866013765335083,
+      "learning_rate": 3.306102654031823e-06,
+      "loss": 0.0547,
+      "step": 940
+    },
+    {
+      "epoch": 5.401563610518834,
+      "grad_norm": 1.5747359991073608,
+      "learning_rate": 2.737616680113758e-06,
+      "loss": 0.0555,
+      "step": 950
+    },
+    {
+      "epoch": 5.458422174840085,
+      "grad_norm": 2.3203916549682617,
+      "learning_rate": 2.221359710692961e-06,
+      "loss": 0.0495,
+      "step": 960
+    },
+    {
+      "epoch": 5.515280739161336,
+      "grad_norm": 0.6227638721466064,
+      "learning_rate": 1.757902254188254e-06,
+      "loss": 0.0522,
+      "step": 970
+    },
+    {
+      "epoch": 5.572139303482587,
+      "grad_norm": 1.4704910516738892,
+      "learning_rate": 1.3477564710088098e-06,
+      "loss": 0.0536,
+      "step": 980
+    },
+    {
+      "epoch": 5.628997867803838,
+      "grad_norm": 0.4436222314834595,
+      "learning_rate": 9.913756075728087e-07,
+      "loss": 0.0518,
+      "step": 990
+    },
+    {
+      "epoch": 5.685856432125089,
+      "grad_norm": 0.6575887799263,
+      "learning_rate": 6.891534954310885e-07,
+      "loss": 0.0491,
+      "step": 1000
+    },
+    {
+      "epoch": 5.74271499644634,
+      "grad_norm": 0.5118672251701355,
+      "learning_rate": 4.4142411604936597e-07,
+      "loss": 0.0355,
+      "step": 1010
+    },
+    {
+      "epoch": 5.79957356076759,
+      "grad_norm": 0.8050460815429688,
+      "learning_rate": 2.4846123172992954e-07,
+      "loss": 0.0425,
+      "step": 1020
+    },
+    {
+      "epoch": 5.856432125088841,
+      "grad_norm": 0.6702028512954712,
+      "learning_rate": 1.1047808308075058e-07,
+      "loss": 0.0553,
+      "step": 1030
+    },
+    {
+      "epoch": 5.9132906894100925,
+      "grad_norm": 0.4666765630245209,
+      "learning_rate": 2.7627153366222013e-08,
+      "loss": 0.061,
+      "step": 1040
+    },
+    {
+      "epoch": 5.970149253731344,
+      "grad_norm": 0.5390927195549011,
+      "learning_rate": 0.0,
+      "loss": 0.0484,
+      "step": 1050
+    },
+    {
+      "epoch": 5.970149253731344,
+      "eval_accuracy": 0.9057333333333334,
+      "eval_loss": 0.43139833211898804,
+      "eval_runtime": 206.2708,
+      "eval_samples_per_second": 12.12,
+      "eval_steps_per_second": 12.12,
+      "step": 1050
+    },
+    {
+      "epoch": 6.056858564321251,
+      "grad_norm": 1.7551038265228271,
+      "learning_rate": 4.0337204110222346e-05,
+      "loss": 0.0469,
+      "step": 1060
+    },
+    {
+      "epoch": 6.113717128642501,
+      "grad_norm": 1.7613005638122559,
+      "learning_rate": 3.936066149022191e-05,
+      "loss": 0.0651,
+      "step": 1070
+    },
+    {
+      "epoch": 6.1705756929637525,
+      "grad_norm": 0.5738229751586914,
+      "learning_rate": 3.838835177848788e-05,
+      "loss": 0.0687,
+      "step": 1080
+    },
+    {
+      "epoch": 6.227434257285004,
+      "grad_norm": 0.678581714630127,
+      "learning_rate": 3.742066181277458e-05,
+      "loss": 0.0707,
+      "step": 1090
+    },
+    {
+      "epoch": 6.284292821606255,
+      "grad_norm": 0.7234874367713928,
+      "learning_rate": 3.6457976592849754e-05,
+      "loss": 0.0681,
+      "step": 1100
+    },
+    {
+      "epoch": 6.341151385927505,
+      "grad_norm": 1.1438580751419067,
+      "learning_rate": 3.550067912732069e-05,
+      "loss": 0.0831,
+      "step": 1110
+    },
+    {
+      "epoch": 6.398009950248756,
+      "grad_norm": 1.3569965362548828,
+      "learning_rate": 3.4549150281252636e-05,
+      "loss": 0.064,
+      "step": 1120
+    },
+    {
+      "epoch": 6.454868514570007,
+      "grad_norm": 0.9517507553100586,
+      "learning_rate": 3.360376862463979e-05,
+      "loss": 0.0723,
+      "step": 1130
+    },
+    {
+      "epoch": 6.5117270788912585,
+      "grad_norm": 1.167060136795044,
+      "learning_rate": 3.266491028178964e-05,
+      "loss": 0.0882,
+      "step": 1140
+    },
+    {
+      "epoch": 6.568585643212509,
+      "grad_norm": 0.5366312861442566,
+      "learning_rate": 3.173294878168025e-05,
+      "loss": 0.0856,
+      "step": 1150
+    },
+    {
+      "epoch": 6.62544420753376,
+      "grad_norm": 0.7633583545684814,
+      "learning_rate": 3.080825490934999e-05,
+      "loss": 0.0766,
+      "step": 1160
+    },
+    {
+      "epoch": 6.682302771855011,
+      "grad_norm": 0.6523817777633667,
+      "learning_rate": 2.989119655837913e-05,
+      "loss": 0.0741,
+      "step": 1170
+    },
+    {
+      "epoch": 6.739161336176261,
+      "grad_norm": 1.3234527111053467,
+      "learning_rate": 2.8982138584521735e-05,
+      "loss": 0.0925,
+      "step": 1180
+    },
+    {
+      "epoch": 6.796019900497512,
+      "grad_norm": 0.8096024990081787,
+      "learning_rate": 2.8081442660546125e-05,
+      "loss": 0.0856,
+      "step": 1190
+    },
+    {
+      "epoch": 6.8528784648187635,
+      "grad_norm": 1.324310064315796,
+      "learning_rate": 2.718946713234185e-05,
+      "loss": 0.079,
+      "step": 1200
+    },
+    {
+      "epoch": 6.909737029140015,
+      "grad_norm": 1.0153288841247559,
+      "learning_rate": 2.630656687635007e-05,
+      "loss": 0.0791,
+      "step": 1210
+    },
+    {
+      "epoch": 6.966595593461265,
+      "grad_norm": 0.7319349646568298,
+      "learning_rate": 2.543309315837444e-05,
+      "loss": 0.0691,
+      "step": 1220
+    },
+    {
+      "epoch": 6.9950248756218905,
+      "eval_accuracy": 0.9058,
+      "eval_loss": 0.4234231412410736,
+      "eval_runtime": 204.7434,
+      "eval_samples_per_second": 12.21,
+      "eval_steps_per_second": 12.21,
+      "step": 1225
+    }
+  ],
+  "logging_steps": 10,
+  "max_steps": 1750,
+  "num_input_tokens_seen": 0,
+  "num_train_epochs": 10,
+  "save_steps": 175,
+  "stateful_callbacks": {
+    "TrainerControl": {
+      "args": {
+        "should_epoch_stop": false,
+        "should_evaluate": false,
+        "should_log": false,
+        "should_save": true,
+        "should_training_stop": false
+      },
+      "attributes": {}
+    }
+  },
+  "total_flos": 3.559828853937144e+18,
+  "train_batch_size": 16,
+  "trial_name": null,
+  "trial_params": null
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1225/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64dea0ae1d7a13058ec50e8bf478ba9f4efe640f9feb0e3577fcd622cf798c19
+size 5368

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/README.md

@@ -0,0 +1,202 @@
+---
+library_name: peft
+base_model: shenzhi-wang/Llama3-8B-Chinese-Chat
+---
+
+# Model Card for Model ID
+
+<!-- Provide a quick summary of what the model is/does. -->
+
+
+
+## Model Details
+
+### Model Description
+
+<!-- Provide a longer summary of what this model is. -->
+
+
+
+- **Developed by:** [More Information Needed]
+- **Funded by [optional]:** [More Information Needed]
+- **Shared by [optional]:** [More Information Needed]
+- **Model type:** [More Information Needed]
+- **Language(s) (NLP):** [More Information Needed]
+- **License:** [More Information Needed]
+- **Finetuned from model [optional]:** [More Information Needed]
+
+### Model Sources [optional]
+
+<!-- Provide the basic links for the model. -->
+
+- **Repository:** [More Information Needed]
+- **Paper [optional]:** [More Information Needed]
+- **Demo [optional]:** [More Information Needed]
+
+## Uses
+
+<!-- Address questions around how the model is intended to be used, including the foreseeable users of the model and those affected by the model. -->
+
+### Direct Use
+
+<!-- This section is for the model use without fine-tuning or plugging into a larger ecosystem/app. -->
+
+[More Information Needed]
+
+### Downstream Use [optional]
+
+<!-- This section is for the model use when fine-tuned for a task, or when plugged into a larger ecosystem/app -->
+
+[More Information Needed]
+
+### Out-of-Scope Use
+
+<!-- This section addresses misuse, malicious use, and uses that the model will not work well for. -->
+
+[More Information Needed]
+
+## Bias, Risks, and Limitations
+
+<!-- This section is meant to convey both technical and sociotechnical limitations. -->
+
+[More Information Needed]
+
+### Recommendations
+
+<!-- This section is meant to convey recommendations with respect to the bias, risk, and technical limitations. -->
+
+Users (both direct and downstream) should be made aware of the risks, biases and limitations of the model. More information needed for further recommendations.
+
+## How to Get Started with the Model
+
+Use the code below to get started with the model.
+
+[More Information Needed]
+
+## Training Details
+
+### Training Data
+
+<!-- This should link to a Dataset Card, perhaps with a short stub of information on what the training data is all about as well as documentation related to data pre-processing or additional filtering. -->
+
+[More Information Needed]
+
+### Training Procedure
+
+<!-- This relates heavily to the Technical Specifications. Content here should link to that section when it is relevant to the training procedure. -->
+
+#### Preprocessing [optional]
+
+[More Information Needed]
+
+
+#### Training Hyperparameters
+
+- **Training regime:** [More Information Needed] <!--fp32, fp16 mixed precision, bf16 mixed precision, bf16 non-mixed precision, fp16 non-mixed precision, fp8 mixed precision -->
+
+#### Speeds, Sizes, Times [optional]
+
+<!-- This section provides information about throughput, start/end time, checkpoint size if relevant, etc. -->
+
+[More Information Needed]
+
+## Evaluation
+
+<!-- This section describes the evaluation protocols and provides the results. -->
+
+### Testing Data, Factors & Metrics
+
+#### Testing Data
+
+<!-- This should link to a Dataset Card if possible. -->
+
+[More Information Needed]
+
+#### Factors
+
+<!-- These are the things the evaluation is disaggregating by, e.g., subpopulations or domains. -->
+
+[More Information Needed]
+
+#### Metrics
+
+<!-- These are the evaluation metrics being used, ideally with a description of why. -->
+
+[More Information Needed]
+
+### Results
+
+[More Information Needed]
+
+#### Summary
+
+
+
+## Model Examination [optional]
+
+<!-- Relevant interpretability work for the model goes here -->
+
+[More Information Needed]
+
+## Environmental Impact
+
+<!-- Total emissions (in grams of CO2eq) and additional considerations, such as electricity usage, go here. Edit the suggested text below accordingly -->
+
+Carbon emissions can be estimated using the [Machine Learning Impact calculator](https://mlco2.github.io/impact#compute) presented in [Lacoste et al. (2019)](https://arxiv.org/abs/1910.09700).
+
+- **Hardware Type:** [More Information Needed]
+- **Hours used:** [More Information Needed]
+- **Cloud Provider:** [More Information Needed]
+- **Compute Region:** [More Information Needed]
+- **Carbon Emitted:** [More Information Needed]
+
+## Technical Specifications [optional]
+
+### Model Architecture and Objective
+
+[More Information Needed]
+
+### Compute Infrastructure
+
+[More Information Needed]
+
+#### Hardware
+
+[More Information Needed]
+
+#### Software
+
+[More Information Needed]
+
+## Citation [optional]
+
+<!-- If there is a paper or blog post introducing the model, the APA and Bibtex information for that should go in this section. -->
+
+**BibTeX:**
+
+[More Information Needed]
+
+**APA:**
+
+[More Information Needed]
+
+## Glossary [optional]
+
+<!-- If relevant, include terms and calculations in this section that can help readers understand the model or model card. -->
+
+[More Information Needed]
+
+## More Information [optional]
+
+[More Information Needed]
+
+## Model Card Authors [optional]
+
+[More Information Needed]
+
+## Model Card Contact
+
+[More Information Needed]
+### Framework versions
+
+- PEFT 0.11.1

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/adapter_config.json

@@ -0,0 +1,34 @@
+{
+  "alpha_pattern": {},
+  "auto_mapping": null,
+  "base_model_name_or_path": "shenzhi-wang/Llama3-8B-Chinese-Chat",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layer_replication": null,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "loftq_config": {},
+  "lora_alpha": 16,
+  "lora_dropout": 0.0,
+  "megatron_config": null,
+  "megatron_core": "megatron.core",
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 8,
+  "rank_pattern": {},
+  "revision": null,
+  "target_modules": [
+    "o_proj",
+    "up_proj",
+    "gate_proj",
+    "v_proj",
+    "down_proj",
+    "k_proj",
+    "q_proj"
+  ],
+  "task_type": "CAUSAL_LM",
+  "use_dora": false,
+  "use_rslora": false
+}

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/adapter_model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8ea6ff5f1ccdbd6e95b0154be023c9ef1d411b6ccdcabb21324c2f7cc7879b63
+size 83945296

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4f457863859f8c8f76191c9248abf35cf6841e7829a54a8b7d1dd227fe78882b
+size 168149394

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:52cca5856c568bc52c683b690919168fa27bfbdfefc6e0a62355afa6011157c3
+size 14244

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3fe6e1508f025ddea17fe8342368da21a823a3eae5bdcdf1197323ff068fffe
+size 1064

llama-factory/saves/llama3-8b/lora/sft_bf16_p1_full/checkpoint-1400/special_tokens_map.json

@@ -0,0 +1,23 @@
+{
+  "bos_token": {
+    "content": "<|begin_of_text|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<|eot_id|>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}