Upload files manually

README.md

@@ -0,0 +1,54 @@
+---
+base_model: asas-ai/bloom_360M_8bit
+tags:
+- generated_from_trainer
+model-index:
+- name: bloom_360M_4bit_qlora_arc
+  results: []
+---
+
+<!-- This model card has been generated automatically according to the information the Trainer had access to. You
+should probably proofread and complete it, then remove this comment. -->
+
+# bloom_360M_4bit_qlora_arc
+
+This model is a fine-tuned version of [asas-ai/bloom_360M_8bit](https://huggingface.co/asas-ai/bloom_360M_8bit) on an unknown dataset.
+
+## Model description
+
+More information needed
+
+## Intended uses & limitations
+
+More information needed
+
+## Training and evaluation data
+
+More information needed
+
+## Training procedure
+
+### Training hyperparameters
+
+The following hyperparameters were used during training:
+- learning_rate: 0.0002
+- train_batch_size: 4
+- eval_batch_size: 8
+- seed: 42
+- gradient_accumulation_steps: 4
+- total_train_batch_size: 16
+- optimizer: Adam with betas=(0.9,0.999) and epsilon=1e-08
+- lr_scheduler_type: constant
+- lr_scheduler_warmup_ratio: 0.03
+- training_steps: 2200
+
+### Training results
+
+
+
+### Framework versions
+
+- Transformers 4.33.1
+- Pytorch 2.0.1+cu117
+- Datasets 2.4.0
+- Tokenizers 0.12.1

adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_360M_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:633abef4603eb85f126b252766cd03e2cc19b2c3af0015878b8ec06d8072f7ce
+size 100733709

checkpoint-1760/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.6.0.dev0
+
+- PEFT 0.6.0.dev0

checkpoint-1760/adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_360M_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-1760/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40941251e5ea65502d6890c9b142ac4d16b5be0a90b51bf11994d28466bca245
+size 100733709

checkpoint-1760/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5bd71dc8e789f76e213337c976ddf4ac28bfb336da0423854184fa17b68ecf49
+size 201442373

checkpoint-1760/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:17137972cc20a4dfd087e2ab0be8a28cffea6b319a176b2a33842cd49d23ead4
+size 14575

checkpoint-1760/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c5ff61dc870ad27d377b69c3d781f714f8b44a93915b84c28547ea26cc839f
+size 627

checkpoint-1760/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-1760/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
+size 14500443

checkpoint-1760/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1760/trainer_state.json

@@ -0,0 +1,67 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 12.59391771019678,
+  "eval_steps": 500,
+  "global_step": 1760,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.57,
+      "learning_rate": 0.0002,
+      "loss": 10.5126,
+      "step": 220
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 0.0002,
+      "loss": 5.6837,
+      "step": 440
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.0002,
+      "loss": 5.0601,
+      "step": 660
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.0002,
+      "loss": 4.883,
+      "step": 880
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 0.0002,
+      "loss": 4.6134,
+      "step": 1100
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 0.0002,
+      "loss": 4.3961,
+      "step": 1320
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 0.0002,
+      "loss": 4.1023,
+      "step": 1540
+    },
+    {
+      "epoch": 12.59,
+      "learning_rate": 0.0002,
+      "loss": 3.9325,
+      "step": 1760
+    }
+  ],
+  "logging_steps": 220,
+  "max_steps": 2200,
+  "num_train_epochs": 16,
+  "save_steps": 220,
+  "total_flos": 4364916519862272.0,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1760/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33c48e4369d3c67fb1a49f7d8c214ee5a313773fc44bee0f4f608f031df052f
+size 4091

checkpoint-1980/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.6.0.dev0
+
+- PEFT 0.6.0.dev0

checkpoint-1980/adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_360M_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-1980/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bbb91f31c2efb556432915a16e23b67bbfbb5c056e3337a1b86f173b315463ce
+size 100733709

checkpoint-1980/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9055509fa08aa2e58543089f9024f3b4a405500c13992b5924623ea7a8038321
+size 201442373

checkpoint-1980/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:14bcf44bcfca2d65f97f35673ce594adb471d67332e3577da7cb67cb3e43e7bf
+size 14575

checkpoint-1980/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b7b3cad452dc4595e700cca73b74b2842931f81f5ad69a77dcc8aa064e6bd19
+size 627

checkpoint-1980/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-1980/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
+size 14500443

checkpoint-1980/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-1980/trainer_state.json

@@ -0,0 +1,73 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 14.168157423971378,
+  "eval_steps": 500,
+  "global_step": 1980,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.57,
+      "learning_rate": 0.0002,
+      "loss": 10.5126,
+      "step": 220
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 0.0002,
+      "loss": 5.6837,
+      "step": 440
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.0002,
+      "loss": 5.0601,
+      "step": 660
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.0002,
+      "loss": 4.883,
+      "step": 880
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 0.0002,
+      "loss": 4.6134,
+      "step": 1100
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 0.0002,
+      "loss": 4.3961,
+      "step": 1320
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 0.0002,
+      "loss": 4.1023,
+      "step": 1540
+    },
+    {
+      "epoch": 12.59,
+      "learning_rate": 0.0002,
+      "loss": 3.9325,
+      "step": 1760
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 0.0002,
+      "loss": 3.6148,
+      "step": 1980
+    }
+  ],
+  "logging_steps": 220,
+  "max_steps": 2200,
+  "num_train_epochs": 16,
+  "save_steps": 220,
+  "total_flos": 4906228006895616.0,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-1980/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33c48e4369d3c67fb1a49f7d8c214ee5a313773fc44bee0f4f608f031df052f
+size 4091

checkpoint-2200/README.md

@@ -0,0 +1,34 @@
+---
+library_name: peft
+---
+## Training procedure
+
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+
+The following `bitsandbytes` quantization config was used during training:
+- quant_method: bitsandbytes
+- load_in_8bit: True
+- load_in_4bit: False
+- llm_int8_threshold: 6.0
+- llm_int8_skip_modules: None
+- llm_int8_enable_fp32_cpu_offload: False
+- llm_int8_has_fp16_weight: False
+- bnb_4bit_quant_type: fp4
+- bnb_4bit_use_double_quant: False
+- bnb_4bit_compute_dtype: float32
+### Framework versions
+
+- PEFT 0.6.0.dev0
+
+- PEFT 0.6.0.dev0

checkpoint-2200/adapter_config.json

@@ -0,0 +1,23 @@
+{
+  "auto_mapping": null,
+  "base_model_name_or_path": "asas-ai/bloom_360M_8bit",
+  "bias": "none",
+  "fan_in_fan_out": false,
+  "inference_mode": true,
+  "init_lora_weights": true,
+  "layers_pattern": null,
+  "layers_to_transform": null,
+  "lora_alpha": 16,
+  "lora_dropout": 0.1,
+  "modules_to_save": null,
+  "peft_type": "LORA",
+  "r": 64,
+  "revision": null,
+  "target_modules": [
+    "query_key_value",
+    "dense",
+    "dense_h_to_4h",
+    "dense_4h_to_h"
+  ],
+  "task_type": "CAUSAL_LM"
+}

checkpoint-2200/adapter_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:633abef4603eb85f126b252766cd03e2cc19b2c3af0015878b8ec06d8072f7ce
+size 100733709

checkpoint-2200/optimizer.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0573c810dcc12b1fbbf51eabc82feda8570e410943fdd99e572c6ebad3c5f93f
+size 201442373

checkpoint-2200/rng_state.pth

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:84eff65af8dc3313985b8c730e0a3478af9957880eff6085bb30076835729333
+size 14575

checkpoint-2200/scheduler.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f10a983aa914555fea6e5c0db8d7ddbaebbe7e28546c78ee0e93ac76cbc28436
+size 627

checkpoint-2200/special_tokens_map.json

@@ -0,0 +1,6 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "pad_token": "</s>",
+  "unk_token": "<unk>"
+}

checkpoint-2200/tokenizer.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8f6efc66e73f1fd69da4f436e48befb519fdff3fe18910850c1d41bd862293a5
+size 14500443

checkpoint-2200/tokenizer_config.json

@@ -0,0 +1,11 @@
+{
+  "add_prefix_space": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "model_max_length": 1000000000000000019884624838656,
+  "pad_token": "<pad>",
+  "padding_side": "left",
+  "tokenizer_class": "BloomTokenizer",
+  "unk_token": "<unk>"
+}

checkpoint-2200/trainer_state.json

@@ -0,0 +1,79 @@
+{
+  "best_metric": null,
+  "best_model_checkpoint": null,
+  "epoch": 15.742397137745975,
+  "eval_steps": 500,
+  "global_step": 2200,
+  "is_hyper_param_search": false,
+  "is_local_process_zero": true,
+  "is_world_process_zero": true,
+  "log_history": [
+    {
+      "epoch": 1.57,
+      "learning_rate": 0.0002,
+      "loss": 10.5126,
+      "step": 220
+    },
+    {
+      "epoch": 3.15,
+      "learning_rate": 0.0002,
+      "loss": 5.6837,
+      "step": 440
+    },
+    {
+      "epoch": 4.72,
+      "learning_rate": 0.0002,
+      "loss": 5.0601,
+      "step": 660
+    },
+    {
+      "epoch": 6.3,
+      "learning_rate": 0.0002,
+      "loss": 4.883,
+      "step": 880
+    },
+    {
+      "epoch": 7.87,
+      "learning_rate": 0.0002,
+      "loss": 4.6134,
+      "step": 1100
+    },
+    {
+      "epoch": 9.45,
+      "learning_rate": 0.0002,
+      "loss": 4.3961,
+      "step": 1320
+    },
+    {
+      "epoch": 11.02,
+      "learning_rate": 0.0002,
+      "loss": 4.1023,
+      "step": 1540
+    },
+    {
+      "epoch": 12.59,
+      "learning_rate": 0.0002,
+      "loss": 3.9325,
+      "step": 1760
+    },
+    {
+      "epoch": 14.17,
+      "learning_rate": 0.0002,
+      "loss": 3.6148,
+      "step": 1980
+    },
+    {
+      "epoch": 15.74,
+      "learning_rate": 0.0002,
+      "loss": 3.2611,
+      "step": 2200
+    }
+  ],
+  "logging_steps": 220,
+  "max_steps": 2200,
+  "num_train_epochs": 16,
+  "save_steps": 220,
+  "total_flos": 5447460898897920.0,
+  "trial_name": null,
+  "trial_params": null
+}

checkpoint-2200/training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33c48e4369d3c67fb1a49f7d8c214ee5a313773fc44bee0f4f608f031df052f
+size 4091

runs/Sep07_00-06-17_n78zj2u8um/events.out.tfevents.1694045181.n78zj2u8um.905.0

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:149e22af47a2ba2344935e8650dcb424fc9a29aa57bdf5233d5f778ad115b1c2
+size 6594

training_args.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e33c48e4369d3c67fb1a49f7d8c214ee5a313773fc44bee0f4f608f031df052f
+size 4091