Commit
·
03b5bc9
1
Parent(s):
b5acbe8
update plot with new naming & reformat the code
Browse files
README.md
CHANGED
|
@@ -42,19 +42,23 @@ from transformers import AutoModelForCausalLM, AutoTokenizer
|
|
| 42 |
|
| 43 |
tokenizer = AutoTokenizer.from_pretrained('stabilityai/stablelm-zephyr-3b')
|
| 44 |
model = AutoModelForCausalLM.from_pretrained(
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
)
|
| 49 |
|
| 50 |
prompt = [{'role': 'user', 'content': 'List 3 synonyms for the word "tiny"'}]
|
| 51 |
-
inputs = tokenizer.apply_chat_template(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
tokens = model.generate(
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
)
|
| 59 |
|
| 60 |
print(tokenizer.decode(tokens[0], skip_special_tokens=False))
|
|
@@ -87,14 +91,15 @@ The dataset is comprised of a mixture of open datasets large-scale datasets avai
|
|
| 87 |
|
| 88 |
## Performance
|
| 89 |
|
| 90 |
-
### MT
|
| 91 |
|
| 92 |
-
<img src="https://cdn-uploads.huggingface.co/production/uploads/6310474ca119d49bc1eb0d80/
|
| 93 |
|
| 94 |
| Model | Size | Alignment | MT-Bench (score) | AlpacaEval (win rate %) |
|
| 95 |
|-------------|-----|----|---------------|--------------|
|
| 96 |
| **StableLM Zephyr 3B** 🪁 | 3B | DPO | 6.64 | 76.00 |
|
| 97 |
| Stable Zephyr (SFT only) | 3B | SFT | 6.04 | 71.15 |
|
|
|
|
| 98 |
| MPT-Chat | 7B |dSFT |5.42| -|
|
| 99 |
| Xwin-LMv0.1 | 7B| dPPO| 6.19| 87.83|
|
| 100 |
| Mistral-Instructv0.1 | 7B| - | 6.84 |-|
|
|
|
|
| 42 |
|
| 43 |
tokenizer = AutoTokenizer.from_pretrained('stabilityai/stablelm-zephyr-3b')
|
| 44 |
model = AutoModelForCausalLM.from_pretrained(
|
| 45 |
+
'stabilityai/stablelm-zephyr-3b',
|
| 46 |
+
trust_remote_code=True,
|
| 47 |
+
device_map="auto"
|
| 48 |
)
|
| 49 |
|
| 50 |
prompt = [{'role': 'user', 'content': 'List 3 synonyms for the word "tiny"'}]
|
| 51 |
+
inputs = tokenizer.apply_chat_template(
|
| 52 |
+
prompt,
|
| 53 |
+
add_generation_prompt=True,
|
| 54 |
+
return_tensors='pt'
|
| 55 |
+
)
|
| 56 |
|
| 57 |
tokens = model.generate(
|
| 58 |
+
inputs.to(model.device),
|
| 59 |
+
max_new_tokens=1024,
|
| 60 |
+
temperature=0.8,
|
| 61 |
+
do_sample=True
|
| 62 |
)
|
| 63 |
|
| 64 |
print(tokenizer.decode(tokens[0], skip_special_tokens=False))
|
|
|
|
| 91 |
|
| 92 |
## Performance
|
| 93 |
|
| 94 |
+
### MT-Bench and Alpaca Bench
|
| 95 |
|
| 96 |
+
<img src="https://cdn-uploads.huggingface.co/production/uploads/6310474ca119d49bc1eb0d80/jwpbBHzdCkHm0rMvPUVxC.png" alt="mt_bench_plot" width="600"/>
|
| 97 |
|
| 98 |
| Model | Size | Alignment | MT-Bench (score) | AlpacaEval (win rate %) |
|
| 99 |
|-------------|-----|----|---------------|--------------|
|
| 100 |
| **StableLM Zephyr 3B** 🪁 | 3B | DPO | 6.64 | 76.00 |
|
| 101 |
| Stable Zephyr (SFT only) | 3B | SFT | 6.04 | 71.15 |
|
| 102 |
+
| Capybara v1.9 | 3B | dSFT | 5.94 | - |
|
| 103 |
| MPT-Chat | 7B |dSFT |5.42| -|
|
| 104 |
| Xwin-LMv0.1 | 7B| dPPO| 6.19| 87.83|
|
| 105 |
| Mistral-Instructv0.1 | 7B| - | 6.84 |-|
|