msr2000
commited on
Commit
·
bf3609a
1
Parent(s):
161a2f2
Update README.md
Browse files
README.md
CHANGED
|
@@ -189,7 +189,7 @@ model_name = "deepseek-ai/DeepSeek-V2"
|
|
| 189 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
| 190 |
# `max_memory` should be set based on your devices
|
| 191 |
max_memory = {i: "75GB" for i in range(8)}
|
| 192 |
-
model = AutoModelForCausalLM.from_pretrained(
|
| 193 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
| 194 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
| 195 |
|
|
@@ -210,7 +210,7 @@ model_name = "deepseek-ai/DeepSeek-V2-Chat"
|
|
| 210 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
| 211 |
# `max_memory` should be set based on your devices
|
| 212 |
max_memory = {i: "75GB" for i in range(8)}
|
| 213 |
-
model = AutoModelForCausalLM.from_pretrained(
|
| 214 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
| 215 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
| 216 |
|
|
|
|
| 189 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
| 190 |
# `max_memory` should be set based on your devices
|
| 191 |
max_memory = {i: "75GB" for i in range(8)}
|
| 192 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory)
|
| 193 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
| 194 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
| 195 |
|
|
|
|
| 210 |
tokenizer = AutoTokenizer.from_pretrained(model_name, trust_remote_code=True)
|
| 211 |
# `max_memory` should be set based on your devices
|
| 212 |
max_memory = {i: "75GB" for i in range(8)}
|
| 213 |
+
model = AutoModelForCausalLM.from_pretrained(model_name, trust_remote_code=True, device_map="auto", torch_dtype=torch.bfloat16, max_memory=max_memory)
|
| 214 |
model.generation_config = GenerationConfig.from_pretrained(model_name)
|
| 215 |
model.generation_config.pad_token_id = model.generation_config.eos_token_id
|
| 216 |
|