Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -57,7 +57,7 @@ bibtext = """
|
|
| 57 |
def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
|
| 58 |
roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
|
| 59 |
|
| 60 |
-
im_start, im_end = tokenizer.additional_special_tokens_ids
|
| 61 |
nl_tokens = tokenizer("\n").input_ids
|
| 62 |
_system = tokenizer("system").input_ids + nl_tokens
|
| 63 |
_user = tokenizer("user").input_ids + nl_tokens
|
|
|
|
| 57 |
def preprocess_qwen(sources, tokenizer: transformers.PreTrainedTokenizer, has_image: bool = False, max_len=2048, system_message: str = "You are a helpful assistant.") -> Dict:
|
| 58 |
roles = {"human": "<|im_start|>user", "gpt": "<|im_start|>assistant"}
|
| 59 |
|
| 60 |
+
im_start, im_end = tokenizer.additional_special_tokens_ids[:2]
|
| 61 |
nl_tokens = tokenizer("\n").input_ids
|
| 62 |
_system = tokenizer("system").input_ids + nl_tokens
|
| 63 |
_user = tokenizer("user").input_ids + nl_tokens
|