Spaces:
Running
Running
remove lru_cache: has some bugs
Browse files- character_app.py +0 -1
- playground_examples.py +1 -1
- playground_util.py +1 -1
character_app.py
CHANGED
|
@@ -18,7 +18,6 @@ abbr2name = {column[1]: column[0].split('-')[-1] for column in all_columns}
|
|
| 18 |
|
| 19 |
|
| 20 |
def get_column_info(columns):
|
| 21 |
-
print(columns)
|
| 22 |
markdown = ""
|
| 23 |
for column in columns:
|
| 24 |
markdown += f"- `num({column})`: num of tokens containing {abbr2name[column]} characters\n" \
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
def get_column_info(columns):
|
|
|
|
| 21 |
markdown = ""
|
| 22 |
for column in columns:
|
| 23 |
markdown += f"- `num({column})`: num of tokens containing {abbr2name[column]} characters\n" \
|
playground_examples.py
CHANGED
|
@@ -37,7 +37,7 @@ examples = {
|
|
| 37 |
|
| 38 |
more_examples = [
|
| 39 |
# bert系列
|
| 40 |
-
("bert-base-cased", "bert-base-uncased", "", ""), # # clue VS kplug, bert VS clue
|
| 41 |
("bert-base-cased", "clue", "", "增加了[]()"),
|
| 42 |
("roberta-chinese-clue", "kplug", "", ""),
|
| 43 |
|
|
|
|
| 37 |
|
| 38 |
more_examples = [
|
| 39 |
# bert系列
|
| 40 |
+
("google-bert/bert-base-cased", "google-bert/bert-base-uncased", "", ""), # # clue VS kplug, bert VS clue
|
| 41 |
("bert-base-cased", "clue", "", "增加了[]()"),
|
| 42 |
("roberta-chinese-clue", "kplug", "", ""),
|
| 43 |
|
playground_util.py
CHANGED
|
@@ -15,7 +15,7 @@ Buenos días!
|
|
| 15 |
default_tokenizer_name_1 = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
|
| 16 |
default_tokenizer_name_2 = "openai/gpt-4"
|
| 17 |
|
| 18 |
-
@lru_cache
|
| 19 |
def tokenize(
|
| 20 |
text: str,
|
| 21 |
tokenizer_name: str,
|
|
|
|
| 15 |
default_tokenizer_name_1 = "gradientai/Llama-3-8B-Instruct-Gradient-1048k"
|
| 16 |
default_tokenizer_name_2 = "openai/gpt-4"
|
| 17 |
|
| 18 |
+
# @lru_cache
|
| 19 |
def tokenize(
|
| 20 |
text: str,
|
| 21 |
tokenizer_name: str,
|