Upload tokenizer

tokenization_dart.py

@@ -15,6 +15,12 @@ VOCAB_FILES_NAMES = {
     "tag_category": "tag_category.json",
 }
 
+PRETRAINED_VOCAB_FILES_MAP = {
+    "tag_category": {
+        "p1atdev/tokenizer_test_1": "https://huggingface.co/p1atdev/tokenizer_test_1/resolve/main/tag_category.json"
+    }
+}
+
 
 @dataclass
 class Category:
@@ -63,6 +69,7 @@ class DartTokenizer(PreTrainedTokenizerFast):
     """Dart tokenizer"""
 
     vocab_files_names = VOCAB_FILES_NAMES
+    pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
 
     def __init__(self, tag_category, **kwargs):
         super().__init__(**kwargs)
@@ -137,7 +144,7 @@ class DartTokenizer(PreTrainedTokenizerFast):
         input_ids: List[int],
         category_mask: Optional[Dict[str, np.ndarray]] = None,
     ) -> Tuple[np.ndarray, Dict[str, np.ndarray]]:
-        """Get the next token's vocab mask to be used"""
+        """Get the next token's vocab mask and a category mask"""
 
         if category_mask == None:
             category_mask = self.category_mask