update

app.py

@@ -40,6 +40,7 @@ DTYPE = os.environ.get("DTYPE", "bfloat16")
 DOWNLOAD_SNAPSHOT = bool(int(os.environ.get("DOWNLOAD_SNAPSHOT", "0")))
 # ! uploaded model path, will be downloaded to MODEL_PATH
 HF_MODEL_NAME = os.environ.get("HF_MODEL_NAME", "DAMO-NLP-SG/seal-13b-chat-a")
+HF_TOKEN = os.environ.get("HF_TOKEN", None)
 MODEL_PATH = os.environ.get("MODEL_PATH", "./seal-13b-chat-a")
 
 
@@ -898,16 +899,19 @@ def launch():
         print(f'Creating in DEBUG MODE')
     else:
         # ! load the model
-        import vllm
-        from vllm import LLM, SamplingParams
-
-        print(F'VLLM: {vllm.__version__}')
 
         if DOWNLOAD_SNAPSHOT:
             print(f'Downloading from HF_MODEL_NAME={hf_model_name} -> {model_path}')
-            snapshot_download(hf_model_name, local_dir=model_path)
+            if HF_TOKEN is not None:
+                print(f'Load with HF_TOKEN: {HF_TOKEN}')
+                snapshot_download(hf_model_name, local_dir=model_path, use_auth_token=True, token=HF_TOKEN)
+            else:
+                snapshot_download(hf_model_name, local_dir=model_path)
 
-        assert os.path.exists(model_path), f'{model_path} not found and no snapshot download'
+        import vllm
+        from vllm import LLM, SamplingParams
+
+        print(F'VLLM: {vllm.__version__}')
         ckpt_info = check_model_path(model_path)
 
         print(f'Load path: {model_path} | {ckpt_info}')

requirements.txt

@@ -5,8 +5,6 @@ evaluate
 datasets
 sacrebleu
 websockets
-fire
-indic-nlp-library
 omegaconf
 scikit-learn
 jiwer
@@ -18,10 +16,8 @@ ray
 psutil
 xformers >= 0.0.19
 fastapi
-tensorboard
 geomloss
 einops
-gdown
 langdetect
 vllm==0.1.4
 transformers