Update app.py

app.py

@@ -47,6 +47,11 @@ HF_TOKEN = os.environ.get("HF_TOKEN", None)
 # ! path where the model is downloaded, either on ./ or persistent disc
 MODEL_PATH = os.environ.get("MODEL_PATH", "./seal-13b-chat-a")
 
+# ! !! Whether to delete the folder, ONLY SET THIS IF YOU WANT TO DELETE SAVED MODEL ON PERSISTENT DISC
+DELETE_FOLDER = os.environ.get("DELETE_FOLDER", "")
+IS_DELETE_FOLDER = DELETE_FOLDER is not None and os.path.exists(DELETE_FOLDER)
+print(f'DELETE_FOLDER: {DELETE_FOLDER} | {DOWNLOAD_SNAPSHOT=}')
+
 # ! list of keywords to disabled as security measures to comply with local regulation
 KEYWORDS = os.environ.get("KEYWORDS", "").strip()
 KEYWORDS = KEYWORDS.split(";") if len(KEYWORDS) > 0 else []
@@ -205,6 +210,9 @@ path_markdown = """
 """
 
 
+
+
+
 def _detect_lang(text):
     # Disable language that may have safety risk
     from langdetect import detect as detect_lang
@@ -1072,8 +1080,10 @@ def chat_response_stream_multiturn(
     )
 
     sampling_params = SamplingParams(
-        temperature=temperature, max_tokens=max_tokens,
+        temperature=temperature, 
+        max_tokens=max_tokens,
         frequency_penalty=frequency_penalty,
+        stop=['<s>', '</s>', '<<SYS>>', '<</SYS>>', '[INST]', '[/INST]']
     )
     cur_out = None
 
@@ -1093,7 +1103,7 @@ def chat_response_stream_multiturn(
         item = next(iter(gen.values()))
         cur_out = item.outputs[0].text
         
-    print(f'{full_prompt}<<<{cur_out}>>>\n\n')
+    print(f'@@@@@@@@@@\n{full_prompt}<<<{cur_out}>>>\n@@@@@@@@@@\n')
     
     if cur_out is not None and "\\n" in cur_out:
         print(f'double slash-n in cur_out:\n{cur_out}')
@@ -1140,6 +1150,21 @@ def check_model_path(model_path) -> str:
     return ckpt_info
 
 
+
+def maybe_delete_folder():
+    if IS_DELETE_FOLDER and DOWNLOAD_SNAPSHOT:
+        print(f'DELETE ALL FILES IN {DELETE_FOLDER}')
+        for filename in os.listdir(DELETE_FOLDER):
+            file_path = os.path.join(DELETE_FOLDER, filename)
+            try:
+                if os.path.isfile(file_path) or os.path.islink(file_path):
+                    os.unlink(file_path)
+                elif os.path.isdir(file_path):
+                    shutil.rmtree(file_path)
+            except Exception as e:
+                print('Failed to delete %s. Reason: %s' % (file_path, e))
+
+
 def launch():
     global demo, llm, DEBUG
     model_desc = MODEL_DESC
@@ -1207,6 +1232,13 @@ def launch():
         except Exception as e:
             print(f'Cannot print model worker: {e}')
 
+        try:
+            llm.llm_engine.scheduler_config.max_model_len = 4096
+            llm.llm_engine.scheduler_config.max_num_batched_tokens = 4096
+            llm.llm_engine.tokenizer.add_special_tokens = False
+        except Exception as e:
+            print(f'Cannot set parameters: {e}')
+
         print(f'Use system prompt:\n{sys_prompt}')
 
         response_fn = chat_response_stream_multiturn