update demo.

app.py

@@ -61,12 +61,13 @@ The service is a research preview intended for non-commercial use only, subject
 
 
 class Chat:
-    def __init__(self, model_path, conv_mode, model_base=None, load_8bit=False, load_4bit=False):
+    def __init__(self, model_path, conv_mode, model_base=None, load_8bit=False, load_4bit=False, device='cuda'):
         # disable_torch_init()
         model_name = get_model_name_from_path(model_path)
         self.tokenizer, self.model, processor, context_len = load_pretrained_model(
             model_path, model_base, model_name,
             load_8bit, load_4bit,
+            device=device,
             offload_folder="save_folder")
         self.processor = processor
         self.conv_mode = conv_mode
@@ -247,7 +248,7 @@ if __name__ == '__main__':
 
     handler = Chat(model_path, conv_mode=conv_mode, load_8bit=False, load_4bit=True)
     # handler.model.to(dtype=torch.float16)
-    handler = handler.model.to(device)
+    # handler = handler.model.to(device)
 
     if not os.path.exists("temp"):
         os.makedirs("temp")