5

app.py

@@ -21,14 +21,13 @@ class GLM4VModel:
     def __init__(self):
         self.processor = None
         self.model = None
-        self.device = "cuda:0" if torch.cuda.is_available() else "cpu"
 
     def load(self):
         self.processor = AutoProcessor.from_pretrained(MODEL_PATH, use_fast=True)
         self.model = Glm4vForConditionalGeneration.from_pretrained(
             MODEL_PATH,
             torch_dtype=torch.bfloat16,
-            device_map=self.device,
+            device_map="auto",
             attn_implementation="sdpa",
         )
 
@@ -137,7 +136,7 @@ class GLM4VModel:
             return_dict=True,
             return_tensors="pt",
             padding=True,
-        ).to(self.device)
+        ).to(self.model.device)
 
         streamer = TextIteratorStreamer(self.processor.tokenizer, skip_prompt=True, skip_special_tokens=False)
         gen_args = dict(
@@ -251,7 +250,7 @@ def chat(files, msg, raw_hist, sys_prompt):
             break
         place["content"] = chunk
         display_hist = create_display_history(raw_hist)
-        yield display_hist, copy.deepcopy(raw_hist), None, ""
+        yield display_hist, copyf.deepcopy(raw_hist), None, ""
 
     display_hist = create_display_history(raw_hist)
     yield display_hist, copy.deepcopy(raw_hist), None, ""