Update llava/model/builder.py

llava/model/builder.py

@@ -139,6 +139,7 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
     if 'llava' in model_name.lower():
         mm_use_x_start_end = getattr(model.config, "mm_use_x_start_end", False)
         mm_use_x_patch_token = getattr(model.config, "mm_use_x_patch_token", True)
+        '''
         X = model.config.X
         if mm_use_x_patch_token:
             for x in X:
@@ -146,23 +147,24 @@ def load_pretrained_model(model_path, model_base, model_name, load_8bit=False, l
         if mm_use_x_start_end:
             for x in X:
                 tokenizer.add_tokens([DEFAULT_X_START_TOKEN[x.upper()], DEFAULT_X_END_TOKEN[x.upper()]], special_tokens=True)
+        '''
         model.resize_token_embeddings(len(tokenizer))
         print(X)    
-        if 'Image' in X:
-            image_tower = model.get_image_tower()
-            if not image_tower.is_loaded:
-                image_tower.load_model()
-            image_tower.to(device=device, dtype=torch.float16)
-            image_processor = image_tower.image_processor
-            processor['image'] = image_processor
+        #if 'Image' in X:
+        image_tower = model.get_image_tower()
+        if not image_tower.is_loaded:
+            image_tower.load_model()
+        image_tower.to(device=device, dtype=torch.float16)
+        image_processor = image_tower.image_processor
+        processor['image'] = image_processor
 
-        if 'Video' in X:
-            video_tower = model.get_video_tower()
-            if not video_tower.is_loaded:
-                video_tower.load_model()
-            video_tower.to(device=device, dtype=torch.float16)
-            video_processor = video_tower.video_processor
-            processor['video'] = video_processor
+        #if 'Video' in X:
+        video_tower = model.get_video_tower()
+        if not video_tower.is_loaded:
+            video_tower.load_model()
+        video_tower.to(device=device, dtype=torch.float16)
+        video_processor = video_tower.video_processor
+        processor['video'] = video_processor
 
     if hasattr(model.config, "max_sequence_length"):
         context_len = model.config.max_sequence_length