fix(pad zero) pad intermediate_size to 29696 to make sure quantized model can use 8 tensor-parallel in vllm

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

config.json

@@ -1,20 +1,16 @@
 {
-  "_name_or_path": "Qwen/Qwen2-VL-72B-Instruct-AWQ",
+  "_name_or_path": "Qwen/Qwen2-VL-72B-Instruct-315-fix-dim-pad",
   "architectures": [
     "Qwen2VLForConditionalGeneration"
   ],
   "attention_dropout": 0.0,
   "bos_token_id": 151643,
   "eos_token_id": 151645,
-  "vision_start_token_id": 151652,
-  "vision_end_token_id": 151653,
-  "vision_token_id": 151654,
-  "image_token_id": 151655,
-  "video_token_id": 151656,
   "hidden_act": "silu",
   "hidden_size": 8192,
+  "image_token_id": 151655,
   "initializer_range": 0.02,
-  "intermediate_size": 29568,
+  "intermediate_size": 29696,
   "max_position_embeddings": 32768,
   "max_window_layers": 80,
   "model_type": "qwen2_vl",
@@ -47,17 +43,15 @@
   "transformers_version": "4.45.0.dev0",
   "use_cache": true,
   "use_sliding_window": false,
+  "video_token_id": 151656,
   "vision_config": {
-    "depth": 32,
-    "embed_dim": 1280,
-    "mlp_ratio": 4,
-    "num_heads": 16,
-    "in_chans": 3,
     "hidden_size": 8192,
-    "patch_size": 14,
-    "spatial_merge_size": 2,
-    "spatial_patch_size": 14,
-    "temporal_patch_size": 2
+    "in_chans": 3,
+    "model_type": "qwen2_vl",
+    "spatial_patch_size": 14
   },
+  "vision_end_token_id": 151653,
+  "vision_start_token_id": 151652,
+  "vision_token_id": 151654,
   "vocab_size": 152064
 }

generation_config.json

@@ -1,13 +1,11 @@
 {
-  "bos_token_id": 151643,
-  "pad_token_id": 151643,
+  "chat_format": "chatml",
   "do_sample": true,
-  "eos_token_id": [
-    151645,
-    151643
-  ],
-  "repetition_penalty": 1.05,
-  "top_p": 0.001,
-  "top_k": 1,
+  "eos_token_id": 151643,
+  "max_new_tokens": 512,
+  "max_window_size": 6144,
+  "pad_token_id": 151643,
+  "top_k": 0,
+  "top_p": 0.01,
   "transformers_version": "4.45.0.dev0"
 }

model-00001-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fb053403e32f0f170f1cf34c593f32c6277cb8c128dca6d13d96b0a5e5488d05
-size 3968639512
+oid sha256:bdacf90b517828c5ff867ea246182475ef74bdc71e1f94bccb363a8e84163b93
+size 1221591040

model-00002-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:833322e4ad4577970c2780fc680c1032fbe111377e50fd3282a74b1f18ec26fb
-size 3899899496
+oid sha256:ae13b4b0707ae2d71a2dc438ce31ab561edd772990e874bf7d3a65a007c23f8c
+size 3914063464

model-00003-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d4abd7986db5540fa8983232efd26ef51b7cf2ecbe2517b2171902dc07a0aade
-size 3978401464
+oid sha256:7fcf53da5c2ff57ea36d9267025b50cf1567d3a6e2cab30361118078d494e910
+size 3992565432

model-00004-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:fe2cba911774e1791578d3243d35a1288106ccface9cc1d742ba5a62d766811c
-size 3978401504
+oid sha256:8f8e5d6ebd7faf0358ffd638d3ad82ac69a4da852d7677aa2b1ba3ef8d4436d2
+size 3992565472

model-00005-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:d95c1e33983b475b72e1273900f63a4a9fd50fc004f650cd424e2c21256e616b
-size 3899899712
+oid sha256:fc264ea4aedcd0699307e1e66981edb6c5a3fdb06e7d4e193b0a01590b351878
+size 3914063680

model-00006-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:ab50438ebcb0cd8f49bbfa6e7451a7e2663b683858488a122b485c0d93fc0579
-size 3978401504
+oid sha256:99aa915bcd142038bb0369663c8af65a5870a37cc198ad0b6e87786a4cec0c46
+size 3992565472

model-00007-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:9e07762ad5a708f1d997eb9eb4a345ae8c4e587448f7585f5c9da4c9d82f0cfe
-size 3978401504
+oid sha256:5d02e7c1fa0a6aca98eb643a2df2c6b483d108dad5e9b7c9b5fd2bdbdc2a5db6
+size 3992565472

model-00008-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:7482977415c7e31d41f103c8f71a4f34dd2abeb49913d533d27ad3d708b52e35
-size 3899899712
+oid sha256:cd4076e8117ce301d7488c2707cc5dea92e726956fd4060b38087d293405b030
+size 3914063680

model-00009-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:a701214ea01fb1a5b3f6ebf13be0afa140b8b74a79b9c4da5efb3e8d0f80ac64
-size 3978401504
+oid sha256:0d0a5d19007565c00522f63e98b7fdcf8e81b8775eedecfeccd1a6161d2aa45f
+size 3992565472

model-00010-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:52d156b63833d447c6022c3b6ef7b36e7295228ccf348c26b640d3d8ada95476
-size 3978401504
+oid sha256:bfcd19e2a82637baeb21a811b49824ccc1bc657f04e57c1161d3c6eacd1b36f8
+size 3992565472

model-00011-of-00011.safetensors

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:f68308b31bda9a52853a75565f701676c5869be3ce8dc64d9c5e8dc46a783b91
-size 3325018408
+oid sha256:24b421ae6c69eef5024b72024cbe743bcc3546b99a068ff5bf1828cbf1f3dd5a
+size 3328287016

model.safetensors.index.json

@@ -1,6 +1,6 @@
 {
     "metadata": {
-        "total_size": 42863488000
+        "total_size": 42994232320
     },
     "weight_map": {
         "visual.patch_embed.proj.weight": "model-00001-of-00011.safetensors",