Update OpusparcusPC & LLM2Vec

app.py

@@ -340,13 +340,13 @@ EXTERNAL_MODELS = [
     "Cohere-embed-multilingual-light-v3.0",
     "DanskBERT",
     "LASER2",
+    "LLM2Vec-Llama-supervised",
+    "LLM2Vec-Llama-unsupervised",
+    "LLM2Vec-Mistral-supervised",
+    "LLM2Vec-Mistral-unsupervised",
+    "LLM2Vec-Sheared-Llama-supervised",
+    "LLM2Vec-Sheared-Llama-unsupervised",
     "LaBSE",
-#    "LLM2Vec-Llama-supervised",
-#    "LLM2Vec-Llama-unsupervised",
-#    "LLM2Vec-Mistral-supervised",
-#    "LLM2Vec-Mistral-unsupervised",
-#    "LLM2Vec-Sheared-Llama-supervised",
-#    "LLM2Vec-Sheared-Llama-unsupervised",
     "OpenSearch-text-hybrid",
     "all-MiniLM-L12-v2",
     "all-MiniLM-L6-v2",
@@ -456,21 +456,25 @@ EXTERNAL_MODELS = [
 ]
 
 EXTERNAL_MODEL_TO_LINK = {
+    "Baichuan-text-embedding": "https://platform.baichuan-ai.com/docs/text-Embedding",
     "Cohere-embed-english-v3.0": "https://huggingface.co/Cohere/Cohere-embed-english-v3.0",
     "Cohere-embed-multilingual-v3.0": "https://huggingface.co/Cohere/Cohere-embed-multilingual-v3.0",
     "Cohere-embed-multilingual-light-v3.0": "https://huggingface.co/Cohere/Cohere-embed-multilingual-light-v3.0",
+    "DanskBERT": "https://huggingface.co/vesteinn/DanskBERT",
+    "LASER2": "https://github.com/facebookresearch/LASER",
     "LLM2Vec-Llama-supervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp-supervised",
     "LLM2Vec-Llama-unsupervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Llama-2-7b-chat-hf-mntp",
     "LLM2Vec-Mistral-supervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp-supervised",
     "LLM2Vec-Mistral-unsupervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Mistral-7B-Instruct-v2-mntp",
     "LLM2Vec-Sheared-Llama-supervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp-supervised",
     "LLM2Vec-Sheared-Llama-unsupervised": "https://huggingface.co/McGill-NLP/LLM2Vec-Sheared-LLaMA-mntp",
+    "LaBSE": "https://huggingface.co/sentence-transformers/LaBSE",
+    "OpenSearch-text-hybrid": "https://help.aliyun.com/zh/open-search/vector-search-edition/hybrid-retrieval",
     "allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter",
     "allenai-specter": "https://huggingface.co/sentence-transformers/allenai-specter",
     "all-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L12-v2",
     "all-MiniLM-L6-v2": "https://huggingface.co/sentence-transformers/all-MiniLM-L6-v2",
     "all-mpnet-base-v2": "https://huggingface.co/sentence-transformers/all-mpnet-base-v2",
-    "Baichuan-text-embedding": "https://platform.baichuan-ai.com/docs/text-Embedding",
     "bert-base-10lang-cased": "https://huggingface.co/Geotrend/bert-base-10lang-cased",
     "bert-base-15lang-cased": "https://huggingface.co/Geotrend/bert-base-15lang-cased",
     "bert-base-25lang-cased": "https://huggingface.co/Geotrend/bert-base-25lang-cased",
@@ -487,7 +491,6 @@ EXTERNAL_MODEL_TO_LINK = {
     "camembert-large": "https://huggingface.co/almanach/camembert-large",
     "contriever-base-msmarco": "https://huggingface.co/nthakur/contriever-base-msmarco",
     "cross-en-de-roberta-sentence-transformer": "https://huggingface.co/T-Systems-onsite/cross-en-de-roberta-sentence-transformer",
-    "DanskBERT": "https://huggingface.co/vesteinn/DanskBERT",
     "distilbert-base-25lang-cased": "https://huggingface.co/Geotrend/distilbert-base-25lang-cased",
     "distilbert-base-en-fr-cased": "https://huggingface.co/Geotrend/distilbert-base-en-fr-cased",
     "distilbert-base-en-fr-es-pt-it-cased": "https://huggingface.co/Geotrend/distilbert-base-en-fr-es-pt-it-cased",
@@ -520,8 +523,6 @@ EXTERNAL_MODEL_TO_LINK = {
     "herbert-base-retrieval-v2": "https://huggingface.co/ipipan/herbert-base-retrieval-v2",
     "komninos": "https://huggingface.co/sentence-transformers/average_word_embeddings_komninos",
     "luotuo-bert-medium": "https://huggingface.co/silk-road/luotuo-bert-medium",
-    "LASER2": "https://github.com/facebookresearch/LASER",
-    "LaBSE": "https://huggingface.co/sentence-transformers/LaBSE",
     "m3e-base": "https://huggingface.co/moka-ai/m3e-base",
     "m3e-large": "https://huggingface.co/moka-ai/m3e-large",
     "mistral-embed": "https://docs.mistral.ai/guides/embeddings",
@@ -538,7 +539,6 @@ EXTERNAL_MODEL_TO_LINK = {
     "nomic-embed-text-v1.5-512": "https://huggingface.co/nomic-ai/nomic-embed-text-v1.5",
     "norbert3-base": "https://huggingface.co/ltg/norbert3-base",
     "norbert3-large": "https://huggingface.co/ltg/norbert3-large",
-    "OpenSearch-text-hybrid": "https://help.aliyun.com/zh/open-search/vector-search-edition/hybrid-retrieval",
     "paraphrase-multilingual-mpnet-base-v2": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-mpnet-base-v2",    
     "paraphrase-multilingual-MiniLM-L12-v2": "https://huggingface.co/sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2",
     "sentence-camembert-base": "https://huggingface.co/dangvantuan/sentence-camembert-base",
@@ -586,20 +586,23 @@ EXTERNAL_MODEL_TO_LINK = {
 }
 
 EXTERNAL_MODEL_TO_DIM = {
+    "Baichuan-text-embedding": 1024,
     "Cohere-embed-english-v3.0": 1024,
     "Cohere-embed-multilingual-v3.0": 1024,
     "Cohere-embed-multilingual-light-v3.0": 384,
+    "DanskBERT": 768,
+    "LASER2": 1024,
     "LLM2Vec-Llama-supervised": 4096,
     "LLM2Vec-Llama-unsupervised": 4096,
     "LLM2Vec-Mistral-supervised": 4096,
     "LLM2Vec-Mistral-unsupervised": 4096,
     "LLM2Vec-Sheared-Llama-supervised": 2048,
     "LLM2Vec-Sheared-Llama-unsupervised": 2048,
+    "LaBSE": 768,
     "all-MiniLM-L12-v2": 384,
     "all-MiniLM-L6-v2": 384,
     "all-mpnet-base-v2": 768,
     "allenai-specter": 768,
-    "Baichuan-text-embedding": 1024,
     "bert-base-10lang-cased": 768,
     "bert-base-15lang-cased": 768,
     "bert-base-25lang-cased": 768,
@@ -616,7 +619,6 @@ EXTERNAL_MODEL_TO_DIM = {
     "camembert-large": 768,
     "contriever-base-msmarco": 768,
     "cross-en-de-roberta-sentence-transformer": 768,
-    "DanskBERT": 768,
     "distilbert-base-25lang-cased": 768,
     "distilbert-base-en-fr-cased": 768,
     "distilbert-base-en-fr-es-pt-it-cased": 768,
@@ -635,8 +637,6 @@ EXTERNAL_MODEL_TO_DIM = {
     "flaubert_base_uncased": 768,
     "flaubert_large_cased": 1024,
     "luotuo-bert-medium": 768,
-    "LASER2": 1024,
-    "LaBSE": 768,
     "gbert-base": 768,
     "gbert-large": 1024,
     "gelectra-base": 768,
@@ -715,20 +715,23 @@ EXTERNAL_MODEL_TO_DIM = {
 }
 
 EXTERNAL_MODEL_TO_SEQLEN = {
+    "Baichuan-text-embedding": 512,
     "Cohere-embed-english-v3.0": 512,
     "Cohere-embed-multilingual-v3.0": 512,
-    "Cohere-embed-multilingual-light-v3.0": 512,    
+    "Cohere-embed-multilingual-light-v3.0": 512,
+    "DanskBERT": 514,
+    "LASER2": "N/A",
     "LLM2Vec-Llama-supervised": 512,
     "LLM2Vec-Llama-unsupervised": 512,
     "LLM2Vec-Mistral-supervised": 512,
     "LLM2Vec-Mistral-unsupervised": 512,
     "LLM2Vec-Sheared-Llama-supervised": 512,
     "LLM2Vec-Sheared-Llama-unsupervised": 512,    
+    "LaBSE": 512,
     "all-MiniLM-L12-v2": 512,
     "all-MiniLM-L6-v2": 512,
     "all-mpnet-base-v2": 514,
     "allenai-specter": 512,
-    "Baichuan-text-embedding": 512,
     "bert-base-10lang-cased": 512,
     "bert-base-15lang-cased": 512,
     "bert-base-25lang-cased": 512,
@@ -749,8 +752,7 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "distilbert-base-en-fr-cased": 512,
     "distilbert-base-en-fr-es-pt-it-cased": 512,
     "distilbert-base-fr-cased": 512,
-    "distilbert-base-uncased": 512,    
-    "DanskBERT": 514,
+    "distilbert-base-uncased": 512,
     "dfm-encoder-large-v1": 512,
     "dfm-sentence-encoder-large-1": 512,
     "distiluse-base-multilingual-cased-v2": 512,
@@ -778,8 +780,6 @@ EXTERNAL_MODEL_TO_SEQLEN = {
     "herbert-base-retrieval-v2": 514,
     "komninos": "N/A",
     "luotuo-bert-medium": 512,
-    "LASER2": "N/A",
-    "LaBSE": 512,
     "m3e-base": 512,
     "m3e-large": 512,
 #    "mistral-embed": "?",
@@ -844,12 +844,15 @@ EXTERNAL_MODEL_TO_SEQLEN = {
 }
 
 EXTERNAL_MODEL_TO_SIZE = {
+    "DanskBERT": 125,
+    "LASER2": 43,
     "LLM2Vec-Llama-supervised": 6607,
     "LLM2Vec-Llama-unsupervised": 6607,
     "LLM2Vec-Mistral-supervised": 7111,
     "LLM2Vec-Mistral-unsupervised": 7111,
     "LLM2Vec-Sheared-Llama-supervised": 1280,
     "LLM2Vec-Sheared-Llama-unsupervised": 1280,
+    "LaBSE": 471,
     "allenai-specter": 110,
     "all-MiniLM-L12-v2": 33,
     "all-MiniLM-L6-v2": 23,
@@ -874,7 +877,6 @@ EXTERNAL_MODEL_TO_SIZE = {
     "distilbert-base-en-fr-es-pt-it-cased": 110,
     "distilbert-base-fr-cased": 110,
     "distilbert-base-uncased": 110,
-    "DanskBERT": 125,
     "distiluse-base-multilingual-cased-v2": 135,
     "dfm-encoder-large-v1": 355,
     "dfm-sentence-encoder-large-1": 355,
@@ -901,9 +903,7 @@ EXTERNAL_MODEL_TO_SIZE = {
     "gtr-t5-xxl": 4865,
     "herbert-base-retrieval-v2": 125,
     "komninos": 134,
-    "luotuo-bert-medium": 328,    
-    "LASER2": 43,
-    "LaBSE": 471,
+    "luotuo-bert-medium": 328,
     "m3e-base": 102,
     "m3e-large": 102,
     "msmarco-bert-co-condensor": 110,
@@ -944,12 +944,12 @@ EXTERNAL_MODEL_TO_SIZE = {
 }
 
 PROPRIETARY_MODELS = {
+    "Baichuan-text-embedding",
     "Cohere-embed-english-v3.0",
     "Cohere-embed-multilingual-v3.0",
     "Cohere-embed-multilingual-light-v3.0",
-    "Baichuan-text-embedding",
-    "mistral-embed",
     "OpenSearch-text-hybrid",
+    "mistral-embed",
     "text-embedding-3-small",
     "text-embedding-3-large",
     "text-embedding-3-large-256",
@@ -973,6 +973,7 @@ PROPRIETARY_MODELS = {
     "google-gecko.text-embedding-preview-0409",
     "google-gecko-256.text-embedding-preview-0409",
 }
+
 PROPRIETARY_MODELS = {
     make_clickable_model(model, link=EXTERNAL_MODEL_TO_LINK.get(model, "https://huggingface.co/spaces/mteb/leaderboard"))
     for model in PROPRIETARY_MODELS