Deploying SHL chatbot to Hugging Face Spaces

.devcontainer/devcontainer.json

@@ -0,0 +1,33 @@
+{
+  "name": "Python 3",
+  // Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+  "image": "mcr.microsoft.com/devcontainers/python:1-3.11-bullseye",
+  "customizations": {
+    "codespaces": {
+      "openFiles": [
+        "README.md",
+        "main.py"
+      ]
+    },
+    "vscode": {
+      "settings": {},
+      "extensions": [
+        "ms-python.python",
+        "ms-python.vscode-pylance"
+      ]
+    }
+  },
+  "updateContentCommand": "[ -f packages.txt ] && sudo apt update && sudo apt upgrade -y && sudo xargs apt install -y <packages.txt; [ -f requirements.txt ] && pip3 install --user -r requirements.txt; pip3 install --user streamlit; echo '✅ Packages installed and Requirements met'",
+  "postAttachCommand": {
+    "server": "streamlit run main.py --server.enableCORS false --server.enableXsrfProtection false"
+  },
+  "portsAttributes": {
+    "8501": {
+      "label": "Application",
+      "onAutoForward": "openPreview"
+    }
+  },
+  "forwardPorts": [
+    8501
+  ]
+}

README.md

@@ -1,13 +1,2 @@
----
-title: Rag Chatbot
-emoji: 👁
-colorFrom: red
-colorTo: yellow
-sdk: streamlit
-sdk_version: 1.44.1
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+This is a prototype chatbot. 
+To run it you need to create .env file with your GROQ_API_KEY and OPEN_API_KEY

Web_Scraper.py

@@ -0,0 +1,121 @@
+from selenium import webdriver
+from selenium.webdriver.chrome.service import Service
+from selenium.webdriver.chrome.options import Options
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.common.exceptions import TimeoutException, NoSuchElementException
+from webdriver_manager.chrome import ChromeDriverManager
+import pandas as pd
+import time
+from urllib.parse import urljoin
+
+
+def scrape_shl_products():
+    # Configure Chrome options
+    chrome_options = Options()
+    chrome_options.add_argument("--headless")  # Optional: Run in background
+    chrome_options.add_argument("--disable-blink-features=AutomationControlled")
+    chrome_options.add_argument(
+        "user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36"
+    )
+
+    # Set up driver
+    service = Service(ChromeDriverManager().install())
+    driver = webdriver.Chrome(service=service, options=chrome_options)
+
+    base_url = "https://www.shl.com"
+    catalog_url = "https://www.shl.com/solutions/products/product-catalog/"
+
+    try:
+        print("Loading SHL product catalog...")
+        driver.get(catalog_url)
+
+        # Wait for products to load
+        WebDriverWait(driver, 15).until(
+            EC.presence_of_element_located((By.CSS_SELECTOR, ".product-card"))
+        )
+
+        # Scroll to load all products
+        print("Scrolling to load all products...")
+        last_height = driver.execute_script("return document.body.scrollHeight")
+        while True:
+            driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
+            time.sleep(2)
+            new_height = driver.execute_script("return document.body.scrollHeight")
+            if new_height == last_height:
+                break
+            last_height = new_height
+
+        product_cards = driver.find_elements(By.CSS_SELECTOR, ".product-card")
+        print(f"Found {len(product_cards)} products.")
+
+        products = []
+
+        for card in product_cards:
+            try:
+                product = {
+                    'Assessment Name': 'Not found',
+                    'URL': 'Not found',
+                    'Remote Testing Support': 'No',
+                    'Adaptive/IRT Support': 'No',
+                    'Duration': 'Not specified',
+                    'Test Type': 'Not specified'
+                }
+
+                # Name
+                name_element = card.find_element(By.CSS_SELECTOR, ".product-card__title")
+                product['Assessment Name'] = name_element.text
+
+                # URL
+                link_element = card.find_element(By.CSS_SELECTOR, "a[href]")
+                product['URL'] = urljoin(base_url, link_element.get_attribute("href"))
+
+                # Metadata
+                meta_items = card.find_elements(By.CSS_SELECTOR, ".product-card__meta-item")
+                for item in meta_items:
+                    try:
+                        label = item.find_element(By.CSS_SELECTOR, ".product-card__meta-label").text.lower()
+                        value = item.find_element(By.CSS_SELECTOR, ".product-card__meta-value").text
+
+                        if 'remote' in label:
+                            product['Remote Testing Support'] = 'Yes' if 'yes' in value.lower() else 'No'
+                        elif 'adaptive' in label or 'irt' in label:
+                            product['Adaptive/IRT Support'] = 'Yes' if 'yes' in value.lower() else 'No'
+                        elif 'duration' in label:
+                            product['Duration'] = value
+                        elif 'type' in label:
+                            product['Test Type'] = value
+                    except NoSuchElementException:
+                        continue
+
+                products.append(product)
+
+            except Exception as e:
+                print(f"Error processing a product card: {str(e)}")
+                continue
+
+        # Save data
+        df = pd.DataFrame(products)
+        df.to_csv('shl_products.csv', index=False)
+        print("Data saved to shl_products.csv")
+
+        return df
+
+    except TimeoutException:
+        print("Timeout loading the page.")
+    except Exception as e:
+        print(f"An error occurred: {str(e)}")
+    finally:
+        driver.quit()
+        print("Browser closed.")
+
+
+if __name__ == "__main__":
+    print("Starting SHL scraper...")   # Debug print
+    df = scrape_shl_products()
+    if df is not None and not df.empty:
+        print("\nFirst 5 results:")
+        print(df.head())
+    else:
+        print("No data scraped.")

all-MiniLM-L6-v2/.gitattributes

@@ -0,0 +1,28 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bin.* filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text 
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zstandard filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text
+model.safetensors filter=lfs diff=lfs merge=lfs -text

all-MiniLM-L6-v2/1_Pooling/config.json

@@ -0,0 +1,7 @@
+{
+  "word_embedding_dimension": 384,
+  "pooling_mode_cls_token": false,
+  "pooling_mode_mean_tokens": true,
+  "pooling_mode_max_tokens": false,
+  "pooling_mode_mean_sqrt_len_tokens": false
+}

all-MiniLM-L6-v2/README.md

@@ -0,0 +1,173 @@
+---
+language: en
+license: apache-2.0
+library_name: sentence-transformers
+tags:
+- sentence-transformers
+- feature-extraction
+- sentence-similarity
+- transformers
+datasets:
+- s2orc
+- flax-sentence-embeddings/stackexchange_xml
+- ms_marco
+- gooaq
+- yahoo_answers_topics
+- code_search_net
+- search_qa
+- eli5
+- snli
+- multi_nli
+- wikihow
+- natural_questions
+- trivia_qa
+- embedding-data/sentence-compression
+- embedding-data/flickr30k-captions
+- embedding-data/altlex
+- embedding-data/simple-wiki
+- embedding-data/QQP
+- embedding-data/SPECTER
+- embedding-data/PAQ_pairs
+- embedding-data/WikiAnswers
+pipeline_tag: sentence-similarity
+---
+
+
+# all-MiniLM-L6-v2
+This is a [sentence-transformers](https://www.SBERT.net) model: It maps sentences & paragraphs to a 384 dimensional dense vector space and can be used for tasks like clustering or semantic search.
+
+## Usage (Sentence-Transformers)
+Using this model becomes easy when you have [sentence-transformers](https://www.SBERT.net) installed:
+
+```
+pip install -U sentence-transformers
+```
+
+Then you can use the model like this:
+```python
+from sentence_transformers import SentenceTransformer
+sentences = ["This is an example sentence", "Each sentence is converted"]
+
+model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+embeddings = model.encode(sentences)
+print(embeddings)
+```
+
+## Usage (HuggingFace Transformers)
+Without [sentence-transformers](https://www.SBERT.net), you can use the model like this: First, you pass your input through the transformer model, then you have to apply the right pooling-operation on-top of the contextualized word embeddings.
+
+```python
+from transformers import AutoTokenizer, AutoModel
+import torch
+import torch.nn.functional as F
+
+#Mean Pooling - Take attention mask into account for correct averaging
+def mean_pooling(model_output, attention_mask):
+    token_embeddings = model_output[0] #First element of model_output contains all token embeddings
+    input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+    return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+
+# Sentences we want sentence embeddings for
+sentences = ['This is an example sentence', 'Each sentence is converted']
+
+# Load model from HuggingFace Hub
+tokenizer = AutoTokenizer.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+model = AutoModel.from_pretrained('sentence-transformers/all-MiniLM-L6-v2')
+
+# Tokenize sentences
+encoded_input = tokenizer(sentences, padding=True, truncation=True, return_tensors='pt')
+
+# Compute token embeddings
+with torch.no_grad():
+    model_output = model(**encoded_input)
+
+# Perform pooling
+sentence_embeddings = mean_pooling(model_output, encoded_input['attention_mask'])
+
+# Normalize embeddings
+sentence_embeddings = F.normalize(sentence_embeddings, p=2, dim=1)
+
+print("Sentence embeddings:")
+print(sentence_embeddings)
+```
+
+------
+
+## Background
+
+The project aims to train sentence embedding models on very large sentence level datasets using a self-supervised 
+contrastive learning objective. We used the pretrained [`nreimers/MiniLM-L6-H384-uncased`](https://huggingface.co/nreimers/MiniLM-L6-H384-uncased) model and fine-tuned in on a 
+1B sentence pairs dataset. We use a contrastive learning objective: given a sentence from the pair, the model should predict which out of a set of randomly sampled other sentences, was actually paired with it in our dataset.
+
+We developed this model during the 
+[Community week using JAX/Flax for NLP & CV](https://discuss.huggingface.co/t/open-to-the-community-community-week-using-jax-flax-for-nlp-cv/7104), 
+organized by Hugging Face. We developed this model as part of the project:
+[Train the Best Sentence Embedding Model Ever with 1B Training Pairs](https://discuss.huggingface.co/t/train-the-best-sentence-embedding-model-ever-with-1b-training-pairs/7354). We benefited from efficient hardware infrastructure to run the project: 7 TPUs v3-8, as well as intervention from Googles Flax, JAX, and Cloud team member about efficient deep learning frameworks.
+
+## Intended uses
+
+Our model is intended to be used as a sentence and short paragraph encoder. Given an input text, it outputs a vector which captures 
+the semantic information. The sentence vector may be used for information retrieval, clustering or sentence similarity tasks.
+
+By default, input text longer than 256 word pieces is truncated.
+
+
+## Training procedure
+
+### Pre-training 
+
+We use the pretrained [`nreimers/MiniLM-L6-H384-uncased`](https://huggingface.co/nreimers/MiniLM-L6-H384-uncased) model. Please refer to the model card for more detailed information about the pre-training procedure.
+
+### Fine-tuning 
+
+We fine-tune the model using a contrastive objective. Formally, we compute the cosine similarity from each possible sentence pairs from the batch.
+We then apply the cross entropy loss by comparing with true pairs.
+
+#### Hyper parameters
+
+We trained our model on a TPU v3-8. We train the model during 100k steps using a batch size of 1024 (128 per TPU core).
+We use a learning rate warm up of 500. The sequence length was limited to 128 tokens. We used the AdamW optimizer with
+a 2e-5 learning rate. The full training script is accessible in this current repository: `train_script.py`.
+
+#### Training data
+
+We use the concatenation from multiple datasets to fine-tune our model. The total number of sentence pairs is above 1 billion sentences.
+We sampled each dataset given a weighted probability which configuration is detailed in the `data_config.json` file.
+
+
+| Dataset                                                  | Paper                                    | Number of training tuples  |
+|--------------------------------------------------------|:----------------------------------------:|:--------------------------:|
+| [Reddit comments (2015-2018)](https://github.com/PolyAI-LDN/conversational-datasets/tree/master/reddit) | [paper](https://arxiv.org/abs/1904.06472) | 726,484,430 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Abstracts) | [paper](https://aclanthology.org/2020.acl-main.447/) | 116,288,806 |
+| [WikiAnswers](https://github.com/afader/oqa#wikianswers-corpus) Duplicate question pairs | [paper](https://doi.org/10.1145/2623330.2623677) | 77,427,422 |
+| [PAQ](https://github.com/facebookresearch/PAQ) (Question, Answer) pairs | [paper](https://arxiv.org/abs/2102.07033) | 64,371,441 |
+| [S2ORC](https://github.com/allenai/s2orc) Citation pairs (Titles) | [paper](https://aclanthology.org/2020.acl-main.447/) | 52,603,982 |
+| [S2ORC](https://github.com/allenai/s2orc) (Title, Abstract) | [paper](https://aclanthology.org/2020.acl-main.447/) | 41,769,185 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Body) pairs  | - | 25,316,456 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title+Body, Answer) pairs  | - | 21,396,559 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) (Title, Answer) pairs  | - | 21,396,559 |
+| [MS MARCO](https://microsoft.github.io/msmarco/) triplets | [paper](https://doi.org/10.1145/3404835.3462804) | 9,144,553 |
+| [GOOAQ: Open Question Answering with Diverse Answer Types](https://github.com/allenai/gooaq) | [paper](https://arxiv.org/pdf/2104.08727.pdf) | 3,012,496 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 1,198,260 |
+| [Code Search](https://huggingface.co/datasets/code_search_net) | - | 1,151,414 |
+| [COCO](https://cocodataset.org/#home) Image captions | [paper](https://link.springer.com/chapter/10.1007%2F978-3-319-10602-1_48) | 828,395|
+| [SPECTER](https://github.com/allenai/specter) citation triplets | [paper](https://doi.org/10.18653/v1/2020.acl-main.207) | 684,100 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Question, Answer) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 681,164 |
+| [Yahoo Answers](https://www.kaggle.com/soumikrakshit/yahoo-answers-dataset) (Title, Question) | [paper](https://proceedings.neurips.cc/paper/2015/hash/250cf8b51c773f3f8dc8b4be867a9a02-Abstract.html) | 659,896 |
+| [SearchQA](https://huggingface.co/datasets/search_qa) | [paper](https://arxiv.org/abs/1704.05179) | 582,261 |
+| [Eli5](https://huggingface.co/datasets/eli5) | [paper](https://doi.org/10.18653/v1/p19-1346) | 325,475 |
+| [Flickr 30k](https://shannon.cs.illinois.edu/DenotationGraph/) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/229/33) | 317,695 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles) | | 304,525 |
+| AllNLI ([SNLI](https://nlp.stanford.edu/projects/snli/) and [MultiNLI](https://cims.nyu.edu/~sbowman/multinli/) | [paper SNLI](https://doi.org/10.18653/v1/d15-1075), [paper MultiNLI](https://doi.org/10.18653/v1/n18-1101) | 277,230 | 
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (bodies) | | 250,519 |
+| [Stack Exchange](https://huggingface.co/datasets/flax-sentence-embeddings/stackexchange_xml) Duplicate questions (titles+bodies) | | 250,460 |
+| [Sentence Compression](https://github.com/google-research-datasets/sentence-compression) | [paper](https://www.aclweb.org/anthology/D13-1155/) | 180,000 |
+| [Wikihow](https://github.com/pvl/wikihow_pairs_dataset) | [paper](https://arxiv.org/abs/1810.09305) | 128,542 |
+| [Altlex](https://github.com/chridey/altlex/) | [paper](https://aclanthology.org/P16-1135.pdf) | 112,696 |
+| [Quora Question Triplets](https://quoradata.quora.com/First-Quora-Dataset-Release-Question-Pairs) | - | 103,663 |
+| [Simple Wikipedia](https://cs.pomona.edu/~dkauchak/simplification/) | [paper](https://www.aclweb.org/anthology/P11-2117/) | 102,225 |
+| [Natural Questions (NQ)](https://ai.google.com/research/NaturalQuestions) | [paper](https://transacl.org/ojs/index.php/tacl/article/view/1455) | 100,231 |
+| [SQuAD2.0](https://rajpurkar.github.io/SQuAD-explorer/) | [paper](https://aclanthology.org/P18-2124.pdf) | 87,599 |
+| [TriviaQA](https://huggingface.co/datasets/trivia_qa) | - | 73,346 |
+| **Total** | | **1,170,060,424** |

all-MiniLM-L6-v2/config.json

@@ -0,0 +1,24 @@
+{
+  "_name_or_path": "nreimers/MiniLM-L6-H384-uncased",
+  "architectures": [
+    "BertModel"
+  ],
+  "attention_probs_dropout_prob": 0.1,
+  "gradient_checkpointing": false,
+  "hidden_act": "gelu",
+  "hidden_dropout_prob": 0.1,
+  "hidden_size": 384,
+  "initializer_range": 0.02,
+  "intermediate_size": 1536,
+  "layer_norm_eps": 1e-12,
+  "max_position_embeddings": 512,
+  "model_type": "bert",
+  "num_attention_heads": 12,
+  "num_hidden_layers": 6,
+  "pad_token_id": 0,
+  "position_embedding_type": "absolute",
+  "transformers_version": "4.8.2",
+  "type_vocab_size": 2,
+  "use_cache": true,
+  "vocab_size": 30522
+}

all-MiniLM-L6-v2/config_sentence_transformers.json

@@ -0,0 +1,7 @@
+{
+  "__version__": {
+    "sentence_transformers": "2.0.0",
+    "transformers": "4.6.1",
+    "pytorch": "1.8.1"
+  }
+}

all-MiniLM-L6-v2/data_config.json

@@ -0,0 +1,1452 @@
+[
+    {
+        "name": "stackexchange_title_body/skeptics.stackexchange.com.jsonl.gz",
+        "lines": 10009,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/islam.stackexchange.com.jsonl.gz",
+        "lines": 10052,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/islam.stackexchange.com.jsonl.gz",
+        "lines": 10052,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/anime.stackexchange.com.jsonl.gz",
+        "lines": 10131,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/anime.stackexchange.com.jsonl.gz",
+        "lines": 10131,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/writers.stackexchange.com.jsonl.gz",
+        "lines": 10157,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/astronomy.stackexchange.com.jsonl.gz",
+        "lines": 10462,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/vi.stackexchange.com.jsonl.gz",
+        "lines": 10551,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/french.stackexchange.com.jsonl.gz",
+        "lines": 10578,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/french.stackexchange.com.jsonl.gz",
+        "lines": 10578,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/cstheory.stackexchange.com.jsonl.gz",
+        "lines": 10642,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/civicrm.stackexchange.com.jsonl.gz",
+        "lines": 10648,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/civicrm.stackexchange.com.jsonl.gz",
+        "lines": 10648,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/expressionengine.stackexchange.com.jsonl.gz",
+        "lines": 10742,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/expressionengine.stackexchange.com.jsonl.gz",
+        "lines": 10742,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/engineering.stackexchange.com.jsonl.gz",
+        "lines": 10753,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/history.stackexchange.com.jsonl.gz",
+        "lines": 10766,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/history.stackexchange.com.jsonl.gz",
+        "lines": 10766,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/french.stackexchange.com.jsonl.gz",
+        "lines": 10794,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/politics.stackexchange.com.jsonl.gz",
+        "lines": 11047,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/politics.stackexchange.com.jsonl.gz",
+        "lines": 11047,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/economics.stackexchange.com.jsonl.gz",
+        "lines": 11115,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/craftcms.stackexchange.com.jsonl.gz",
+        "lines": 11236,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/craftcms.stackexchange.com.jsonl.gz",
+        "lines": 11236,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/anime.stackexchange.com.jsonl.gz",
+        "lines": 11444,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/christianity.stackexchange.com.jsonl.gz",
+        "lines": 11498,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/christianity.stackexchange.com.jsonl.gz",
+        "lines": 11498,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/softwarerecs.stackexchange.com.jsonl.gz",
+        "lines": 11761,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/softwarerecs.stackexchange.com.jsonl.gz",
+        "lines": 11761,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/boardgames.stackexchange.com.jsonl.gz",
+        "lines": 11805,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_Title_Answer/boardgames.stackexchange.com.jsonl.gz",
+        "lines": 11805,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/islam.stackexchange.com.jsonl.gz",
+        "lines": 11853,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/expressionengine.stackexchange.com.jsonl.gz",
+        "lines": 11866,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/politics.stackexchange.com.jsonl.gz",
+        "lines": 11894,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/history.stackexchange.com.jsonl.gz",
+        "lines": 12021,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/christianity.stackexchange.com.jsonl.gz",
+        "lines": 12108,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/boardgames.stackexchange.com.jsonl.gz",
+        "lines": 12149,
+        "weight": 1
+    },
+    {
+        "name": "flickr30k_captions.jsonl.gz",
+        "lines": 317695,
+        "weight": 1
+    },
+    {
+        "name": "coco_captions.jsonl.gz",
+        "lines": 828395,
+        "weight": 1
+    },
+    {
+        "name": "codesearchnet.jsonl.gz",
+        "lines": 1151414,
+        "weight": 1
+    },
+    {
+        "name": "stackexchange_title_body/civicrm.stackexchange.com.jsonl.gz",
+        "lines": 12543,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/craftcms.stackexchange.com.jsonl.gz",
+        "lines": 12574,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/networkengineering.stackexchange.com.jsonl.gz",
+        "lines": 12590,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/networkengineering.stackexchange.com.jsonl.gz",
+        "lines": 12590,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/space.stackexchange.com.jsonl.gz",
+        "lines": 12893,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/space.stackexchange.com.jsonl.gz",
+        "lines": 12893,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/quant.stackexchange.com.jsonl.gz",
+        "lines": 12933,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/quant.stackexchange.com.jsonl.gz",
+        "lines": 12933,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/philosophy.stackexchange.com.jsonl.gz",
+        "lines": 13114,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/philosophy.stackexchange.com.jsonl.gz",
+        "lines": 13114,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/gardening.stackexchange.com.jsonl.gz",
+        "lines": 13246,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/gardening.stackexchange.com.jsonl.gz",
+        "lines": 13246,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/hinduism.stackexchange.com.jsonl.gz",
+        "lines": 13450,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/networkengineering.stackexchange.com.jsonl.gz",
+        "lines": 13454,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/german.stackexchange.com.jsonl.gz",
+        "lines": 13733,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/german.stackexchange.com.jsonl.gz",
+        "lines": 13733,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/german.stackexchange.com.jsonl.gz",
+        "lines": 13950,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/philosophy.stackexchange.com.jsonl.gz",
+        "lines": 14829,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/gardening.stackexchange.com.jsonl.gz",
+        "lines": 15136,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/space.stackexchange.com.jsonl.gz",
+        "lines": 15142,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/bicycles.stackexchange.com.jsonl.gz",
+        "lines": 15708,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/bicycles.stackexchange.com.jsonl.gz",
+        "lines": 15708,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/law.stackexchange.com.jsonl.gz",
+        "lines": 16133,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/law.stackexchange.com.jsonl.gz",
+        "lines": 16133,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/arduino.stackexchange.com.jsonl.gz",
+        "lines": 16281,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/arduino.stackexchange.com.jsonl.gz",
+        "lines": 16281,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/bicycles.stackexchange.com.jsonl.gz",
+        "lines": 16353,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/emacs.stackexchange.com.jsonl.gz",
+        "lines": 16830,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/emacs.stackexchange.com.jsonl.gz",
+        "lines": 16830,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/quant.stackexchange.com.jsonl.gz",
+        "lines": 17261,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/dsp.stackexchange.com.jsonl.gz",
+        "lines": 17430,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/dsp.stackexchange.com.jsonl.gz",
+        "lines": 17430,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/puzzling.stackexchange.com.jsonl.gz",
+        "lines": 17448,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/puzzling.stackexchange.com.jsonl.gz",
+        "lines": 17448,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/puzzling.stackexchange.com.jsonl.gz",
+        "lines": 17851,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/law.stackexchange.com.jsonl.gz",
+        "lines": 17941,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/movies.stackexchange.com.jsonl.gz",
+        "lines": 18243,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/movies.stackexchange.com.jsonl.gz",
+        "lines": 18243,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/mechanics.stackexchange.com.jsonl.gz",
+        "lines": 18613,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/mechanics.stackexchange.com.jsonl.gz",
+        "lines": 18613,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/aviation.stackexchange.com.jsonl.gz",
+        "lines": 18755,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/aviation.stackexchange.com.jsonl.gz",
+        "lines": 18755,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/biology.stackexchange.com.jsonl.gz",
+        "lines": 19277,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/biology.stackexchange.com.jsonl.gz",
+        "lines": 19277,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/crypto.stackexchange.com.jsonl.gz",
+        "lines": 19404,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/crypto.stackexchange.com.jsonl.gz",
+        "lines": 19404,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/arduino.stackexchange.com.jsonl.gz",
+        "lines": 19553,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/music.stackexchange.com.jsonl.gz",
+        "lines": 19936,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/music.stackexchange.com.jsonl.gz",
+        "lines": 19936,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/aviation.stackexchange.com.jsonl.gz",
+        "lines": 20139,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/softwarerecs.stackexchange.com.jsonl.gz",
+        "lines": 20142,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/movies.stackexchange.com.jsonl.gz",
+        "lines": 20181,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/datascience.stackexchange.com.jsonl.gz",
+        "lines": 20503,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/datascience.stackexchange.com.jsonl.gz",
+        "lines": 20503,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/music.stackexchange.com.jsonl.gz",
+        "lines": 20636,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/japanese.stackexchange.com.jsonl.gz",
+        "lines": 20948,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/japanese.stackexchange.com.jsonl.gz",
+        "lines": 20948,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/emacs.stackexchange.com.jsonl.gz",
+        "lines": 21055,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/dsp.stackexchange.com.jsonl.gz",
+        "lines": 21252,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/japanese.stackexchange.com.jsonl.gz",
+        "lines": 22056,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/bitcoin.stackexchange.com.jsonl.gz",
+        "lines": 22474,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/bitcoin.stackexchange.com.jsonl.gz",
+        "lines": 22474,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/cooking.stackexchange.com.jsonl.gz",
+        "lines": 22641,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/cooking.stackexchange.com.jsonl.gz",
+        "lines": 22641,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/mechanics.stackexchange.com.jsonl.gz",
+        "lines": 22868,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/photo.stackexchange.com.jsonl.gz",
+        "lines": 23204,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/photo.stackexchange.com.jsonl.gz",
+        "lines": 23204,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/crypto.stackexchange.com.jsonl.gz",
+        "lines": 23231,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/cooking.stackexchange.com.jsonl.gz",
+        "lines": 23705,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/photo.stackexchange.com.jsonl.gz",
+        "lines": 23753,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/workplace.stackexchange.com.jsonl.gz",
+        "lines": 24012,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/workplace.stackexchange.com.jsonl.gz",
+        "lines": 24012,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/meta.stackoverflow.com.jsonl.gz",
+        "lines": 24044,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/meta.stackoverflow.com.jsonl.gz",
+        "lines": 24044,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/raspberrypi.stackexchange.com.jsonl.gz",
+        "lines": 24143,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_Title_Answer/raspberrypi.stackexchange.com.jsonl.gz",
+        "lines": 24143,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/workplace.stackexchange.com.jsonl.gz",
+        "lines": 24189,
+        "weight": 2
+    },
+    {
+        "name": "stackexchange_title_body/biology.stackexchange.com.jsonl.gz",
+        "lines": 24447,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/webapps.stackexchange.com.jsonl.gz",
+        "lines": 24867,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/webapps.stackexchange.com.jsonl.gz",
+        "lines": 24867,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/bitcoin.stackexchange.com.jsonl.gz",
+        "lines": 25374,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/judaism.stackexchange.com.jsonl.gz",
+        "lines": 26085,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/judaism.stackexchange.com.jsonl.gz",
+        "lines": 26085,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/ethereum.stackexchange.com.jsonl.gz",
+        "lines": 26124,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/ethereum.stackexchange.com.jsonl.gz",
+        "lines": 26124,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/worldbuilding.stackexchange.com.jsonl.gz",
+        "lines": 26210,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/worldbuilding.stackexchange.com.jsonl.gz",
+        "lines": 26210,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/worldbuilding.stackexchange.com.jsonl.gz",
+        "lines": 26763,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/chemistry.stackexchange.com.jsonl.gz",
+        "lines": 27061,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/chemistry.stackexchange.com.jsonl.gz",
+        "lines": 27061,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/datascience.stackexchange.com.jsonl.gz",
+        "lines": 27397,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/graphicdesign.stackexchange.com.jsonl.gz",
+        "lines": 28083,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/graphicdesign.stackexchange.com.jsonl.gz",
+        "lines": 28083,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/ux.stackexchange.com.jsonl.gz",
+        "lines": 28901,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/ux.stackexchange.com.jsonl.gz",
+        "lines": 28901,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/ux.stackexchange.com.jsonl.gz",
+        "lines": 29403,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/money.stackexchange.com.jsonl.gz",
+        "lines": 29404,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/money.stackexchange.com.jsonl.gz",
+        "lines": 29404,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/webapps.stackexchange.com.jsonl.gz",
+        "lines": 29697,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/cs.stackexchange.com.jsonl.gz",
+        "lines": 30010,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/cs.stackexchange.com.jsonl.gz",
+        "lines": 30010,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/graphicdesign.stackexchange.com.jsonl.gz",
+        "lines": 30233,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/webmasters.stackexchange.com.jsonl.gz",
+        "lines": 30370,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/webmasters.stackexchange.com.jsonl.gz",
+        "lines": 30370,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/raspberrypi.stackexchange.com.jsonl.gz",
+        "lines": 30625,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/money.stackexchange.com.jsonl.gz",
+        "lines": 32021,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/judaism.stackexchange.com.jsonl.gz",
+        "lines": 32028,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/academia.stackexchange.com.jsonl.gz",
+        "lines": 32137,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_Title_Answer/academia.stackexchange.com.jsonl.gz",
+        "lines": 32137,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/ethereum.stackexchange.com.jsonl.gz",
+        "lines": 32760,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/academia.stackexchange.com.jsonl.gz",
+        "lines": 34331,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/chemistry.stackexchange.com.jsonl.gz",
+        "lines": 34506,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/webmasters.stackexchange.com.jsonl.gz",
+        "lines": 34559,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_title_body/meta.stackoverflow.com.jsonl.gz",
+        "lines": 36456,
+        "weight": 3
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/travel.stackexchange.com.jsonl.gz",
+        "lines": 36533,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_Title_Answer/travel.stackexchange.com.jsonl.gz",
+        "lines": 36533,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/android.stackexchange.com.jsonl.gz",
+        "lines": 38077,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_Title_Answer/android.stackexchange.com.jsonl.gz",
+        "lines": 38077,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_title_body/cs.stackexchange.com.jsonl.gz",
+        "lines": 38314,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/gamedev.stackexchange.com.jsonl.gz",
+        "lines": 40154,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_Title_Answer/gamedev.stackexchange.com.jsonl.gz",
+        "lines": 40154,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/rpg.stackexchange.com.jsonl.gz",
+        "lines": 40435,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_Title_Answer/rpg.stackexchange.com.jsonl.gz",
+        "lines": 40435,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_title_body/travel.stackexchange.com.jsonl.gz",
+        "lines": 41227,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/codereview.stackexchange.com.jsonl.gz",
+        "lines": 41748,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_Title_Answer/codereview.stackexchange.com.jsonl.gz",
+        "lines": 41748,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_title_body/rpg.stackexchange.com.jsonl.gz",
+        "lines": 42303,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_title_body/codereview.stackexchange.com.jsonl.gz",
+        "lines": 45765,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_title_body/gamedev.stackexchange.com.jsonl.gz",
+        "lines": 46485,
+        "weight": 4
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/softwareengineering.stackexchange.com.jsonl.gz",
+        "lines": 51326,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/softwareengineering.stackexchange.com.jsonl.gz",
+        "lines": 51326,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/security.stackexchange.com.jsonl.gz",
+        "lines": 51355,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/security.stackexchange.com.jsonl.gz",
+        "lines": 51355,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_title_body/android.stackexchange.com.jsonl.gz",
+        "lines": 51608,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/diy.stackexchange.com.jsonl.gz",
+        "lines": 52896,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/diy.stackexchange.com.jsonl.gz",
+        "lines": 52896,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_title_body/softwareengineering.stackexchange.com.jsonl.gz",
+        "lines": 53942,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/blender.stackexchange.com.jsonl.gz",
+        "lines": 54153,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/blender.stackexchange.com.jsonl.gz",
+        "lines": 54153,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/scifi.stackexchange.com.jsonl.gz",
+        "lines": 54805,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/scifi.stackexchange.com.jsonl.gz",
+        "lines": 54805,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_title_body/security.stackexchange.com.jsonl.gz",
+        "lines": 58000,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/mathematica.stackexchange.com.jsonl.gz",
+        "lines": 59895,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/mathematica.stackexchange.com.jsonl.gz",
+        "lines": 59895,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_title_body/diy.stackexchange.com.jsonl.gz",
+        "lines": 60083,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/meta.stackexchange.com.jsonl.gz",
+        "lines": 60744,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_Title_Answer/meta.stackexchange.com.jsonl.gz",
+        "lines": 60744,
+        "weight": 5
+    },
+    {
+        "name": "stackexchange_title_body/scifi.stackexchange.com.jsonl.gz",
+        "lines": 61528,
+        "weight": 6
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/drupal.stackexchange.com.jsonl.gz",
+        "lines": 67817,
+        "weight": 6
+    },
+    {
+        "name": "stackexchange_Title_Answer/drupal.stackexchange.com.jsonl.gz",
+        "lines": 67817,
+        "weight": 6
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/dba.stackexchange.com.jsonl.gz",
+        "lines": 71449,
+        "weight": 6
+    },
+    {
+        "name": "stackexchange_Title_Answer/dba.stackexchange.com.jsonl.gz",
+        "lines": 71449,
+        "weight": 6
+    },
+    {
+        "name": "stackexchange_title_body/mathematica.stackexchange.com.jsonl.gz",
+        "lines": 73131,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/ell.stackexchange.com.jsonl.gz",
+        "lines": 77892,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_Title_Answer/ell.stackexchange.com.jsonl.gz",
+        "lines": 77892,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/magento.stackexchange.com.jsonl.gz",
+        "lines": 79241,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_Title_Answer/magento.stackexchange.com.jsonl.gz",
+        "lines": 79241,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_title_body/drupal.stackexchange.com.jsonl.gz",
+        "lines": 79717,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/sharepoint.stackexchange.com.jsonl.gz",
+        "lines": 80420,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_Title_Answer/sharepoint.stackexchange.com.jsonl.gz",
+        "lines": 80420,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_title_body/blender.stackexchange.com.jsonl.gz",
+        "lines": 80766,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_title_body/dba.stackexchange.com.jsonl.gz",
+        "lines": 81871,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/gaming.stackexchange.com.jsonl.gz",
+        "lines": 82887,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_Title_Answer/gaming.stackexchange.com.jsonl.gz",
+        "lines": 82887,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_title_body/ell.stackexchange.com.jsonl.gz",
+        "lines": 83271,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_title_body/meta.stackexchange.com.jsonl.gz",
+        "lines": 83510,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/wordpress.stackexchange.com.jsonl.gz",
+        "lines": 83621,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_Title_Answer/wordpress.stackexchange.com.jsonl.gz",
+        "lines": 83621,
+        "weight": 7
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/mathoverflow.net.jsonl.gz",
+        "lines": 85289,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_Title_Answer/mathoverflow.net.jsonl.gz",
+        "lines": 85289,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/salesforce.stackexchange.com.jsonl.gz",
+        "lines": 87272,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_Title_Answer/salesforce.stackexchange.com.jsonl.gz",
+        "lines": 87272,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_title_body/gaming.stackexchange.com.jsonl.gz",
+        "lines": 88912,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/apple.stackexchange.com.jsonl.gz",
+        "lines": 92487,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_Title_Answer/apple.stackexchange.com.jsonl.gz",
+        "lines": 92487,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_title_body/sharepoint.stackexchange.com.jsonl.gz",
+        "lines": 94011,
+        "weight": 8
+    },
+    {
+        "name": "stackexchange_title_body/magento.stackexchange.com.jsonl.gz",
+        "lines": 99991,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/gis.stackexchange.com.jsonl.gz",
+        "lines": 100254,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_Title_Answer/gis.stackexchange.com.jsonl.gz",
+        "lines": 100254,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_title_body/wordpress.stackexchange.com.jsonl.gz",
+        "lines": 100474,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/english.stackexchange.com.jsonl.gz",
+        "lines": 100640,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_Title_Answer/english.stackexchange.com.jsonl.gz",
+        "lines": 100640,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_title_body/salesforce.stackexchange.com.jsonl.gz",
+        "lines": 105260,
+        "weight": 9
+    },
+    {
+        "name": "stackexchange_title_body/english.stackexchange.com.jsonl.gz",
+        "lines": 109522,
+        "weight": 10
+    },
+    {
+        "name": "stackexchange_title_body/apple.stackexchange.com.jsonl.gz",
+        "lines": 110622,
+        "weight": 10
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/stats.stackexchange.com.jsonl.gz",
+        "lines": 115679,
+        "weight": 10
+    },
+    {
+        "name": "stackexchange_Title_Answer/stats.stackexchange.com.jsonl.gz",
+        "lines": 115679,
+        "weight": 10
+    },
+    {
+        "name": "stackexchange_title_body/mathoverflow.net.jsonl.gz",
+        "lines": 120851,
+        "weight": 10
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/electronics.stackexchange.com.jsonl.gz",
+        "lines": 129494,
+        "weight": 11
+    },
+    {
+        "name": "stackexchange_Title_Answer/electronics.stackexchange.com.jsonl.gz",
+        "lines": 129494,
+        "weight": 11
+    },
+    {
+        "name": "stackexchange_title_body/gis.stackexchange.com.jsonl.gz",
+        "lines": 131000,
+        "weight": 11
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/physics.stackexchange.com.jsonl.gz",
+        "lines": 141230,
+        "weight": 12
+    },
+    {
+        "name": "stackexchange_Title_Answer/physics.stackexchange.com.jsonl.gz",
+        "lines": 141230,
+        "weight": 12
+    },
+    {
+        "name": "stackexchange_title_body/electronics.stackexchange.com.jsonl.gz",
+        "lines": 143582,
+        "weight": 12
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/unix.stackexchange.com.jsonl.gz",
+        "lines": 155414,
+        "weight": 13
+    },
+    {
+        "name": "stackexchange_Title_Answer/unix.stackexchange.com.jsonl.gz",
+        "lines": 155414,
+        "weight": 13
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/tex.stackexchange.com.jsonl.gz",
+        "lines": 171628,
+        "weight": 15
+    },
+    {
+        "name": "stackexchange_Title_Answer/tex.stackexchange.com.jsonl.gz",
+        "lines": 171628,
+        "weight": 15
+    },
+    {
+        "name": "stackexchange_title_body/physics.stackexchange.com.jsonl.gz",
+        "lines": 173307,
+        "weight": 15
+    },
+    {
+        "name": "stackexchange_title_body/stats.stackexchange.com.jsonl.gz",
+        "lines": 173466,
+        "weight": 15
+    },
+    {
+        "name": "stackexchange_title_body/unix.stackexchange.com.jsonl.gz",
+        "lines": 185997,
+        "weight": 16
+    },
+    {
+        "name": "stackexchange_title_body/tex.stackexchange.com.jsonl.gz",
+        "lines": 202954,
+        "weight": 17
+    },
+    {
+        "name": "TriviaQA_pairs.jsonl.gz",
+        "lines": 73346,
+        "weight": 19
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/serverfault.com.jsonl.gz",
+        "lines": 238507,
+        "weight": 20
+    },
+    {
+        "name": "stackexchange_Title_Answer/serverfault.com.jsonl.gz",
+        "lines": 238507,
+        "weight": 20
+    },
+    {
+        "name": "stackexchange_duplicate_questions_title-body_title-body.jsonl.gz",
+        "lines": 250460,
+        "weight": 21
+    },
+    {
+        "name": "stackexchange_duplicate_questions_body_body.jsonl.gz",
+        "lines": 250519,
+        "weight": 21
+    },
+    {
+        "name": "squad_pairs.jsonl.gz",
+        "lines": 87599,
+        "weight": 22
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/askubuntu.com.jsonl.gz",
+        "lines": 267135,
+        "weight": 22
+    },
+    {
+        "name": "stackexchange_Title_Answer/askubuntu.com.jsonl.gz",
+        "lines": 267135,
+        "weight": 22
+    },
+    {
+        "name": "stackexchange_title_body/serverfault.com.jsonl.gz",
+        "lines": 270904,
+        "weight": 23
+    },
+    {
+        "name": "NQ-train_pairs.jsonl.gz",
+        "lines": 100231,
+        "weight": 25
+    },
+    {
+        "name": "SimpleWiki.jsonl.gz",
+        "lines": 102225,
+        "weight": 26
+    },
+    {
+        "name": "quora_duplicates_triplets.jsonl.gz",
+        "lines": 103663,
+        "weight": 26
+    },
+    {
+        "name": "stackexchange_duplicate_questions_title_title.jsonl.gz",
+        "lines": 304525,
+        "weight": 26
+    },
+    {
+        "name": "altlex.jsonl.gz",
+        "lines": 112696,
+        "weight": 28
+    },
+    {
+        "name": "stackexchange_title_body/askubuntu.com.jsonl.gz",
+        "lines": 347925,
+        "weight": 29
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/superuser.com.jsonl.gz",
+        "lines": 352610,
+        "weight": 30
+    },
+    {
+        "name": "stackexchange_Title_Answer/superuser.com.jsonl.gz",
+        "lines": 352610,
+        "weight": 30
+    },
+    {
+        "name": "wikihow.jsonl.gz",
+        "lines": 128542,
+        "weight": 32
+    },
+    {
+        "name": "stackexchange_title_body/superuser.com.jsonl.gz",
+        "lines": 435463,
+        "weight": 36
+    },
+    {
+        "name": "stackexchange_title_body/small_stackexchanges.jsonl.gz",
+        "lines": 448146,
+        "weight": 37
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/small_stackexchanges.jsonl.gz",
+        "lines": 460256,
+        "weight": 38
+    },
+    {
+        "name": "stackexchange_Title_Answer/small_stackexchanges.jsonl.gz",
+        "lines": 460256,
+        "weight": 38
+    },
+    {
+        "name": "sentence-compression.jsonl.gz",
+        "lines": 180000,
+        "weight": 45
+    },
+    {
+        "name": "AllNLI.jsonl.gz",
+        "lines": 277230,
+        "weight": 69
+    },
+    {
+        "name": "eli5_question_answer.jsonl.gz",
+        "lines": 325475,
+        "weight": 81
+    },
+    {
+        "name": "reddit/reddit_2015.jsonl.gz",
+        "lines": 135108166,
+        "weight": 82
+    },
+    {
+        "name": "reddit/reddit_2016.jsonl.gz",
+        "lines": 159164386,
+        "weight": 82
+    },
+    {
+        "name": "reddit/reddit_2017.jsonl.gz",
+        "lines": 191485219,
+        "weight": 82
+    },
+    {
+        "name": "reddit/reddit_2018.jsonl.gz",
+        "lines": 240726659,
+        "weight": 82
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/math.stackexchange.com.jsonl.gz",
+        "lines": 1100953,
+        "weight": 83
+    },
+    {
+        "name": "stackexchange_Title_Answer/math.stackexchange.com.jsonl.gz",
+        "lines": 1100953,
+        "weight": 83
+    },
+    {
+        "name": "stackexchange_title_body/math.stackexchange.com.jsonl.gz",
+        "lines": 1338443,
+        "weight": 83
+    },
+    {
+        "name": "stackexchange_TitleBody_Answer/stackoverflow.com-Posts.jsonl.gz",
+        "lines": 15768211,
+        "weight": 83
+    },
+    {
+        "name": "stackexchange_Title_Answer/stackoverflow.com-Posts.jsonl.gz",
+        "lines": 15768211,
+        "weight": 83
+    },
+    {
+        "name": "stackexchange_title_body/stackoverflow.com-Posts.jsonl.gz",
+        "lines": 18562443,
+        "weight": 83
+    },
+    {
+        "name": "specter_train_triples.jsonl.gz",
+        "lines": 684100,
+        "weight": 84
+    },
+    {
+        "name": "S2ORC_title_abstract.jsonl.gz",
+        "lines": 41769185,
+        "weight": 123
+    },
+    {
+        "name": "S2ORC_citation_pairs.jsonl.gz",
+        "lines": 52603982,
+        "weight": 123
+    },
+    {
+        "name": "PAQ_pairs.jsonl.gz",
+        "lines": 64371441,
+        "weight": 123
+    },
+    {
+        "name": "WikiAnswers_pairs.jsonl.gz",
+        "lines": 77427422,
+        "weight": 123
+    },
+    {
+        "name": "S2ORC_citation_pairs_abstract.jsonl.gz",
+        "lines": 116288806,
+        "weight": 123
+    },
+    {
+        "name": "searchQA_question_top5_snippets_merged.jsonl.gz",
+        "lines": 582261,
+        "weight": 144
+    },
+    {
+        "name": "yahoo_answers_title_question.jsonl.gz",
+        "lines": 659896,
+        "weight": 163
+    },
+    {
+        "name": "yahoo_answers_question_answer.jsonl.gz",
+        "lines": 681164,
+        "weight": 169
+    },
+    {
+        "name": "yahoo_answers_title_answer.jsonl.gz",
+        "lines": 1198260,
+        "weight": 247
+    },
+    {
+        "name": "amazon-qa-train-pairs.jsonl.gz",
+        "lines": 2448839,
+        "weight": 247
+    },
+    {
+        "name": "gooaq_pairs.jsonl.gz",
+        "lines": 3012496,
+        "weight": 247
+    },
+    {
+        "name": "msmarco-query_passage_negative.jsonl.gz",
+        "lines": 9144553,
+        "weight": 247
+    }
+]

all-MiniLM-L6-v2/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:53aa51172d142c89d9012cce15ae4d6cc0ca6895895114379cacb4fab128d9db
+size 90868376

all-MiniLM-L6-v2/modules.json

@@ -0,0 +1,20 @@
+[
+  {
+    "idx": 0,
+    "name": "0",
+    "path": "",
+    "type": "sentence_transformers.models.Transformer"
+  },
+  {
+    "idx": 1,
+    "name": "1",
+    "path": "1_Pooling",
+    "type": "sentence_transformers.models.Pooling"
+  },
+  {
+    "idx": 2,
+    "name": "2",
+    "path": "2_Normalize",
+    "type": "sentence_transformers.models.Normalize"
+  }
+]

all-MiniLM-L6-v2/onnx/model.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6fd5d72fe4589f189f8ebc006442dbb529bb7ce38f8082112682524616046452
+size 90405214

all-MiniLM-L6-v2/onnx/model_O1.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1391c6fc20b5530250bc15cbe1f47578ffeca55ab0551d335cc668b6299a88ec
+size 90360328

all-MiniLM-L6-v2/onnx/model_O2.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1de3905029190b398c7d300b530e320cf4b5e7d3dfb9af1429ebd73fd9a16faf
+size 90326566

all-MiniLM-L6-v2/onnx/model_O3.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a44f671e364dddbac31f203f07b91be6b0a35e51936e5ebfab65b6d9538b83ff
+size 90326497

all-MiniLM-L6-v2/onnx/model_O4.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1667d7f3ba669048b13a96ee3a44456d5e42c8f44588ae8b603430e16160c485
+size 45212349

all-MiniLM-L6-v2/onnx/model_qint8_arm64.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4278337fd0ff3c68bfb6291042cad8ab363e1d9fbc43dcb499fe91c871902474
+size 23026053

all-MiniLM-L6-v2/onnx/model_qint8_avx512.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4278337fd0ff3c68bfb6291042cad8ab363e1d9fbc43dcb499fe91c871902474
+size 23026053

all-MiniLM-L6-v2/onnx/model_qint8_avx512_vnni.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4278337fd0ff3c68bfb6291042cad8ab363e1d9fbc43dcb499fe91c871902474
+size 23026053

all-MiniLM-L6-v2/onnx/model_quint8_avx2.onnx

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b941bf19f1f1283680f449fa6a7336bb5600bdcd5f84d10ddc5cd72218a0fd21
+size 23046789

all-MiniLM-L6-v2/openvino/openvino_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8b86cab4722e2aefab310cf96d4d5a9eb3b187f7d9670a082afc55c7fa0d392a
+size 90265744

all-MiniLM-L6-v2/openvino/openvino_model_qint8_quantized.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c92ea4af3c6bc7b4a0f3b3d61b147c850f4dbdd7c9e7beee0c0c70dc12da289b
+size 22933664

all-MiniLM-L6-v2/pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c3a85f238711653950f6a79ece63eb0ea93d76f6a6284be04019c53733baf256
+size 90888945

all-MiniLM-L6-v2/rust_model.ot

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d98d96d278348988f2744e6445b8bc16d921c3f6e17c667362f3cb353007aea
+size 90887379

all-MiniLM-L6-v2/sentence_bert_config.json

@@ -0,0 +1,4 @@
+{
+  "max_seq_length": 256,
+  "do_lower_case": false
+}

all-MiniLM-L6-v2/special_tokens_map.json

@@ -0,0 +1 @@
+{"unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]"}

all-MiniLM-L6-v2/tf_model.h5

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24c06a7429b843d46e40c6b167122053921bf94dce2e5550ea5c07fabc597646
+size 91005696

all-MiniLM-L6-v2/tokenizer_config.json

@@ -0,0 +1 @@
+{"do_lower_case": true, "unk_token": "[UNK]", "sep_token": "[SEP]", "pad_token": "[PAD]", "cls_token": "[CLS]", "mask_token": "[MASK]", "tokenize_chinese_chars": true, "strip_accents": null, "name_or_path": "nreimers/MiniLM-L6-H384-uncased", "do_basic_tokenize": true, "never_split": null, "tokenizer_class": "BertTokenizer", "model_max_length": 512}

all-MiniLM-L6-v2/train_script.py

@@ -0,0 +1,344 @@
+"""
+Train script for a single file
+
+Need to set the TPU address first:
+export XRT_TPU_CONFIG="localservice;0;localhost:51011"
+"""
+
+import torch.multiprocessing as mp
+import threading
+import time
+import random
+import sys
+import argparse
+import gzip
+import json
+import logging
+import tqdm
+import torch
+from torch import nn
+from torch.utils.data import DataLoader
+import torch
+import torch_xla
+import torch_xla.core
+import torch_xla.core.functions
+import torch_xla.core.xla_model as xm
+import torch_xla.distributed.xla_multiprocessing as xmp
+import torch_xla.distributed.parallel_loader as pl
+import os
+from shutil import copyfile
+
+
+from transformers import (
+    AdamW,
+    AutoModel,
+    AutoTokenizer,
+    get_linear_schedule_with_warmup,
+    set_seed,
+)
+
+class AutoModelForSentenceEmbedding(nn.Module):
+    def __init__(self, model_name, tokenizer, normalize=True):
+        super(AutoModelForSentenceEmbedding, self).__init__()
+
+        self.model = AutoModel.from_pretrained(model_name)
+        self.normalize = normalize
+        self.tokenizer = tokenizer
+
+    def forward(self, **kwargs):
+        model_output = self.model(**kwargs)
+        embeddings = self.mean_pooling(model_output, kwargs['attention_mask'])
+        if self.normalize:
+            embeddings = torch.nn.functional.normalize(embeddings, p=2, dim=1)
+
+        return embeddings
+
+    def mean_pooling(self, model_output, attention_mask):
+        token_embeddings = model_output[0]  # First element of model_output contains all token embeddings
+        input_mask_expanded = attention_mask.unsqueeze(-1).expand(token_embeddings.size()).float()
+        return torch.sum(token_embeddings * input_mask_expanded, 1) / torch.clamp(input_mask_expanded.sum(1), min=1e-9)
+
+    def save_pretrained(self, output_path):
+        if xm.is_master_ordinal():
+            self.tokenizer.save_pretrained(output_path)
+            self.model.config.save_pretrained(output_path)
+
+        xm.save(self.model.state_dict(), os.path.join(output_path, "pytorch_model.bin"))
+       
+
+
+
+def train_function(index, args, queue):
+    tokenizer = AutoTokenizer.from_pretrained(args.model)
+    model = AutoModelForSentenceEmbedding(args.model, tokenizer)
+    
+  
+    ### Train Loop
+    device = xm.xla_device()
+    model = model.to(device)
+
+    # Instantiate optimizer
+    optimizer = AdamW(params=model.parameters(), lr=2e-5, correct_bias=True)
+
+    lr_scheduler = get_linear_schedule_with_warmup(
+        optimizer=optimizer,
+        num_warmup_steps=500,
+        num_training_steps=args.steps,
+    )
+    
+    # Now we train the model
+    cross_entropy_loss = nn.CrossEntropyLoss()
+    max_grad_norm = 1
+
+    model.train()
+   
+    for global_step in tqdm.trange(args.steps, disable=not xm.is_master_ordinal()):
+        #### Get the batch data
+        batch = queue.get()
+        #print(index, "batch {}x{}".format(len(batch), ",".join([str(len(b)) for b in batch])))
+        
+
+        if len(batch[0]) == 2: #(anchor, positive)
+            text1 = tokenizer([b[0] for b in batch], return_tensors="pt", max_length=args.max_length, truncation=True, padding="max_length")
+            text2 = tokenizer([b[1] for b in batch], return_tensors="pt", max_length=args.max_length, truncation=True, padding="max_length")
+
+            ### Compute embeddings
+            embeddings_a = model(**text1.to(device))
+            embeddings_b = model(**text2.to(device))
+            
+            ### Gather all embedings 
+            embeddings_a = torch_xla.core.functions.all_gather(embeddings_a)
+            embeddings_b = torch_xla.core.functions.all_gather(embeddings_b)
+
+            ### Compute similarity scores 512 x 512
+            scores = torch.mm(embeddings_a, embeddings_b.transpose(0, 1)) * args.scale
+        
+            ### Compute cross-entropy loss
+            labels = torch.tensor(range(len(scores)), dtype=torch.long, device=embeddings_a.device)  # Example a[i] should match with b[i]
+            
+            ## Symmetric loss as in CLIP
+            loss = (cross_entropy_loss(scores, labels) + cross_entropy_loss(scores.transpose(0, 1), labels)) / 2
+
+        else:   #(anchor, positive, negative)
+            text1 = tokenizer([b[0] for b in batch], return_tensors="pt", max_length=args.max_length, truncation=True, padding="max_length")
+            text2 = tokenizer([b[1] for b in batch], return_tensors="pt", max_length=args.max_length, truncation=True, padding="max_length")
+            text3 = tokenizer([b[2] for b in batch], return_tensors="pt", max_length=args.max_length, truncation=True, padding="max_length")
+
+            embeddings_a  = model(**text1.to(device))
+            embeddings_b1 = model(**text2.to(device))
+            embeddings_b2 = model(**text3.to(device))
+
+            embeddings_a  = torch_xla.core.functions.all_gather(embeddings_a)
+            embeddings_b1 = torch_xla.core.functions.all_gather(embeddings_b1)
+            embeddings_b2 = torch_xla.core.functions.all_gather(embeddings_b2)
+
+            embeddings_b = torch.cat([embeddings_b1, embeddings_b2])
+
+            ### Compute similarity scores 512 x 1024
+            scores = torch.mm(embeddings_a, embeddings_b.transpose(0, 1)) * args.scale
+        
+            ### Compute cross-entropy loss
+            labels = torch.tensor(range(len(scores)), dtype=torch.long, device=embeddings_a.device)  # Example a[i] should match with b[i]
+            
+            ## One-way loss
+            loss = cross_entropy_loss(scores, labels)
+
+        
+        # Backward pass
+        optimizer.zero_grad()
+        loss.backward()
+        torch.nn.utils.clip_grad_norm_(model.parameters(), max_grad_norm)
+        
+        xm.optimizer_step(optimizer, barrier=True)
+        lr_scheduler.step()
+
+
+        #Save model
+        if (global_step+1) % args.save_steps == 0:
+            output_path = os.path.join(args.output, str(global_step+1))
+            xm.master_print("save model: "+output_path)
+            model.save_pretrained(output_path)
+          
+            
+    output_path = os.path.join(args.output, "final")
+    xm.master_print("save model final: "+ output_path)
+    model.save_pretrained(output_path)
+
+
+def produce_data(args, queue, filepaths, dataset_indices):
+    global_batch_size = args.batch_size*args.nprocs    #Global batch size
+    size_per_dataset = int(global_batch_size / args.datasets_per_batch)    #How many datasets per batch
+    num_same_dataset = int(size_per_dataset / args.batch_size)
+    print("producer", "global_batch_size", global_batch_size)
+    print("producer", "size_per_dataset", size_per_dataset)
+    print("producer", "num_same_dataset", num_same_dataset)
+    
+    datasets = []
+    for filepath in filepaths:
+        if "reddit_" in filepath:       #Special dataset class for Reddit files
+            data_obj = RedditDataset(filepath)
+        else:
+            data_obj = Dataset(filepath)
+        datasets.append(iter(data_obj)) 
+    
+    # Store if dataset is in a 2 col or 3 col format
+    num_cols = {idx: len(next(dataset)) for idx, dataset in enumerate(datasets)}
+
+    while True:
+        texts_in_batch = set()
+        batch_format = None     #2 vs 3 col format for this batch
+        
+        #Add data from several sub datasets
+        for _ in range(args.datasets_per_batch):
+            valid_dataset = False   #Check that datasets have the same 2/3 col format
+            while not valid_dataset:
+                data_idx = random.choice(dataset_indices)
+                if batch_format is None:
+                    batch_format = num_cols[data_idx]
+                    valid_dataset = True
+                else:   #Check that this dataset has the same format
+                    valid_dataset = (batch_format == num_cols[data_idx])
+            
+            #Get data from this dataset
+            dataset = datasets[data_idx]
+            for _ in range(num_same_dataset):
+                for _ in range(args.nprocs):
+                    batch_device = []   #A batch for one device
+                    while len(batch_device) < args.batch_size:
+                        sample = next(dataset)
+                        in_batch = False
+                        for text in sample:
+                            if text in texts_in_batch:
+                                in_batch = True
+                                break
+                        
+                        if not in_batch:
+                            for text in sample:
+                                texts_in_batch.add(text)
+                            batch_device.append(sample)
+
+                    queue.put(batch_device)
+                      
+
+class RedditDataset:
+    """
+    A class that handles the reddit data files
+    """
+    def __init__(self, filepath):
+        self.filepath = filepath
+
+    def __iter__(self):
+        while True:
+            with gzip.open(self.filepath, "rt") as fIn:
+                    for line in fIn:
+                        data = json.loads(line)
+
+                        if "response" in data and "context" in data:
+                            yield [data["response"], data["context"]]
+
+class Dataset:
+    """
+    A class that handles one dataset
+    """
+    def __init__(self, filepath):
+        self.filepath = filepath
+
+    def __iter__(self):
+        max_dataset_size = 10*1000*1000    #Cache small datasets in memory
+        dataset = []
+        data_format = None
+
+        while dataset is None or len(dataset) == 0:
+            with gzip.open(self.filepath, "rt") as fIn:
+                for line in fIn:
+                    data = json.loads(line)
+                    if isinstance(data, dict):
+                        data = data['texts']
+
+                    if data_format is None:
+                        data_format = len(data)
+                    
+                    #Ensure that all entries are of the same 2/3 col format
+                    assert len(data) == data_format
+
+                    if dataset is not None:
+                        dataset.append(data)
+                        if len(dataset) >= max_dataset_size:
+                            dataset = None
+
+                    yield data
+                
+        # Data loaded. Now stream to the queue
+        # Shuffle for each epoch
+        while True:
+            random.shuffle(dataset)
+            for data in dataset:
+                yield data
+                
+               
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--model', default='nreimers/MiniLM-L6-H384-uncased')
+    parser.add_argument('--steps', type=int, default=2000)
+    parser.add_argument('--save_steps', type=int, default=10000)
+    parser.add_argument('--batch_size', type=int, default=64)
+    parser.add_argument('--max_length', type=int, default=128)
+    parser.add_argument('--nprocs', type=int, default=8)
+    parser.add_argument('--datasets_per_batch', type=int, default=2, help="Number of datasets per batch")
+    parser.add_argument('--scale', type=float, default=20, help="Use 20 for cossim, and 1 when you work with unnormalized embeddings with dot product")
+    parser.add_argument('--data_folder', default="/data", help="Folder with your dataset files")
+    parser.add_argument('data_config', help="A data_config.json file")
+    parser.add_argument('output')
+    args = parser.parse_args()
+
+    # Ensure global batch size is divisble by data_sample_size
+    assert (args.batch_size*args.nprocs) % args.datasets_per_batch == 0
+
+    logging.info("Output: "+args.output)
+    if os.path.exists(args.output):
+        print("Output folder already exists.")
+        input("Continue?")
+
+    # Write train script to output path
+    os.makedirs(args.output, exist_ok=True)
+
+    data_config_path = os.path.join(args.output, 'data_config.json')
+    copyfile(args.data_config, data_config_path)
+
+    train_script_path = os.path.join(args.output, 'train_script.py')
+    copyfile(__file__, train_script_path)
+    with open(train_script_path, 'a') as fOut:
+        fOut.write("\n\n# Script was called via:\n#python " + " ".join(sys.argv))
+
+
+
+    #Load data config
+    with open(args.data_config) as fIn:
+        data_config = json.load(fIn)
+
+    queue = mp.Queue(maxsize=100*args.nprocs)
+    
+    filepaths = []
+    dataset_indices = []
+    for idx, data in enumerate(data_config):
+        filepaths.append(os.path.join(os.path.expanduser(args.data_folder), data['name']))
+        dataset_indices.extend([idx]*data['weight'])
+
+    # Start producer
+    p = mp.Process(target=produce_data, args=(args, queue, filepaths, dataset_indices))
+    p.start()
+
+    # Run training
+    print("Start processes:", args.nprocs)
+    xmp.spawn(train_function, args=(args, queue), nprocs=args.nprocs, start_method='fork')
+    print("Training done")
+    print("It might be that not all processes exit automatically. In that case you must manually kill this process.")
+    print("With 'pkill python' you can kill all remaining python processes")
+    p.kill()
+    exit()
+
+
+
+# Script was called via:
+#python train_many_data_files_v2.py --steps 1000000 --batch_size 128 --model nreimers/MiniLM-L6-H384-uncased train_data_configs/all_datasets_v4.json output/all_datasets_v4_MiniLM-L6-H384-uncased-batch128

endpoint.py

@@ -0,0 +1,231 @@
+from flask import Flask, request, jsonify
+import os
+from dotenv import load_dotenv
+from llama_index.core import VectorStoreIndex, Settings, StorageContext, load_index_from_storage
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.llms.groq import Groq
+import pandas as pd
+from llama_index.core import Document
+
+app = Flask(__name__)
+
+# --- Configuration ---
+PERSIST_DIR = "./storage"
+EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+LLM_MODEL = "llama3-8b-8192"
+CSV_FILE_PATH = "shl_assessments.csv"
+
+# --- Root Route (for health check) ---
+@app.route("/", methods=["GET"])
+def home():
+    return "🧠 SHL Chatbot API is running!", 200
+
+# --- Utility Functions ---
+def load_groq_llm():
+    load_dotenv()
+    api_key = os.getenv("GROQ_API_KEY")
+    if not api_key:
+        raise ValueError("GROQ_API_KEY not found in .env file or environment variables")
+    return Groq(model=LLM_MODEL, api_key=api_key, temperature=0.1)
+
+def load_embeddings():
+    return HuggingFaceEmbedding(model_name=EMBED_MODEL)
+
+def load_data_from_csv(csv_path):
+    try:
+        df = pd.read_csv(csv_path)
+        required_columns = ["Assessment Name", "URL", "Remote Testing Support",
+                            "Adaptive/IRT Support", "Duration (min)", "Test Type"]
+        if not all(col in df.columns for col in required_columns):
+            raise ValueError(f"CSV file must contain columns: {', '.join(required_columns)}")
+        return df.to_dict(orient="records")
+    except FileNotFoundError:
+        raise FileNotFoundError(f"CSV file not found at {csv_path}")
+    except Exception as e:
+        raise Exception(f"Error reading CSV: {e}")
+
+def build_index(data):
+    Settings.embed_model = load_embeddings()
+    Settings.llm = load_groq_llm()
+    documents = [
+        Document(text=f"Name: {item['Assessment Name']}, URL: {item['URL']}, Remote Testing: {item['Remote Testing Support']}, Adaptive/IRT: {item['Adaptive/IRT Support']}, Duration: {item['Duration (min)']}, Type: {item['Test Type']}")
+        for item in data
+    ]
+    index = VectorStoreIndex.from_documents(documents)
+    index.storage_context.persist(persist_dir=PERSIST_DIR)
+    return index
+
+def load_chat_engine():
+    if not os.path.exists(PERSIST_DIR):
+        return None
+    Settings.embed_model = load_embeddings()
+    Settings.llm = load_groq_llm()
+    storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
+    index = load_index_from_storage(storage_context)
+    return index.as_chat_engine(chat_mode="context", verbose=True)
+
+# --- Load or Build Index ---
+try:
+    chat_engine = load_chat_engine()
+    if chat_engine is None:
+        assessment_data = load_data_from_csv(CSV_FILE_PATH)
+        build_index(assessment_data)
+        chat_engine = load_chat_engine()
+except Exception as e:
+    print(f"❌ Error initializing chat engine: {e}")
+    chat_engine = None
+
+# --- Endpoint ---
+@app.route("/assessments", methods=["POST"])
+def get_assessments():
+    data = request.get_json()
+    query = data.get("query")
+
+    if not query:
+        return jsonify({"error": "No query provided"}), 400
+
+    if chat_engine:
+        try:
+            response = chat_engine.chat(query)
+            results = []
+
+            for node in response.source_nodes:
+                try:
+                    parts = node.node.text.split(", ")
+                    results.append({
+                        "assessment_name": parts[0].split(": ")[1] if len(parts) > 0 else "N/A",
+                        "assessment_url": parts[1].split(": ")[1] if len(parts) > 1 else "N/A",
+                        "remote_testing_support": parts[2].split(": ")[1] if len(parts) > 2 else "N/A",
+                        "adaptive_irt_support": parts[3].split(": ")[1] if len(parts) > 3 else "N/A",
+                        "duration": parts[4].split(": ")[1] if len(parts) > 4 else "N/A",
+                        "test_type": parts[5].split(": ")[1] if len(parts) > 5 else "N/A"
+                    })
+                except:
+                    results.append({"error": "Error parsing assessment info"})
+
+            return jsonify({"query": query, "response": results}), 200
+
+        except Exception as e:
+            return jsonify({"error": f"Chat processing error: {e}"}), 500
+    else:
+        return jsonify({"error": "Chat engine not initialized"}), 500
+
+# --- Entry Point for Local Debugging ---
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=10000)
+
+from flask import Flask, request, jsonify
+import os
+from dotenv import load_dotenv
+from llama_index.core import VectorStoreIndex, Settings, StorageContext, load_index_from_storage
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.llms.groq import Groq
+import pandas as pd
+from llama_index.core import Document
+
+app = Flask(__name__)
+
+# --- Configuration ---
+PERSIST_DIR = "./storage"
+EMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2"
+LLM_MODEL = "llama3-8b-8192"
+CSV_FILE_PATH = "shl_assessments.csv"
+
+# --- Root Route (for health check) ---
+@app.route("/", methods=["GET"])
+def home():
+    return "🧠 SHL Chatbot API is running!", 200
+
+# --- Utility Functions ---
+def load_groq_llm():
+    load_dotenv()
+    api_key = os.getenv("GROQ_API_KEY")
+    if not api_key:
+        raise ValueError("GROQ_API_KEY not found in .env file or environment variables")
+    return Groq(model=LLM_MODEL, api_key=api_key, temperature=0.1)
+
+def load_embeddings():
+    return HuggingFaceEmbedding(model_name=EMBED_MODEL)
+
+def load_data_from_csv(csv_path):
+    try:
+        df = pd.read_csv(csv_path)
+        required_columns = ["Assessment Name", "URL", "Remote Testing Support",
+                            "Adaptive/IRT Support", "Duration (min)", "Test Type"]
+        if not all(col in df.columns for col in required_columns):
+            raise ValueError(f"CSV file must contain columns: {', '.join(required_columns)}")
+        return df.to_dict(orient="records")
+    except FileNotFoundError:
+        raise FileNotFoundError(f"CSV file not found at {csv_path}")
+    except Exception as e:
+        raise Exception(f"Error reading CSV: {e}")
+
+def build_index(data):
+    Settings.embed_model = load_embeddings()
+    Settings.llm = load_groq_llm()
+    documents = [
+        Document(text=f"Name: {item['Assessment Name']}, URL: {item['URL']}, Remote Testing: {item['Remote Testing Support']}, Adaptive/IRT: {item['Adaptive/IRT Support']}, Duration: {item['Duration (min)']}, Type: {item['Test Type']}")
+        for item in data
+    ]
+    index = VectorStoreIndex.from_documents(documents)
+    index.storage_context.persist(persist_dir=PERSIST_DIR)
+    return index
+
+def load_chat_engine():
+    if not os.path.exists(PERSIST_DIR):
+        return None
+    Settings.embed_model = load_embeddings()
+    Settings.llm = load_groq_llm()
+    storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
+    index = load_index_from_storage(storage_context)
+    return index.as_chat_engine(chat_mode="context", verbose=True)
+
+# --- Load or Build Index ---
+try:
+    chat_engine = load_chat_engine()
+    if chat_engine is None:
+        assessment_data = load_data_from_csv(CSV_FILE_PATH)
+        build_index(assessment_data)
+        chat_engine = load_chat_engine()
+except Exception as e:
+    print(f"❌ Error initializing chat engine: {e}")
+    chat_engine = None
+
+# --- Endpoint ---
+@app.route("/assessments", methods=["POST"])
+def get_assessments():
+    data = request.get_json()
+    query = data.get("query")
+
+    if not query:
+        return jsonify({"error": "No query provided"}), 400
+
+    if chat_engine:
+        try:
+            response = chat_engine.chat(query)
+            results = []
+
+            for node in response.source_nodes:
+                try:
+                    parts = node.node.text.split(", ")
+                    results.append({
+                        "assessment_name": parts[0].split(": ")[1] if len(parts) > 0 else "N/A",
+                        "assessment_url": parts[1].split(": ")[1] if len(parts) > 1 else "N/A",
+                        "remote_testing_support": parts[2].split(": ")[1] if len(parts) > 2 else "N/A",
+                        "adaptive_irt_support": parts[3].split(": ")[1] if len(parts) > 3 else "N/A",
+                        "duration": parts[4].split(": ")[1] if len(parts) > 4 else "N/A",
+                        "test_type": parts[5].split(": ")[1] if len(parts) > 5 else "N/A"
+                    })
+                except:
+                    results.append({"error": "Error parsing assessment info"})
+
+            return jsonify({"query": query, "response": results}), 200
+
+        except Exception as e:
+            return jsonify({"error": f"Chat processing error: {e}"}), 500
+    else:
+        return jsonify({"error": "Chat engine not initialized"}), 500
+
+# --- Entry Point for Local Debugging ---
+if __name__ == "__main__":
+    app.run(host="0.0.0.0", port=10000)

main.py

@@ -0,0 +1,199 @@
+import os
+import shutil
+import streamlit as st
+from dotenv import load_dotenv
+from llama_index.core import (
+    VectorStoreIndex,
+    Settings,
+    StorageContext,
+    load_index_from_storage,
+)
+from llama_index.embeddings.huggingface import HuggingFaceEmbedding
+from llama_index.llms.groq import Groq
+import pandas as pd
+from llama_index.core import Document
+
+
+PERSIST_DIR = "./storage"
+EMBED_MODEL = "./all-MiniLM-L6-v2"
+EMBED_MODEL = "./all-MiniLM-L6-v2"
+EMBED_MODEL = "./all-MiniLM-L6-v2"
+LLM_MODEL = "llama3-8b-8192" 
+CSV_FILE_PATH = "shl_assessments.csv"
+GROQ_API_KEY = st.secrets["GROQ_API_KEY"]  or os.getenv["GROQ_API_KEY"]  or os.getenv("GROQ_API_KEY")
+
+
+def load_data_from_csv(csv_path):
+    """Loads assessment data from a CSV file."""
+    try:
+        df = pd.read_csv(csv_path)
+        required_columns = ["Assessment Name", "URL", "Remote Testing Support",
+                            "Adaptive/IRT Support", "Duration (min)", "Test Type"]
+        if not all(col in df.columns for col in required_columns):
+            raise ValueError(f"CSV file must contain columns: {', '.join(required_columns)}")
+        return df.to_dict(orient="records")
+    except FileNotFoundError:
+        raise FileNotFoundError(f"Error: CSV file not found at {csv_path}")
+    except ValueError as e:
+        raise ValueError(f"Error reading CSV: {e}")
+    except Exception as e:
+        raise Exception(f"An unexpected error occurred while loading CSV data: {e}")
+    
+
+def load_groq_llm():
+    try:
+        api_key = st.secrets.get("GROQ_API_KEY") or os.getenv("GROQ_API_KEY")
+        api_key = st.secrets.get("GROQ_API_KEY") or os.getenv("GROQ_API_KEY")
+    except KeyError:
+        raise ValueError("GROQ_API_KEY not found in Streamlit secrets.")
+    
+    return Groq(model=LLM_MODEL, api_key=api_key, temperature=0.1)
+
+
+
+def load_embeddings():
+    return HuggingFaceEmbedding(model_name="all-MiniLM-L6-v2")   
+    return HuggingFaceEmbedding(model_name="all-MiniLM-L6-v2")   
+
+def build_index(data):
+    """Builds the vector index from the provided assessment data."""
+    return HuggingFaceEmbedding(model_name=EMBED_MODEL) 
+    return HuggingFaceEmbedding(model_name=EMBED_MODEL) 
+    Settings.llm = load_groq_llm()
+
+    documents = [Document(text=f"Name: {item['Assessment Name']}, URL: {item['URL']}, Remote Testing: {item['Remote Testing Support']}, Adaptive/IRT: {item['Adaptive/IRT Support']}, Duration: {item['Duration (min)']}, Type: {item['Test Type']}") for item in data]
+
+    index = VectorStoreIndex.from_documents(documents)
+    index.storage_context.persist(persist_dir=PERSIST_DIR)
+    return index
+
+def load_chat_engine():
+    """Loads the chat engine from the persisted index."""
+    if not os.path.exists(PERSIST_DIR):
+        return None
+
+    Settings.embed_model = load_embeddings()
+    Settings.llm = load_groq_llm()
+    storage_context = StorageContext.from_defaults(persist_dir=PERSIST_DIR)
+    index = load_index_from_storage(storage_context)
+    return index.as_chat_engine(chat_mode="context", verbose=True)
+
+def reset_index():
+    """Resets the persisted index and chat history."""
+    try:
+        shutil.rmtree(PERSIST_DIR, ignore_errors=True)
+        st.success("Knowledge index reset successfully!")
+        st.session_state.messages = [{"role": "assistant", "content": "Hello! I'm your SHL assessment assistant. How can I help you?"}]
+        st.session_state["index_built"] = False
+        if 'chat_engine' in st.session_state:
+            del st.session_state['chat_engine']
+        return None
+    except Exception as e:
+        st.error(f"Error resetting index: {str(e)}")
+        return None
+
+def main():
+    st.set_page_config(
+        page_title="SHL Assessment Chatbot",
+        layout="wide",
+        initial_sidebar_state="collapsed"
+    )
+
+    st.markdown("""
+    <style>
+    :root {
+        --primary: #6eb5ff;
+        --background: #000000;
+        --card: #f0f2f6;
+        --text: #ffffff;
+        --background: #000000;
+        --card: #f0f2f6;
+        --text: #ffffff;
+    }
+    .stApp {
+        background-color: var(--background) !important;
+        color: var(--text) !important;
+    }
+    .stMarkdown, .stTextInput, .stChatMessage, .stChatInputContainer, .css-10trblm, .css-1cpxqw2 {
+        color: var(--text) !important;
+    }
+    .stApp {
+        background-color: var(--background) !important;
+        color: var(--text) !important;
+    }
+    .stMarkdown, .stTextInput, .stChatMessage, .stChatInputContainer, .css-10trblm, .css-1cpxqw2 {
+        color: var(--text) !important;
+    }
+    .stApp {
+        background-color: var(--background) !important;
+        color: var(--text) !important;
+    }
+    .stMarkdown, .stTextInput, .stChatMessage, .stChatInputContainer, .css-10trblm, .css-1cpxqw2 {
+        color: var(--text) !important;
+    }
+    </style>
+    """, unsafe_allow_html=True)
+
+    load_dotenv()
+    os.environ["STREAMLIT_SERVER_ENABLE_FILE_WATCHER"] = "false"
+    os.environ["TORCH_DISABLE_STREAMLIT_WATCHER"] = "1"
+    os.environ["LLAMA_INDEX_DISABLE_OPENAI"] = "1"
+
+    if "messages" not in st.session_state:
+        st.session_state.messages = [{
+            "role": "assistant",
+            "content": "Hello! I'm your SHL assessment assistant. How can I help you?"
+        }]
+        st.session_state.messages = [{
+            "role": "assistant",
+            "content": "Hello! I'm your SHL assessment assistant. How can I help you?"
+        }]
+    if "index_built" not in st.session_state:
+        st.session_state["index_built"] = False
+
+
+
+
+    if not st.session_state["index_built"]:
+        try:
+            with st.spinner("Loading data and building index..."):
+                assessment_data = load_data_from_csv(CSV_FILE_PATH)
+                if assessment_data:
+                    build_index(assessment_data)
+                    st.session_state['chat_engine'] = load_chat_engine()
+                    st.session_state["index_built"] = True
+                else:
+                    st.error("Failed to load assessment data. Please check the CSV file.")
+        except Exception as e:
+            st.error(f"Error initializing application: {e}")
+
+    # --- Chat Interface ---
+    chat_engine = st.session_state.get('chat_engine')
+    if chat_engine:
+        for msg in st.session_state.messages:
+            icon = "🤖" if msg["role"] == "assistant" else "👤"
+            with st.chat_message(msg["role"]):
+                st.markdown(f"<span style='color: white;'>{icon} {msg['content']}</span>", unsafe_allow_html=True)
+
+        if prompt := st.chat_input("Ask me about SHL assessments..."):
+            st.session_state.messages.append({"role": "user", "content": prompt})
+            with st.chat_message("user"):
+                st.markdown(f"<span style='color: white;'>👤 {prompt}</span>", unsafe_allow_html=True)
+                
+
+            with st.chat_message("assistant"):
+                try:
+                    # Add formatting instructions to the prompt
+                    formatted_prompt = f"{prompt}. Please provide a list of all matching SHL assessments (minimum 1, maximum 10). For each assessment, include the following details: Assessment Name: [Name], URL: [URL], Remote Testing Support: [Yes/No], Adaptive/IRT Support: [Yes/No], Duration: [Duration], Test Type: [Type]. If there are no matching assessments, please state that."
+                    response = chat_engine.chat(formatted_prompt)
+                    st.markdown(f"<span style='color: white;'>🤖 {response.response}</span>", unsafe_allow_html=True)
+                    st.session_state.messages.append({"role": "assistant", "content": response.response})
+                except Exception as e:
+                    st.error(f"An error occurred during chat: {e}")
+
+    else:
+        st.info("💬 Chat is ready! Ask me anything about SHL assessments.")
+
+if __name__ == "__main__":
+    main()
+

requirements.txt

@@ -0,0 +1,12 @@
+streamlit==1.44.1
+llama-index==0.12.28
+llama-index-core==0.12.28
+llama-index-embeddings-huggingface==0.5.2
+llama-index-llms-groq==0.3.1
+pandas==2.2.3
+python-dotenv==1.1.0
+sentence-transformers==4.0.2  
+groq==0.22.0
+streamlit
+Flask
+gunicorn

shl_assessments.csv

@@ -0,0 +1,101 @@
+Assessment Name,URL,Remote Testing Support,Adaptive/IRT Support,Duration (min),Test Type
+Cognitive Ability Test,https://www.shl.com/solutions/products/cognitive-ability-test/,Yes,Yes,20,Cognitive
+Personality Profiler,https://www.shl.com/solutions/products/personality-profiler/,No,No,25,Personality
+Java Developer Test,https://www.shl.com/solutions/products/java-developer-test/,Yes,No,30,Technical
+Team Collaboration Assessment,https://www.shl.com/solutions/products/team-collaboration-assessment/,No,Yes,35,Behavioral
+SQL Proficiency Test,https://www.shl.com/solutions/products/sql-proficiency-test/,Yes,No,40,Aptitude
+Leadership Style Inventory,https://www.shl.com/solutions/products/leadership-style-inventory/,No,No,45,Situational
+Customer Service Aptitude,https://www.shl.com/solutions/products/customer-service-aptitude/,Yes,Yes,20,Reasoning
+Verbal Reasoning Test,https://www.shl.com/solutions/products/verbal-reasoning-test/,No,No,25,Communication
+Numerical Reasoning Test,https://www.shl.com/solutions/products/numerical-reasoning-test/,Yes,No,30,Cognitive
+Critical Thinking Assessment,https://www.shl.com/solutions/products/critical-thinking-assessment/,No,Yes,35,Personality
+Python Coding Test,https://www.shl.com/solutions/products/python-coding-test/,Yes,No,40,Technical
+Emotional Intelligence Survey,https://www.shl.com/solutions/products/emotional-intelligence-survey/,No,No,45,Behavioral
+Situational Judgment Test,https://www.shl.com/solutions/products/situational-judgment-test/,Yes,Yes,20,Aptitude
+Time Management Evaluation,https://www.shl.com/solutions/products/time-management-evaluation/,No,No,25,Situational
+Data Analysis Assessment,https://www.shl.com/solutions/products/data-analysis-assessment/,Yes,No,30,Reasoning
+Software Engineering Exam,https://www.shl.com/solutions/products/software-engineering-exam/,No,Yes,35,Communication
+Finance & Accounting Aptitude,https://www.shl.com/solutions/products/finance-and-accounting-aptitude/,Yes,No,40,Cognitive
+Logical Reasoning Test,https://www.shl.com/solutions/products/logical-reasoning-test/,No,No,45,Personality
+Communication Skills Test,https://www.shl.com/solutions/products/communication-skills-test/,Yes,Yes,20,Technical
+Machine Learning Quiz,https://www.shl.com/solutions/products/machine-learning-quiz/,No,No,25,Behavioral
+Cognitive Ability Test,https://www.shl.com/solutions/products/cognitive-ability-test/,Yes,No,30,Aptitude
+Personality Profiler,https://www.shl.com/solutions/products/personality-profiler/,No,Yes,35,Situational
+Java Developer Test,https://www.shl.com/solutions/products/java-developer-test/,Yes,No,40,Reasoning
+Team Collaboration Assessment,https://www.shl.com/solutions/products/team-collaboration-assessment/,No,No,45,Communication
+SQL Proficiency Test,https://www.shl.com/solutions/products/sql-proficiency-test/,Yes,Yes,20,Cognitive
+Leadership Style Inventory,https://www.shl.com/solutions/products/leadership-style-inventory/,No,No,25,Personality
+Customer Service Aptitude,https://www.shl.com/solutions/products/customer-service-aptitude/,Yes,No,30,Technical
+Verbal Reasoning Test,https://www.shl.com/solutions/products/verbal-reasoning-test/,No,Yes,35,Behavioral
+Numerical Reasoning Test,https://www.shl.com/solutions/products/numerical-reasoning-test/,Yes,No,40,Aptitude
+Critical Thinking Assessment,https://www.shl.com/solutions/products/critical-thinking-assessment/,No,No,45,Situational
+Python Coding Test,https://www.shl.com/solutions/products/python-coding-test/,Yes,Yes,20,Reasoning
+Emotional Intelligence Survey,https://www.shl.com/solutions/products/emotional-intelligence-survey/,No,No,25,Communication
+Situational Judgment Test,https://www.shl.com/solutions/products/situational-judgment-test/,Yes,No,30,Cognitive
+Time Management Evaluation,https://www.shl.com/solutions/products/time-management-evaluation/,No,Yes,35,Personality
+Data Analysis Assessment,https://www.shl.com/solutions/products/data-analysis-assessment/,Yes,No,40,Technical
+Software Engineering Exam,https://www.shl.com/solutions/products/software-engineering-exam/,No,No,45,Behavioral
+Finance & Accounting Aptitude,https://www.shl.com/solutions/products/finance-and-accounting-aptitude/,Yes,Yes,20,Aptitude
+Logical Reasoning Test,https://www.shl.com/solutions/products/logical-reasoning-test/,No,No,25,Situational
+Communication Skills Test,https://www.shl.com/solutions/products/communication-skills-test/,Yes,No,30,Reasoning
+Machine Learning Quiz,https://www.shl.com/solutions/products/machine-learning-quiz/,No,Yes,35,Communication
+Cognitive Ability Test,https://www.shl.com/solutions/products/cognitive-ability-test/,Yes,No,40,Cognitive
+Personality Profiler,https://www.shl.com/solutions/products/personality-profiler/,No,No,45,Personality
+Java Developer Test,https://www.shl.com/solutions/products/java-developer-test/,Yes,Yes,20,Technical
+Team Collaboration Assessment,https://www.shl.com/solutions/products/team-collaboration-assessment/,No,No,25,Behavioral
+SQL Proficiency Test,https://www.shl.com/solutions/products/sql-proficiency-test/,Yes,No,30,Aptitude
+Leadership Style Inventory,https://www.shl.com/solutions/products/leadership-style-inventory/,No,Yes,35,Situational
+Customer Service Aptitude,https://www.shl.com/solutions/products/customer-service-aptitude/,Yes,No,40,Reasoning
+Verbal Reasoning Test,https://www.shl.com/solutions/products/verbal-reasoning-test/,No,No,45,Communication
+Numerical Reasoning Test,https://www.shl.com/solutions/products/numerical-reasoning-test/,Yes,Yes,20,Cognitive
+Critical Thinking Assessment,https://www.shl.com/solutions/products/critical-thinking-assessment/,No,No,25,Personality
+Python Coding Test,https://www.shl.com/solutions/products/python-coding-test/,Yes,No,30,Technical
+Emotional Intelligence Survey,https://www.shl.com/solutions/products/emotional-intelligence-survey/,No,Yes,35,Behavioral
+Situational Judgment Test,https://www.shl.com/solutions/products/situational-judgment-test/,Yes,No,40,Aptitude
+Time Management Evaluation,https://www.shl.com/solutions/products/time-management-evaluation/,No,No,45,Situational
+Data Analysis Assessment,https://www.shl.com/solutions/products/data-analysis-assessment/,Yes,Yes,20,Reasoning
+Software Engineering Exam,https://www.shl.com/solutions/products/software-engineering-exam/,No,No,25,Communication
+Finance & Accounting Aptitude,https://www.shl.com/solutions/products/finance-and-accounting-aptitude/,Yes,No,30,Cognitive
+Logical Reasoning Test,https://www.shl.com/solutions/products/logical-reasoning-test/,No,Yes,35,Personality
+Communication Skills Test,https://www.shl.com/solutions/products/communication-skills-test/,Yes,No,40,Technical
+Machine Learning Quiz,https://www.shl.com/solutions/products/machine-learning-quiz/,No,No,45,Behavioral
+Cognitive Ability Test,https://www.shl.com/solutions/products/cognitive-ability-test/,Yes,Yes,20,Aptitude
+Personality Profiler,https://www.shl.com/solutions/products/personality-profiler/,No,No,25,Situational
+Java Developer Test,https://www.shl.com/solutions/products/java-developer-test/,Yes,No,30,Reasoning
+Team Collaboration Assessment,https://www.shl.com/solutions/products/team-collaboration-assessment/,No,Yes,35,Communication
+SQL Proficiency Test,https://www.shl.com/solutions/products/sql-proficiency-test/,Yes,No,40,Cognitive
+Leadership Style Inventory,https://www.shl.com/solutions/products/leadership-style-inventory/,No,No,45,Personality
+Customer Service Aptitude,https://www.shl.com/solutions/products/customer-service-aptitude/,Yes,Yes,20,Technical
+Verbal Reasoning Test,https://www.shl.com/solutions/products/verbal-reasoning-test/,No,No,25,Behavioral
+Numerical Reasoning Test,https://www.shl.com/solutions/products/numerical-reasoning-test/,Yes,No,30,Aptitude
+Critical Thinking Assessment,https://www.shl.com/solutions/products/critical-thinking-assessment/,No,Yes,35,Situational
+Python Coding Test,https://www.shl.com/solutions/products/python-coding-test/,Yes,No,40,Reasoning
+Emotional Intelligence Survey,https://www.shl.com/solutions/products/emotional-intelligence-survey/,No,No,45,Communication
+Situational Judgment Test,https://www.shl.com/solutions/products/situational-judgment-test/,Yes,Yes,20,Cognitive
+Time Management Evaluation,https://www.shl.com/solutions/products/time-management-evaluation/,No,No,25,Personality
+Data Analysis Assessment,https://www.shl.com/solutions/products/data-analysis-assessment/,Yes,No,30,Technical
+Software Engineering Exam,https://www.shl.com/solutions/products/software-engineering-exam/,No,Yes,35,Behavioral
+Finance & Accounting Aptitude,https://www.shl.com/solutions/products/finance-and-accounting-aptitude/,Yes,No,40,Aptitude
+Logical Reasoning Test,https://www.shl.com/solutions/products/logical-reasoning-test/,No,No,45,Situational
+Communication Skills Test,https://www.shl.com/solutions/products/communication-skills-test/,Yes,Yes,20,Reasoning
+Machine Learning Quiz,https://www.shl.com/solutions/products/machine-learning-quiz/,No,No,25,Communication
+Cognitive Ability Test,https://www.shl.com/solutions/products/cognitive-ability-test/,Yes,No,30,Cognitive
+Personality Profiler,https://www.shl.com/solutions/products/personality-profiler/,No,Yes,35,Personality
+Java Developer Test,https://www.shl.com/solutions/products/java-developer-test/,Yes,No,40,Technical
+Team Collaboration Assessment,https://www.shl.com/solutions/products/team-collaboration-assessment/,No,No,45,Behavioral
+SQL Proficiency Test,https://www.shl.com/solutions/products/sql-proficiency-test/,Yes,Yes,20,Aptitude
+Leadership Style Inventory,https://www.shl.com/solutions/products/leadership-style-inventory/,No,No,25,Situational
+Customer Service Aptitude,https://www.shl.com/solutions/products/customer-service-aptitude/,Yes,No,30,Reasoning
+Verbal Reasoning Test,https://www.shl.com/solutions/products/verbal-reasoning-test/,No,Yes,35,Communication
+Numerical Reasoning Test,https://www.shl.com/solutions/products/numerical-reasoning-test/,Yes,No,40,Cognitive
+Critical Thinking Assessment,https://www.shl.com/solutions/products/critical-thinking-assessment/,No,No,45,Personality
+Python Coding Test,https://www.shl.com/solutions/products/python-coding-test/,Yes,Yes,20,Technical
+Emotional Intelligence Survey,https://www.shl.com/solutions/products/emotional-intelligence-survey/,No,No,25,Behavioral
+Situational Judgment Test,https://www.shl.com/solutions/products/situational-judgment-test/,Yes,No,30,Aptitude
+Time Management Evaluation,https://www.shl.com/solutions/products/time-management-evaluation/,No,Yes,35,Situational
+Data Analysis Assessment,https://www.shl.com/solutions/products/data-analysis-assessment/,Yes,No,40,Reasoning
+Software Engineering Exam,https://www.shl.com/solutions/products/software-engineering-exam/,No,No,45,Communication
+Finance & Accounting Aptitude,https://www.shl.com/solutions/products/finance-and-accounting-aptitude/,Yes,Yes,20,Cognitive
+Logical Reasoning Test,https://www.shl.com/solutions/products/logical-reasoning-test/,No,No,25,Personality
+Communication Skills Test,https://www.shl.com/solutions/products/communication-skills-test/,Yes,No,30,Technical
+Machine Learning Quiz,https://www.shl.com/solutions/products/machine-learning-quiz/,No,Yes,35,Behavioral

storage/graph_store.json

@@ -0,0 +1 @@
+{"graph_dict": {}}

storage/image__vector_store.json

@@ -0,0 +1 @@
+{"embedding_dict": {}, "text_id_to_ref_doc_id": {}, "metadata_dict": {}}

storage/index_store.json

@@ -0,0 +1 @@
+{"index_store/data": {"d05d7679-e3c3-4463-8ebb-3546191f3a69": {"__type__": "vector_store", "__data__": "{\"index_id\": \"d05d7679-e3c3-4463-8ebb-3546191f3a69\", \"summary\": null, \"nodes_dict\": {\"1c33403f-62ee-4a45-8c7e-4a1a8d295921\": \"1c33403f-62ee-4a45-8c7e-4a1a8d295921\", \"94d2cdb9-f5bd-41fa-bf79-92f1d4b03898\": \"94d2cdb9-f5bd-41fa-bf79-92f1d4b03898\", \"fa619f4e-9527-4fd1-ae9c-701f88213a71\": \"fa619f4e-9527-4fd1-ae9c-701f88213a71\", \"ae0ad3e2-3a03-4520-b182-d62d8006c433\": \"ae0ad3e2-3a03-4520-b182-d62d8006c433\", \"06e1cf6f-ef51-439f-bc9f-3baf7233f443\": \"06e1cf6f-ef51-439f-bc9f-3baf7233f443\", \"2b84c0ec-82d8-49a4-ade7-77bcbeffe9f4\": \"2b84c0ec-82d8-49a4-ade7-77bcbeffe9f4\", \"1233d6bf-5664-4e92-9581-5be18b141baa\": \"1233d6bf-5664-4e92-9581-5be18b141baa\", \"523f5875-5fa0-46ad-980c-b0777c53af31\": \"523f5875-5fa0-46ad-980c-b0777c53af31\", \"b1b769b7-6f0a-4728-b00c-29c41ec07af7\": \"b1b769b7-6f0a-4728-b00c-29c41ec07af7\", \"187bf2bd-3bbb-47df-80e9-672cf571d181\": \"187bf2bd-3bbb-47df-80e9-672cf571d181\", \"5d43e57c-eada-4b04-83f0-96fa9bea2765\": \"5d43e57c-eada-4b04-83f0-96fa9bea2765\", \"20ffb540-1ef5-4c9f-921b-ea7f872f685c\": \"20ffb540-1ef5-4c9f-921b-ea7f872f685c\", \"3addec3f-721d-4b64-a709-5a5380a0aba4\": \"3addec3f-721d-4b64-a709-5a5380a0aba4\", \"8438f7ae-1514-472d-80b4-9a4d60f48e6b\": \"8438f7ae-1514-472d-80b4-9a4d60f48e6b\", \"5da3e70f-6934-4c40-9d5b-acf7aeeac6a3\": \"5da3e70f-6934-4c40-9d5b-acf7aeeac6a3\", \"995edbf0-74e1-4083-8f60-98f89b680336\": \"995edbf0-74e1-4083-8f60-98f89b680336\", \"02602778-ab96-4e35-9430-03c30aabe5ab\": \"02602778-ab96-4e35-9430-03c30aabe5ab\", \"5b736259-6f8b-4aa3-9ceb-dfd1e4620c3a\": \"5b736259-6f8b-4aa3-9ceb-dfd1e4620c3a\", \"540730ac-02c6-4c53-b1af-1b99cd6ce86a\": \"540730ac-02c6-4c53-b1af-1b99cd6ce86a\", \"7ab316eb-e53e-49ef-bdce-2d4152f78c9e\": \"7ab316eb-e53e-49ef-bdce-2d4152f78c9e\", \"d55e4b8f-579c-4e4e-a3cb-dbfcfd38463c\": \"d55e4b8f-579c-4e4e-a3cb-dbfcfd38463c\", \"7cd5bbf6-222b-451a-9adc-5215048d4219\": \"7cd5bbf6-222b-451a-9adc-5215048d4219\", \"b9b546c1-e457-4368-ad28-843ef5bd4196\": \"b9b546c1-e457-4368-ad28-843ef5bd4196\", \"e617ad14-34b7-4d09-b33c-b17b7b480a2a\": \"e617ad14-34b7-4d09-b33c-b17b7b480a2a\", \"8bcd6f8e-b7fc-4328-ad2d-68c9510b02da\": \"8bcd6f8e-b7fc-4328-ad2d-68c9510b02da\", \"8c32be74-a616-449f-a0f7-451c8d4f3232\": \"8c32be74-a616-449f-a0f7-451c8d4f3232\", \"980a6d6c-44af-4802-a8a4-855c6945c637\": \"980a6d6c-44af-4802-a8a4-855c6945c637\", \"1917b51e-cb83-4c91-82ce-cae806dbde88\": \"1917b51e-cb83-4c91-82ce-cae806dbde88\", \"32f55832-2c1a-49f4-b5f5-5c1a4ce91f71\": \"32f55832-2c1a-49f4-b5f5-5c1a4ce91f71\", \"8466a2c8-16a8-4bfc-bb0d-bad4aed77bc0\": \"8466a2c8-16a8-4bfc-bb0d-bad4aed77bc0\", \"754e6307-ef57-4c74-8e69-ad123ab179a2\": \"754e6307-ef57-4c74-8e69-ad123ab179a2\", \"e4890f6e-e88e-41af-8906-720b2154ac30\": \"e4890f6e-e88e-41af-8906-720b2154ac30\", \"1a753576-b223-41a1-91ae-af50f10eca6f\": \"1a753576-b223-41a1-91ae-af50f10eca6f\", \"1450402c-60fe-4173-aa26-9fd29e35dc0e\": \"1450402c-60fe-4173-aa26-9fd29e35dc0e\", \"8538e7e2-51fa-4177-9fd3-c0a7beef9945\": \"8538e7e2-51fa-4177-9fd3-c0a7beef9945\", \"f7f9e2d6-cdd0-4000-af62-07aad08a6add\": \"f7f9e2d6-cdd0-4000-af62-07aad08a6add\", \"5969c678-6c10-4e40-ab81-cb81f4230004\": \"5969c678-6c10-4e40-ab81-cb81f4230004\", \"f28a2e8f-a3e9-4223-ba20-e16dd7f75566\": \"f28a2e8f-a3e9-4223-ba20-e16dd7f75566\", \"cad9cc59-708e-4564-98f0-0c8fec149377\": \"cad9cc59-708e-4564-98f0-0c8fec149377\", \"a1f0ccb5-c233-4e1c-bcf6-1ea87e7ba749\": \"a1f0ccb5-c233-4e1c-bcf6-1ea87e7ba749\", \"0ae1fe03-dc86-444f-8579-056647ae1758\": \"0ae1fe03-dc86-444f-8579-056647ae1758\", \"77221aed-93b6-4299-ba17-4c07113db5fe\": \"77221aed-93b6-4299-ba17-4c07113db5fe\", \"200dd2d9-eece-4cd4-bbcd-4ff9f36e4960\": \"200dd2d9-eece-4cd4-bbcd-4ff9f36e4960\", \"4ef820cc-20bb-47f7-bc23-3ebf974d2b5c\": \"4ef820cc-20bb-47f7-bc23-3ebf974d2b5c\", \"ec52500d-12a9-4de6-b11a-87b06eac8f31\": \"ec52500d-12a9-4de6-b11a-87b06eac8f31\", \"3bc85d9e-6c26-458e-a5ad-fb2b263fca7a\": \"3bc85d9e-6c26-458e-a5ad-fb2b263fca7a\", \"9e4e052b-956e-4856-bd05-c4a7723ba2c4\": \"9e4e052b-956e-4856-bd05-c4a7723ba2c4\", \"5911c107-f421-41d7-973c-2d208f784693\": \"5911c107-f421-41d7-973c-2d208f784693\", \"619bfdc1-a169-4e65-b197-78b9bef745c8\": \"619bfdc1-a169-4e65-b197-78b9bef745c8\", \"8b3112e4-1ad7-4a9d-a4a4-6f7f3e598124\": \"8b3112e4-1ad7-4a9d-a4a4-6f7f3e598124\", \"d2b2745d-a122-4dce-b9f1-a8d4e0fe8d74\": \"d2b2745d-a122-4dce-b9f1-a8d4e0fe8d74\", \"b9c0d3da-7c39-46b0-bcd9-0bf10b678d01\": \"b9c0d3da-7c39-46b0-bcd9-0bf10b678d01\", \"0f97c2ee-ab52-4487-8fbe-7170421d5dca\": \"0f97c2ee-ab52-4487-8fbe-7170421d5dca\", \"3c7c8128-6c56-418b-97e9-94221af751ea\": \"3c7c8128-6c56-418b-97e9-94221af751ea\", \"a7c3ca4d-0e8e-40b5-bfc9-3fe82d457bfc\": \"a7c3ca4d-0e8e-40b5-bfc9-3fe82d457bfc\", \"b6cdf91b-5b9d-4aef-9f0e-f52361934133\": \"b6cdf91b-5b9d-4aef-9f0e-f52361934133\", \"22a8d6f1-7b24-4ba2-8b94-adc105330ef8\": \"22a8d6f1-7b24-4ba2-8b94-adc105330ef8\", \"c72b32ae-e7e2-4056-9266-19ffc974d05f\": \"c72b32ae-e7e2-4056-9266-19ffc974d05f\", \"917897e6-c6f1-4d0b-acd9-8e0c919af3d2\": \"917897e6-c6f1-4d0b-acd9-8e0c919af3d2\", \"9565237a-1f5c-48d9-84ca-34f60145891f\": \"9565237a-1f5c-48d9-84ca-34f60145891f\", \"61b2074a-77ab-465f-8e77-a6680d252cb6\": \"61b2074a-77ab-465f-8e77-a6680d252cb6\", \"9d289d93-3ab4-490b-ad8d-4c378fe554b3\": \"9d289d93-3ab4-490b-ad8d-4c378fe554b3\", \"c8c0261a-5fbc-41b4-b34a-8b2098400427\": \"c8c0261a-5fbc-41b4-b34a-8b2098400427\", \"373ed326-a94c-40f6-80a7-1be07052d343\": \"373ed326-a94c-40f6-80a7-1be07052d343\", \"3826da16-4c66-4b29-a6fa-3f7102c6b379\": \"3826da16-4c66-4b29-a6fa-3f7102c6b379\", \"56d19ec3-4e0a-49c0-922d-eea6884c785c\": \"56d19ec3-4e0a-49c0-922d-eea6884c785c\", \"4ad46bae-4f08-448c-b892-4268764fe1bc\": \"4ad46bae-4f08-448c-b892-4268764fe1bc\", \"173ff29a-d800-4d7d-ba2b-da737169f2cf\": \"173ff29a-d800-4d7d-ba2b-da737169f2cf\", \"5db5a672-91a9-4798-bead-8d400c0d0bb1\": \"5db5a672-91a9-4798-bead-8d400c0d0bb1\", \"7c2f64bb-ec14-4b8d-bfd0-5b14191c5809\": \"7c2f64bb-ec14-4b8d-bfd0-5b14191c5809\", \"2f278108-f352-41d0-82f7-1828de3fb16e\": \"2f278108-f352-41d0-82f7-1828de3fb16e\", \"ad259806-6807-45d9-a425-8a8c73a5f4ba\": \"ad259806-6807-45d9-a425-8a8c73a5f4ba\", \"be5dabf6-2231-4d00-bc61-7c62a61b7fc4\": \"be5dabf6-2231-4d00-bc61-7c62a61b7fc4\", \"09b364a9-d72d-4623-9800-4ebc13774ba7\": \"09b364a9-d72d-4623-9800-4ebc13774ba7\", \"2165fda3-a9a8-4830-9a61-421008e38c0c\": \"2165fda3-a9a8-4830-9a61-421008e38c0c\", \"b043c9d3-0b83-4ce4-ae23-6161b069af66\": \"b043c9d3-0b83-4ce4-ae23-6161b069af66\", \"36714536-310e-4c8e-9e7c-4ccdf48cc2b7\": \"36714536-310e-4c8e-9e7c-4ccdf48cc2b7\", \"e325972d-c75a-4020-a535-5c66932fd90e\": \"e325972d-c75a-4020-a535-5c66932fd90e\", \"53e85b60-39b3-4ae7-ab12-40433c4fea8f\": \"53e85b60-39b3-4ae7-ab12-40433c4fea8f\", \"6a38fae6-b26d-419f-a7ee-7bcf9a316ed4\": \"6a38fae6-b26d-419f-a7ee-7bcf9a316ed4\", \"a3e73773-7d62-4a6d-9ecd-5e0a1c581f9b\": \"a3e73773-7d62-4a6d-9ecd-5e0a1c581f9b\", \"330ee559-4eeb-4a4e-8ba4-d9ad41a8a6d4\": \"330ee559-4eeb-4a4e-8ba4-d9ad41a8a6d4\", \"eb05cf8d-2988-4a56-a27f-daeba8cd4351\": \"eb05cf8d-2988-4a56-a27f-daeba8cd4351\", \"3ceab828-6131-4e14-a38c-b7a49e93fa00\": \"3ceab828-6131-4e14-a38c-b7a49e93fa00\", \"575261c8-b595-449c-9910-f731dfbbc524\": \"575261c8-b595-449c-9910-f731dfbbc524\", \"2b804415-dea0-4228-bb01-cdef7b7314f1\": \"2b804415-dea0-4228-bb01-cdef7b7314f1\", \"b81e8926-a94d-4a0d-a131-2b094e0f544e\": \"b81e8926-a94d-4a0d-a131-2b094e0f544e\", \"67438345-98b2-4ecc-9a32-7737b81a377f\": \"67438345-98b2-4ecc-9a32-7737b81a377f\", \"22912616-4dd1-4fa9-a721-b703015efbce\": \"22912616-4dd1-4fa9-a721-b703015efbce\", \"14d8706f-030b-41a5-89ba-9c8da9ffa437\": \"14d8706f-030b-41a5-89ba-9c8da9ffa437\", \"046bddfd-74ca-4f19-90fb-7e467fd20451\": \"046bddfd-74ca-4f19-90fb-7e467fd20451\", \"df56b2a3-3630-401b-ad25-7f8102b56957\": \"df56b2a3-3630-401b-ad25-7f8102b56957\", \"28e480fb-1b36-4316-99e2-4ceccc37f0ca\": \"28e480fb-1b36-4316-99e2-4ceccc37f0ca\", \"c3466ff2-beeb-4d67-8812-60e43ada861c\": \"c3466ff2-beeb-4d67-8812-60e43ada861c\", \"ac7db4fa-eba7-418f-9b7e-0be0148f52e6\": \"ac7db4fa-eba7-418f-9b7e-0be0148f52e6\", \"cf422ea9-3371-407d-a5de-3718a41642ff\": \"cf422ea9-3371-407d-a5de-3718a41642ff\", \"aaa8a4bb-d027-48bd-b725-4637335e425c\": \"aaa8a4bb-d027-48bd-b725-4637335e425c\", \"9d7f56dd-17c7-440c-8295-bc3a84e11317\": \"9d7f56dd-17c7-440c-8295-bc3a84e11317\", \"747c0f41-c1cc-461d-bb8e-ea78071f0bc3\": \"747c0f41-c1cc-461d-bb8e-ea78071f0bc3\", \"0bd536f3-3582-4159-aa20-9e3bd05e223a\": \"0bd536f3-3582-4159-aa20-9e3bd05e223a\"}, \"doc_id_dict\": {}, \"embeddings_dict\": {}}"}}}