Upload 35 files

.gitattributes

@@ -33,3 +33,8 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+data/docs_2/Attention_Is_All_You_Need.pdf filter=lfs diff=lfs merge=lfs -text
+data/docs/Internal[[:space:]]Medicine,[[:space:]]Getachew[[:space:]]Tizazu,[[:space:]]Tadesse[[:space:]]Anteneh.pdf filter=lfs diff=lfs merge=lfs -text
+data/vectordb/processed/chroma/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+data/vectordb1/processed/chroma/chroma.sqlite3 filter=lfs diff=lfs merge=lfs -text
+images/test.png filter=lfs diff=lfs merge=lfs -text

configs/app_config.yml

@@ -0,0 +1,46 @@
+directories:
+  data_directory: data/docs
+  data_directory_2: data/docs_2
+  persist_directory: data/vectordb/processed/chroma/
+  custom_persist_directory: data/vectordb/uploaded/chroma/
+
+llm_config:
+    embedding_model: "NeuML/pubmedbert-base-embeddings"
+    llm_system_role_with_history: "Answer the question based on the given content without using on your own knowledge.
+    You will receive a prompt with the the following format:
+
+    # Chat history:\n
+    [user query, response]\n\n
+
+    # Retrieved content number:\n
+    Content\n\n
+    Source\n\n
+
+    # User question:\n
+    New question
+    "
+    llm_system_role_without_history: "In the following you recieve a prompt. 
+    Answer it based on given content. Provide only the response, dont say 'Answer:'."
+    engine: "BioMistral/BioMistral-7B"
+    temperature: 0.1
+    device: "cuda"
+    max_new_tokens: 4096
+    do_sample: True
+    top_k: 10
+    top_p: 0.1
+    add_history: False
+splitter_config:
+  chunk_size: 1500
+  chunk_overlap: 250
+
+retrieval_config:
+  k: 2
+
+serve:
+  port: 8000
+
+memory:
+  number_of_q_a_pairs: 2
+
+
+  

data/docs/Internal Medicine, Getachew Tizazu, Tadesse Anteneh.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0169c575e3a4a5e39326a18ad57f1c0f82d9e78d33d06fbb895d395394d75b0a
+size 2035855

data/docs_2/Attention_Is_All_You_Need.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b7d72988fd8107d07f7d278bf0ba6621adb6ed47df74be4014fa4a01f03aff6a
+size 2215244

data/vectordb/processed/chroma/6987cbb9-f35b-4396-b0c0-a911c7c462df/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d17e4a33d5f6f07d497f04499ed371b0b9688fb5fd10be3295b8cd02c9bcd4e
+size 3212000

data/vectordb/processed/chroma/6987cbb9-f35b-4396-b0c0-a911c7c462df/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b27fdd8437a04493bf8a7d1f9a2e0fe99a426f9dd1cd05fc39e36645b2f4892
+size 100

data/vectordb/processed/chroma/6987cbb9-f35b-4396-b0c0-a911c7c462df/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:aaabaf8bd5deb2ce724216d3d22742448e6bb36ad6fd47a8b7546943ac7bc2d9
+size 55974

data/vectordb/processed/chroma/6987cbb9-f35b-4396-b0c0-a911c7c462df/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd0e081293048e3304f94190eb77ea8a43b86ea1d325125dde693add12c583c0
+size 4000

data/vectordb/processed/chroma/6987cbb9-f35b-4396-b0c0-a911c7c462df/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d369861227544a7d964868dd4119f8d689bf3bad375dfbde0801806d57931359
+size 8624

data/vectordb/processed/chroma/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b2e23cc1f35dd063d2cc3d566647382f22b7f130a53c30a82f65e7d57d22658
+size 13520896

data/vectordb1/processed/chroma/chroma.sqlite3

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5159675380044145b50f2c946dcb6015b30199c9475fc4d3763410a052770fca
+size 14123008

data/vectordb1/processed/chroma/d65c6700-15e7-4d1b-8a9c-16cefb1f4e1e/data_level0.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d340a83df102dfb094b470e14917af51d9cc9b4d2899aac7e8d03f5281dae4af
+size 4236000

data/vectordb1/processed/chroma/d65c6700-15e7-4d1b-8a9c-16cefb1f4e1e/header.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2a4f6ac52066e3cc07e4dbd3cf3ba996e47b2dc0f3f3b483d9667139b648464d
+size 100

data/vectordb1/processed/chroma/d65c6700-15e7-4d1b-8a9c-16cefb1f4e1e/index_metadata.pickle

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6791402d0c0c55745efcc92b7b2b686b854ca13405ce507c8019494d5be482cb
+size 55974

data/vectordb1/processed/chroma/d65c6700-15e7-4d1b-8a9c-16cefb1f4e1e/length.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4570a3076ed944f19171251810fe12aa927bf0befa0961c875f5838be34af681
+size 4000

data/vectordb1/processed/chroma/d65c6700-15e7-4d1b-8a9c-16cefb1f4e1e/link_lists.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:15f5b49c9cc9564ead2b3490bfd89131af9469aec01ba17df0234f56400380aa
+size 8624

src/app.py

@@ -0,0 +1,116 @@
+import gradio as gr
+from utils.upload_file import UploadFile
+from utils.chatbot import ChatBot
+from utils.ui_settings import UISettings
+from utils.load_config import LoadConfig
+
+APPCFG = LoadConfig()
+# # Prepare the LLm and Tokenizer
+# tokenizer = AutoTokenizer.from_pretrained(
+#     APPCFG.llm_engine, token=APPCFG.gemma_token, device=APPCFG.device)
+# model = model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path="google/gemma-7b-it",
+#                                                      token=APPCFG.gemma_token,
+#                                                      torch_dtype=torch.float16,
+#                                                      device_map=APPCFG.device
+#                                                      )
+# app_pipeline = pipeline(
+#     "text-generation",
+#     model=model,
+#     tokenizer=tokenizer
+# )
+with gr.Blocks() as demo:
+    with gr.Tabs():
+        with gr.TabItem("Med-App"):
+            ##############
+            # First ROW:
+            ##############
+            with gr.Row() as row_one:
+                with gr.Column(visible=False) as reference_bar:
+                    ref_output = gr.Markdown()
+
+                with gr.Column() as chatbot_output:
+                    chatbot = gr.Chatbot(
+                        [],
+                        elem_id="chatbot",
+                        bubble_full_width=False,
+                        height=500,
+                        avatar_images=(
+                            ("images/test.png"), "images/Gemma-logo.png"),
+                        # render=False
+                    )
+                    # **Adding like/dislike icons
+                    chatbot.like(UISettings.feedback, None, None)
+            ##############
+            # SECOND ROW:
+            ##############
+            with gr.Row():
+                input_txt = gr.Textbox(
+                    lines=4,
+                    scale=8,
+                    placeholder="Enter text and press enter, or upload PDF files",
+                    container=False,
+                )
+
+            ##############
+            # Third ROW:
+            ##############
+            with gr.Row() as row_two:
+                text_submit_btn = gr.Button(value="Submit text")
+                sidebar_state = gr.State(False)
+                btn_toggle_sidebar = gr.Button(
+                    value="References")
+                btn_toggle_sidebar.click(UISettings.toggle_sidebar, [sidebar_state], [
+                    reference_bar, sidebar_state])
+                upload_btn = gr.UploadButton(
+                    "📁 Upload PDF or doc files", file_types=[
+                        '.pdf',
+                        '.doc'
+                    ],
+                    file_count="multiple")
+                clear_button = gr.ClearButton([input_txt, chatbot])
+                rag_with_dropdown = gr.Dropdown(
+                    label="RAG with", choices=["Preprocessed doc", "Upload doc: Process for RAG"], value="Preprocessed doc")
+            ##############
+            # Fourth ROW:
+            ##############
+            with gr.Row() as row_four:
+                temperature_bar = gr.Slider(minimum=0.1, maximum=1, value=0.1, step=0.1,
+                                            label="Temperature", info="Increasing the temperature will make the model answer more creatively.")
+                top_k = gr.Slider(minimum=0.0,
+                                  maximum=100.0,
+                                  step=1,
+                                  label="top_k",
+                                  value=50,
+                                  info="A lower value (e.g. 10) will result in more conservative answers.")
+                top_p = gr.Slider(minimum=0.0,
+                                  maximum=1.0,
+                                  step=0.01,
+                                  label="top_p",
+                                  value=0.95,
+                                  info=" Works together with top-k. lower value (e.g., 0.5) will generate more focused and conservative text. (Default: 0.0)")
+
+            ##############
+            # Process:
+            ##############
+            file_msg = upload_btn.upload(fn=UploadFile.process_uploaded_files, inputs=[
+                upload_btn, chatbot, rag_with_dropdown], outputs=[input_txt, chatbot], queue=False)
+
+            txt_msg = input_txt.submit(fn=ChatBot.respond,
+                                       inputs=[chatbot, input_txt,
+                                               rag_with_dropdown, temperature_bar, top_k, top_p],
+                                       outputs=[input_txt,
+                                                chatbot, ref_output],
+                                       queue=False).then(lambda: gr.Textbox(interactive=True),
+                                                         None, [input_txt], queue=False)
+
+            txt_msg = text_submit_btn.click(fn=ChatBot.respond,
+                                            inputs=[chatbot, input_txt,
+                                                    rag_with_dropdown, temperature_bar, top_k, top_p],
+                                            outputs=[input_txt,
+                                                     chatbot, ref_output],
+                                            queue=False).then(lambda: gr.Textbox(interactive=True),
+                                                              None, [input_txt], queue=False)
+
+
+if __name__ == "__main__":
+    demo.launch()

src/llm_serve.py

@@ -0,0 +1,50 @@
+import torch
+from flask import Flask, request, jsonify
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from utils.load_config import LoadConfig
+
+APPCFG = LoadConfig()
+
+app = Flask(__name__)
+
+# Load the LLM and Tokenizer
+tokenizer = AutoTokenizer.from_pretrained(
+    APPCFG.llm_engine, token=APPCFG.gemma_token, device=APPCFG.device)
+model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path="BioMistral/BioMistral-7B",
+                                             token=APPCFG.gemma_token,
+                                             torch_dtype=torch.float16,
+                                             device_map=APPCFG.device
+                                             )
+app_pipeline = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer
+)
+
+
+@app.route("/generate_text", methods=["POST"])
+def generate_Text():
+    data = request.json
+    prompt = data.get("prompt", "")
+    max_new_tokens = data.get("max_new_tokens", 1000)
+    do_sample = data.get("do_sample", True)
+    temperature = data.get("temperature", 0.1)
+    top_k = data.get("top_k", 50)
+    top_p = data.get("top_p", 0.95)
+
+    tokenized_prompt = app_pipeline.tokenizer.apply_chat_template(
+        prompt, tokenize=False, add_generation_prompt=True)
+    outputs = app_pipeline(
+        tokenized_prompt,
+        max_new_tokens=max_new_tokens,
+        do_sample=do_sample,
+        temperature=temperature,
+        top_k=top_k,
+        top_p=top_p
+    )
+
+    return jsonify({"response": outputs[0]["generated_text"][len(tokenized_prompt):]})
+
+
+if __name__ == "__main__":
+    app.run(debug=False, port=8888)

src/llm_service.py

@@ -0,0 +1,58 @@
+import torch
+from flask import Flask, request, jsonify
+from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
+from utils.load_config import LoadConfig
+APPCFG = LoadConfig()
+
+app = Flask(__name__)
+
+# Load the LLM and Tokenizer
+tokenizer = AutoTokenizer.from_pretrained(
+    APPCFG.llm_engine, token=APPCFG.gemma_token, device=APPCFG.device)
+model = AutoModelForCausalLM.from_pretrained(pretrained_model_name_or_path="BioMistral/BioMistral-7B",
+                                             token=APPCFG.gemma_token,
+                                             torch_dtype=torch.float16,
+                                             device_map=APPCFG.device
+                                             )
+app_pipeline = pipeline(
+    "text-generation",
+    model=model,
+    tokenizer=tokenizer
+)
+
+# Endpoint to generate text
+
+
+@app.route("/generate_text", methods=["POST"])
+def generate_text():
+    data = request.json
+    prompt = data.get("prompt", "")
+    max_new_tokens = data.get("max_new_tokens", 1000)
+    do_sample = data.get("do_sample", True)
+    temperature = data.get("temperature", 0.1)
+    top_k = data.get("top_k", 50)
+    top_p = data.get("top_p", 0.95)
+    tokenized_prompt = app_pipeline.tokenizer.apply_chat_template(
+        prompt, tokenize=False, add_generation_prompt=True)
+
+    # Generate text based on the prompt
+    response = app_pipeline(
+        tokenized_prompt,
+        max_new_tokens=max_new_tokens,
+        do_sample=do_sample,
+        temperature=temperature,
+        top_k=top_k,
+        top_p=top_p
+    )
+    print("==================")
+    print("top_k:", top_k, "top_p:", top_p, "temperature:",
+          temperature, "max_new_tokens:", max_new_tokens)
+    print("==================")
+    print(response[0]["generated_text"][len(tokenized_prompt):])
+    print("==================")
+
+    return jsonify({"response": response[0]["generated_text"][len(tokenized_prompt):]})
+
+
+if __name__ == "__main__":
+    app.run(debug=False, port=8888)

src/reference_serve.py

@@ -0,0 +1,76 @@
+import http.server
+import socketserver
+import yaml
+import os
+from pyprojroot import here
+
+with open(here("configs/app_config.yml")) as cfg:
+    app_config = yaml.load(cfg, Loader=yaml.FullLoader)
+
+PORT = app_config["serve"]["port"]
+DIRECTORY1 = app_config["directories"]["data_directory"]
+DIRECTORY2 = app_config["directories"]["data_directory_2"]
+
+
+class SingleDirectoryHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+    """
+    Custom HTTP request handler that serves files from a single directory.
+
+    This class extends the SimpleHTTPRequestHandler and sets the serving directory to DIRECTORY1.
+    """
+
+    def __init__(self, *args, **kwargs):
+        """
+        Initialize the SingleDirectoryHTTPRequestHandler.
+
+        Parameters:
+            args: Additional positional arguments for the base class.
+            kwargs: Additional keyword arguments for the base class.
+        """
+        super().__init__(*args, directory=DIRECTORY1, **kwargs)
+
+
+class MultiDirectoryHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
+    """
+    Custom HTTP request handler that serves files from multiple directories.
+
+    This class extends the SimpleHTTPRequestHandler and allows serving files from DIRECTORY1 and DIRECTORY2
+    based on the first directory component in the requested path.
+    """
+
+    def translate_path(self, path):
+        """
+        Translate the requested path to the actual file path.
+
+        Parameters:
+            path (str): The requested path.
+
+        Returns:
+            str: The translated file path.
+        """
+        # Split the path to get the first directory component
+        parts = path.split('/', 2)
+        if len(parts) > 1:
+            first_directory = parts[1]
+            # Check if the first directory matches any of your target directories
+            if first_directory == os.path.basename(DIRECTORY1):
+                path = os.path.join(DIRECTORY1, *parts[2:])
+
+            elif first_directory == os.path.basename(DIRECTORY2):
+                path = os.path.join(DIRECTORY2, *parts[2:])
+            else:
+                # If the first part of the path is not a directory, check both directories for the file
+                file_path1 = os.path.join(DIRECTORY1, first_directory)
+                file_path2 = os.path.join(DIRECTORY2, first_directory)
+                if os.path.isfile(file_path1):
+                    return file_path1
+                elif os.path.isfile(file_path2):
+                    return file_path2
+        # If there's no match, use the default directory
+        return super().translate_path(path)
+
+
+if __name__ == "__main__":
+    with socketserver.TCPServer(("", PORT), MultiDirectoryHTTPRequestHandler) as httpd:
+        print(f"Serving at port {PORT}")
+        httpd.serve_forever()

src/upload_data_manually.py

@@ -0,0 +1,35 @@
+
+import os
+from utils.prepare_vectordb import PrepareVectorDB
+from utils.load_config import LoadConfig
+CONFIG = LoadConfig()
+
+
+def upload_data_manually() -> None:
+    """
+    Uploads data manually to the VectorDB.
+
+    This function initializes a PrepareVectorDB instance with configuration parameters
+    and chunk_overlap. It then checks if the VectorDB already exists in the specified
+    persist_directory. If not, it calls the prepare_and_save_vectordb method to
+    create and save the VectorDB. If the VectorDB already exists, a message is printed
+    indicating its presence.
+
+    Returns:
+        None
+    """
+    prepare_vectordb_instance = PrepareVectorDB(
+        data_directory=CONFIG.data_directory,
+        persist_directory=CONFIG.persist_directory,
+        chunk_size=CONFIG.chunk_size,
+        chunk_overlap=CONFIG.chunk_overlap,
+    )
+    if not len(os.listdir(CONFIG.persist_directory)) != 0:
+        prepare_vectordb_instance.prepare_and_save_vectordb()
+    else:
+        print(f"VectorDB already exists in {CONFIG.persist_directory}")
+    return None
+
+
+if __name__ == "__main__":
+    upload_data_manually()

src/utils/chatbot.py

@@ -0,0 +1,171 @@
+import gradio as gr
+import time
+import os
+from langchain.vectorstores import Chroma
+from typing import List, Tuple
+import re
+import ast
+import html
+from utils.load_config import LoadConfig
+from langchain.embeddings import HuggingFaceEmbeddings
+import requests
+import torch
+FLASK_APP_ENDPOINT = "http://127.0.0.1:8888/generate_text"
+
+APPCFG = LoadConfig()
+URL = ""
+hyperlink = f"[RAG]({URL})"
+
+
+class ChatBot:
+    """
+    Class representing a chatbot with document retrieval and response generation capabilities.
+
+    This class provides static methods for responding to user queries, handling feedback, and
+    cleaning references from retrieved documents.
+    """
+    @staticmethod
+    def respond(chatbot: List,
+                message: str,
+                data_type: str = "Preprocessed doc",
+                temperature: float = 0.1,
+                top_k: int = 10,
+                top_p: float = 0.1) -> Tuple:
+        """
+        Generate a response to a user query using document retrieval and language model completion.
+
+        Parameters:
+            chatbot (List): List representing the chatbot's conversation history.
+            message (str): The user's query.
+            data_type (str): Type of data used for document retrieval ("Preprocessed doc" or "Upload doc: Process for RAG").
+            temperature (float): Temperature parameter for language model completion.
+
+        Returns:
+            Tuple: A tuple containing an empty string, the updated chat history, and references from retrieved documents.
+        """
+
+        # Retrieve embedding function from code env resources
+        # emb_model = "sentence-transformers/all-MiniLM-L6-v2"
+        embedding_function = HuggingFaceEmbeddings(
+            model_name="NeuML/pubmedbert-base-embeddings",
+            # cache_folder=os.getenv('SENTENCE_TRANSFORMERS_HOME')
+        )
+        if data_type == "Preprocessed doc":
+            # directories
+            if os.path.exists(APPCFG.persist_directory):
+                vectordb = Chroma(persist_directory=APPCFG.persist_directory,
+                                  embedding_function=embedding_function)
+            else:
+                chatbot.append(
+                    (message, f"VectorDB does not exist. Please first execute the 'upload_data_manually.py' module. For further information please visit {hyperlink}."))
+                return "", chatbot, None
+
+        elif data_type == "Upload doc: Process for RAG":
+            if os.path.exists(APPCFG.custom_persist_directory):
+                vectordb = Chroma(persist_directory=APPCFG.custom_persist_directory,
+                                  embedding_function=embedding_function)
+            else:
+                chatbot.append(
+                    (message, f"No file was uploaded. Please first upload your files using the 'upload' button."))
+                return "", chatbot, None
+
+        docs = vectordb.similarity_search(message, k=APPCFG.k)
+        question = "# Prompt that you have to answer:\n" + message
+        retrieved_content, markdown_documents = ChatBot.clean_references(docs)
+        # Memory: previous two Q&A pairs
+        chat_history = f"Chat history:\n {str(chatbot[-APPCFG.number_of_q_a_pairs:])}\n\n"
+        if APPCFG.add_history:
+            prompt_wrapper = f"{APPCFG.llm_system_role_with_history}\n\n{chat_history}\n\n{retrieved_content}{question}"
+        else:
+            prompt_wrapper = f"{APPCFG.llm_system_role_without_history}\n\n{question}\n\n{retrieved_content}"
+
+        print("========================")
+        print(prompt_wrapper)
+        print("========================")
+        messages = [
+            {"role": "user", "content": prompt_wrapper},
+        ]
+        data = {
+            "prompt": messages,
+            "max_new_tokens": APPCFG.max_new_tokens,
+            "do_sample": APPCFG.do_sample,
+            "temperature": temperature,
+            "top_k": top_k,
+            "top_p": top_p
+        }
+        response = requests.post(FLASK_APP_ENDPOINT, json=data)
+        # print(response.text)
+        response_json = response.json()
+
+        chatbot.append(
+            (message, response_json["response"]))
+        # Clean up GPU memory
+        del vectordb
+        del docs
+        torch.cuda.empty_cache()
+        return "", chatbot, markdown_documents
+
+    @staticmethod
+    def clean_references(documents: List) -> str:
+        """
+        Clean and format references from retrieved documents.
+
+        Parameters:
+            documents (List): List of retrieved documents.
+
+        Returns:
+            str: A string containing cleaned and formatted references.
+        """
+        server_url = "http://localhost:8000"
+        documents = [str(x)+"\n\n" for x in documents]
+        markdown_documents = ""
+        retrieved_content = ""
+        counter = 1
+        for doc in documents:
+            # Extract content and metadata
+            content, metadata = re.match(
+                r"page_content=(.*?)( metadata=\{.*\})", doc).groups()
+            metadata = metadata.split('=', 1)[1]
+            metadata_dict = ast.literal_eval(metadata)
+
+            # Decode newlines and other escape sequences
+            content = bytes(content, "utf-8").decode("unicode_escape")
+
+            # Replace escaped newlines with actual newlines
+            content = re.sub(r'\\n', '\n', content)
+            content = re.sub(r'\s*<EOS>\s*<pad>\s*', ' ', content)
+            content = re.sub(r'\s+', ' ', content).strip()
+
+
+            # Decode HTML entities
+            content = html.unescape(content)
+
+            # Replace incorrect unicode characters with correct ones
+            #content = content.encode('utf-8').decode('utf-8', 'ignore')
+            # Use UTF-8 encoding instead of latin-1 to avoid encoding issues
+            content = content.encode('utf-8', 'ignore').decode('utf-8', 'ignore')
+            
+
+
+            # Remove or replace special characters and mathematical symbols
+            # This step may need to be customized based on the specific symbols in your documents
+            content = re.sub(r'–', '-', content)
+            content = re.sub(r'∈', '∈', content)
+            content = re.sub(r'×', '×', content)
+            content = re.sub(r'fi', 'fi', content)
+            content = re.sub(r'∈', '∈', content)
+            content = re.sub(r'·', '·', content)
+            content = re.sub(r'fl', 'fl', content)
+
+            pdf_url = f"{server_url}/{os.path.basename(metadata_dict['source'])}"
+            retrieved_content += f"# Content {counter}:\n" + \
+                content + "\n\n"
+
+            # Append cleaned content to the markdown string with two newlines between documents
+            markdown_documents += f"# Retrieved content {counter}:\n" + content + "\n\n" + \
+                f"Source: {os.path.basename(metadata_dict['source'])}" + " | " +\
+                f"Page number: {str(metadata_dict['page'])}" + " | " +\
+                f"[View PDF]({pdf_url})" "\n\n"
+            counter += 1
+
+        return retrieved_content, markdown_documents

src/utils/load_config.py

@@ -0,0 +1,89 @@
+
+import os
+from dotenv import load_dotenv
+import yaml
+from pyprojroot import here
+import shutil
+
+load_dotenv()
+
+
+class LoadConfig:
+    """
+    A class for loading configuration settings and managing directories.
+
+    This class loads various configuration settings from the 'app_config.yml' file,
+    including language model (LLM) configurations, retrieval configurations, summarizer
+    configurations, and memory configurations. It also sets up OpenAI API credentials
+    and performs directory-related operations such as creating and removing directories.
+    """
+
+    def __init__(self) -> None:
+        with open(here("configs/app_config.yml")) as cfg:
+            app_config = yaml.load(cfg, Loader=yaml.FullLoader)
+
+        # LLM configs
+        self.llm_engine = app_config["llm_config"]["engine"]
+        self.llm_system_role_with_history = app_config["llm_config"]["llm_system_role_with_history"]
+        self.llm_system_role_without_history = app_config[
+            "llm_config"]["llm_system_role_without_history"]
+        self.persist_directory = str(here(
+            app_config["directories"]["persist_directory"]))  # needs to be strin for summation in chromadb backend: self._settings.require("persist_directory") + "/chroma.sqlite3"
+        self.custom_persist_directory = str(here(
+            app_config["directories"]["custom_persist_directory"]))
+        self.gemma_token = os.getenv("GEMMA_TOKEN")
+        self.device = app_config["llm_config"]["device"]
+        # Retrieval configs
+        self.data_directory = app_config["directories"]["data_directory"]
+        self.k = app_config["retrieval_config"]["k"]
+        self.chunk_size = int(app_config["splitter_config"]["chunk_size"])
+        self.chunk_overlap = int(
+            app_config["splitter_config"]["chunk_overlap"])
+        self.temperature = float(app_config["llm_config"]["temperature"])
+        self.add_history = bool(app_config["llm_config"]["add_history"])
+        self.top_k = int(app_config["llm_config"]["top_k"])
+        self.top_p = float(app_config["llm_config"]["top_p"])
+        self.max_new_tokens = int(app_config["llm_config"]["max_new_tokens"])
+        self.do_sample = bool(app_config["llm_config"]["do_sample"])
+        self.embedding_model = app_config["llm_config"]["embedding_model"]
+
+        # Memory
+        self.number_of_q_a_pairs = int(
+            app_config["memory"]["number_of_q_a_pairs"])
+
+        # clean up the upload doc vectordb if it exists
+        self.create_directory(self.persist_directory)
+        self.remove_directory(self.custom_persist_directory)
+
+    def create_directory(self, directory_path: str):
+        """
+        Create a directory if it does not exist.
+
+        Parameters:
+            directory_path (str): The path of the directory to be created.
+        """
+        if not os.path.exists(directory_path):
+            os.makedirs(directory_path)
+
+    def remove_directory(self, directory_path: str):
+        """
+        Removes the specified directory.
+
+        Parameters:
+            directory_path (str): The path of the directory to be removed.
+
+        Raises:
+            OSError: If an error occurs during the directory removal process.
+
+        Returns:
+            None
+        """
+        if os.path.exists(directory_path):
+            try:
+                shutil.rmtree(directory_path)
+                print(
+                    f"The directory '{directory_path}' has been successfully removed.")
+            except OSError as e:
+                print(f"Error: {e}")
+        else:
+            print(f"The directory '{directory_path}' does not exist.")

src/utils/prepare_vectordb.py

@@ -0,0 +1,117 @@
+from langchain.vectorstores import Chroma
+from langchain.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+import os
+from typing import List
+from langchain.embeddings import HuggingFaceEmbeddings
+
+
+class PrepareVectorDB:
+    """
+    A class for preparing and saving a VectorDB using OpenAI embeddings.
+
+    This class facilitates the process of loading documents, chunking them, and creating a VectorDB
+    with OpenAI embeddings. It provides methods to prepare and save the VectorDB.
+
+    Parameters:
+        data_directory (str or List[str]): The directory or list of directories containing the documents.
+        persist_directory (str): The directory to save the VectorDB.
+        chunk_size (int): The size of the chunks for document processing.
+        chunk_overlap (int): The overlap between chunks.
+    """
+
+    def __init__(
+            self,
+            data_directory: str,
+            persist_directory: str,
+            chunk_size: int,
+            chunk_overlap: int
+    ) -> None:
+        """
+        Initialize the PrepareVectorDB instance.
+
+        Parameters:
+            data_directory (str or List[str]): The directory or list of directories containing the documents.
+            persist_directory (str): The directory to save the VectorDB.
+            chunk_size (int): The size of the chunks for document processing.
+            chunk_overlap (int): The overlap between chunks.
+
+        """
+
+        self.text_splitter = RecursiveCharacterTextSplitter(
+            chunk_size=chunk_size,
+            chunk_overlap=chunk_overlap,
+            separators=["\n\n", "\n", " ", ""]
+        )
+        """Other options: CharacterTextSplitter, TokenTextSplitter, etc."""
+        self.data_directory = data_directory
+        self.persist_directory = persist_directory
+        self.embedding_function = HuggingFaceEmbeddings(
+            model_name="NeuML/pubmedbert-base-embeddings",
+            # cache_folder=os.getenv('SENTENCE_TRANSFORMERS_HOME')
+        )
+
+    def __load_all_documents(self) -> List:
+        """
+        Load all documents from the specified directory or directories.
+
+        Returns:
+            List: A list of loaded documents.
+        """
+        doc_counter = 0
+        if isinstance(self.data_directory, list):
+            print("Loading the uploaded documents...")
+            docs = []
+            for doc_dir in self.data_directory:
+                docs.extend(PyPDFLoader(doc_dir).load())
+                doc_counter += 1
+            print("Number of loaded documents:", doc_counter)
+            print("Number of pages:", len(docs), "\n\n")
+        else:
+            print("Loading documents manually...")
+            document_list = os.listdir(self.data_directory)
+            docs = []
+            for doc_name in document_list:
+                docs.extend(PyPDFLoader(os.path.join(
+                    self.data_directory, doc_name)).load())
+                doc_counter += 1
+            print("Number of loaded documents:", doc_counter)
+            print("Number of pages:", len(docs), "\n\n")
+
+        return docs
+
+    def __chunk_documents(self, docs: List) -> List:
+        """
+        Chunk the loaded documents using the specified text splitter.
+
+        Parameters:
+            docs (List): The list of loaded documents.
+
+        Returns:
+            List: A list of chunked documents.
+
+        """
+        print("Chunking documents...")
+        chunked_documents = self.text_splitter.split_documents(docs)
+        print("Number of chunks:", len(chunked_documents), "\n\n")
+        return chunked_documents
+
+    def prepare_and_save_vectordb(self):
+        """
+        Load, chunk, and create a VectorDB with OpenAI embeddings, and save it.
+
+        Returns:
+            Chroma: The created VectorDB.
+        """
+        docs = self.__load_all_documents()
+        chunked_documents = self.__chunk_documents(docs)
+        print("Preparing vectordb...")
+        vectordb = Chroma.from_documents(
+            documents=chunked_documents,
+            embedding=self.embedding_function,
+            persist_directory=self.persist_directory
+        )
+        print("VectorDB is created and saved.")
+        print("Number of vectors in vectordb:",
+              vectordb._collection.count(), "\n\n")
+        return vectordb

src/utils/ui_settings.py

@@ -0,0 +1,35 @@
+import gradio as gr
+
+
+class UISettings:
+    """
+    Utility class for managing UI settings.
+
+    This class provides static methods for toggling UI components, such as a sidebar.
+    """
+    @staticmethod
+    def toggle_sidebar(state):
+        """
+        Toggle the visibility state of a UI component.
+
+        Parameters:
+            state: The current state of the UI component.
+
+        Returns:
+            Tuple: A tuple containing the updated UI component state and the new state.
+        """
+        state = not state
+        return gr.update(visible=state), state
+
+    @staticmethod
+    def feedback(data: gr.LikeData):
+        """
+        Process user feedback on the generated response.
+
+        Parameters:
+            data (gr.LikeData): Gradio LikeData object containing user feedback.
+        """
+        if data.liked:
+            print("You upvoted this response: " + data.value)
+        else:
+            print("You downvoted this response: " + data.value)

src/utils/upload_file.py

@@ -0,0 +1,39 @@
+from utils.prepare_vectordb import PrepareVectorDB
+from typing import List, Tuple
+from utils.load_config import LoadConfig
+
+APPCFG = LoadConfig()
+
+
+class UploadFile:
+    """
+    Utility class for handling file uploads and processing.
+
+    This class provides static methods for checking directories and processing uploaded files
+    to prepare a VectorDB.
+    """
+
+    @staticmethod
+    def process_uploaded_files(files_dir: List, chatbot: List, rag_with_dropdown: str) -> Tuple:
+        """
+        Process uploaded files to prepare a VectorDB.
+
+        Parameters:
+            files_dir (List): List of paths to the uploaded files.
+            chatbot: An instance of the chatbot for communication.
+
+        Returns:
+            Tuple: A tuple containing an empty string and the updated chatbot instance.
+        """
+        if rag_with_dropdown == "Upload doc: Process for RAG":
+            prepare_vectordb_instance = PrepareVectorDB(data_directory=files_dir,
+                                                        persist_directory=APPCFG.custom_persist_directory,
+                                                        chunk_size=APPCFG.chunk_size,
+                                                        chunk_overlap=APPCFG.chunk_overlap)
+            prepare_vectordb_instance.prepare_and_save_vectordb()
+            chatbot.append(
+                (" ", "Uploaded files are ready. Please ask your question"))
+        else:
+            chatbot.append(
+                (" ", "If you would like to upload a PDF, please select your desired action in 'rag_with' dropdown."))
+        return "", chatbot