scripts/run.py

import sys
sys.path.append(sys.path[0].replace('scripts', ''))
import os

from scripts.prompt_engine import PromptSearchEngine
import argparse
from config.data_paths import PROCESSED_DATA_PATH
from scripts.preprocess import clean_corpus
from scripts.utils import load_config

config = load_config()
clean_corpus()
search_engine = PromptSearchEngine(corpus=os.path.join(PROCESSED_DATA_PATH, 'prompt_corpus_clean.parquet'),
                                    model_name=config['search_engine']['model_name'],
                                    use_index=config['search_engine']['use_index'])