Upload scripts/preprocess.py with huggingface_hub

scripts/preprocess.py

@@ -4,7 +4,7 @@ sys.path.append(sys.path[0].replace('scripts', ''))
 from urllib.request import urlretrieve
 import pandas as pd
 
-from config.data_paths import RAW_DATA_PATH, PROCESSED_DATA_PATH
+from config.data_paths import PROCESSED_DATA_PATH
 import re
 
 from scripts.utils import load_config
@@ -28,6 +28,7 @@ def clean_corpus():
     Utility function to clean and preprocess the prompt corpus.
     """
     if not os.path.isfile(os.path.join(PROCESSED_DATA_PATH, 'prompt_corpus_clean.parquet')): # to speed up the process
+        os.makedirs(PROCESSED_DATA_PATH, exist_ok=True)
         df = pd.read_parquet(PROMPTS_URL).sample(10000, random_state=123)
         assert 'prompt' in df.columns, "Parquet file must contain a 'prompt' column."
         df = df[df['prompt'].notna()][['prompt']]     # drop missing rows