Spaces:

HF-test-lab
/

bulk_embeddings

Runtime error

nbroad commited on Jul 15, 2023

Commit

6f9442d

1 Parent(s): 7754e24

fix path to point to mounted drive

Files changed (1) hide show

utils.py CHANGED Viewed

@@ -132,8 +132,8 @@ def download_wikipedia(ds_name, ds_config):
         temp = ds2.select(range(start_idx, end_idx))
-        temp.to_parquet(f"wiki_chunks/chunk_{chunk_num}")
-        filenames.append(f"wiki_chunks/chunk_{chunk_num}")
     return load_dataset("parquet", data_files=filenames, split="train")

         temp = ds2.select(range(start_idx, end_idx))
+        temp.to_parquet(f"/data/wiki_chunks/chunk_{chunk_num}")
+        filenames.append(f"/data/wiki_chunks/chunk_{chunk_num}")
     return load_dataset("parquet", data_files=filenames, split="train")