change auth token setting back
Browse files
src/axolotl/utils/data.py
CHANGED
|
@@ -60,12 +60,11 @@ def load_tokenized_prepared_datasets(
|
|
| 60 |
else Path(default_dataset_prepared_path) / ds_hash
|
| 61 |
)
|
| 62 |
dataset = None
|
| 63 |
-
use_auth_token = False
|
| 64 |
try:
|
| 65 |
if cfg.push_dataset_to_hub:
|
| 66 |
use_auth_token = True
|
| 67 |
dataset = load_dataset(
|
| 68 |
-
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=
|
| 69 |
)
|
| 70 |
dataset = dataset["train"]
|
| 71 |
except:
|
|
@@ -85,7 +84,7 @@ def load_tokenized_prepared_datasets(
|
|
| 85 |
ds = None
|
| 86 |
ds_from_hub = False
|
| 87 |
try:
|
| 88 |
-
load_dataset(d.path, streaming=True, use_auth_token=
|
| 89 |
ds_from_hub = True
|
| 90 |
except FileNotFoundError:
|
| 91 |
pass
|
|
@@ -104,7 +103,7 @@ def load_tokenized_prepared_datasets(
|
|
| 104 |
use_auth_token=use_auth_token,
|
| 105 |
)
|
| 106 |
else:
|
| 107 |
-
ds = load_dataset(d.path, streaming=False, use_auth_token=
|
| 108 |
else:
|
| 109 |
fp = hf_hub_download(
|
| 110 |
repo_id=d.path, repo_type="dataset", filename=d.data_files
|
|
|
|
| 60 |
else Path(default_dataset_prepared_path) / ds_hash
|
| 61 |
)
|
| 62 |
dataset = None
|
|
|
|
| 63 |
try:
|
| 64 |
if cfg.push_dataset_to_hub:
|
| 65 |
use_auth_token = True
|
| 66 |
dataset = load_dataset(
|
| 67 |
+
f"{cfg.push_dataset_to_hub}/{ds_hash}", use_auth_token=True
|
| 68 |
)
|
| 69 |
dataset = dataset["train"]
|
| 70 |
except:
|
|
|
|
| 84 |
ds = None
|
| 85 |
ds_from_hub = False
|
| 86 |
try:
|
| 87 |
+
load_dataset(d.path, streaming=True, use_auth_token=True)
|
| 88 |
ds_from_hub = True
|
| 89 |
except FileNotFoundError:
|
| 90 |
pass
|
|
|
|
| 103 |
use_auth_token=use_auth_token,
|
| 104 |
)
|
| 105 |
else:
|
| 106 |
+
ds = load_dataset(d.path, streaming=False, use_auth_token=True)
|
| 107 |
else:
|
| 108 |
fp = hf_hub_download(
|
| 109 |
repo_id=d.path, repo_type="dataset", filename=d.data_files
|