Black formatting
Browse files- src/axolotl/utils/data.py +5 -2
- src/axolotl/utils/validation.py +3 -1
- tests/test_validation.py +0 -1
src/axolotl/utils/data.py
CHANGED
|
@@ -107,7 +107,9 @@ def load_tokenized_prepared_datasets(
|
|
| 107 |
use_auth_token=use_auth_token,
|
| 108 |
)
|
| 109 |
else:
|
| 110 |
-
ds: Dataset = load_dataset(
|
|
|
|
|
|
|
| 111 |
else:
|
| 112 |
fp = hf_hub_download(
|
| 113 |
repo_id=d.path, repo_type="dataset", filename=d.data_files
|
|
@@ -293,7 +295,8 @@ def load_prepare_datasets(
|
|
| 293 |
f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
|
| 294 |
)
|
| 295 |
dataset = load_dataset(
|
| 296 |
-
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
|
|
|
| 297 |
)
|
| 298 |
dataset = dataset["train"]
|
| 299 |
except Exception: # pylint: disable=broad-except
|
|
|
|
| 107 |
use_auth_token=use_auth_token,
|
| 108 |
)
|
| 109 |
else:
|
| 110 |
+
ds: Dataset = load_dataset(
|
| 111 |
+
d.path, streaming=False, use_auth_token=use_auth_token
|
| 112 |
+
)
|
| 113 |
else:
|
| 114 |
fp = hf_hub_download(
|
| 115 |
repo_id=d.path, repo_type="dataset", filename=d.data_files
|
|
|
|
| 295 |
f"Checking for packed prepared dataset from hub... {cfg.push_dataset_to_hub}/{ds_hash}"
|
| 296 |
)
|
| 297 |
dataset = load_dataset(
|
| 298 |
+
f"{cfg.push_dataset_to_hub}/{ds_hash}",
|
| 299 |
+
use_auth_token=use_auth_token,
|
| 300 |
)
|
| 301 |
dataset = dataset["train"]
|
| 302 |
except Exception: # pylint: disable=broad-except
|
src/axolotl/utils/validation.py
CHANGED
|
@@ -40,7 +40,9 @@ def validate_config(cfg):
|
|
| 40 |
)
|
| 41 |
|
| 42 |
if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
|
| 43 |
-
raise ValueError(
|
|
|
|
|
|
|
| 44 |
|
| 45 |
# TODO
|
| 46 |
# MPT 7b
|
|
|
|
| 40 |
)
|
| 41 |
|
| 42 |
if cfg.push_dataset_to_hub and cfg.hf_use_auth_token is not True:
|
| 43 |
+
raise ValueError(
|
| 44 |
+
"Require cfg.hf_use_auth_token to be True for push_dataset_to_hub"
|
| 45 |
+
)
|
| 46 |
|
| 47 |
# TODO
|
| 48 |
# MPT 7b
|
tests/test_validation.py
CHANGED
|
@@ -117,4 +117,3 @@ class ValidationTest(unittest.TestCase):
|
|
| 117 |
}
|
| 118 |
)
|
| 119 |
validate_config(cfg)
|
| 120 |
-
|
|
|
|
| 117 |
}
|
| 118 |
)
|
| 119 |
validate_config(cfg)
|
|
|