drop length column for issues with eval without packing (#1711)
Browse files
src/axolotl/core/trainer_builder.py
CHANGED
|
@@ -459,6 +459,8 @@ class AxolotlTrainer(Trainer):
|
|
| 459 |
self.data_collator = ( # pylint: disable=attribute-defined-outside-init
|
| 460 |
self.eval_data_collator
|
| 461 |
)
|
|
|
|
|
|
|
| 462 |
dataloader = super().get_eval_dataloader(eval_dataset)
|
| 463 |
self.data_collator = ( # pylint: disable=attribute-defined-outside-init
|
| 464 |
self.train_data_collator
|
|
|
|
| 459 |
self.data_collator = ( # pylint: disable=attribute-defined-outside-init
|
| 460 |
self.eval_data_collator
|
| 461 |
)
|
| 462 |
+
if eval_dataset:
|
| 463 |
+
eval_dataset = eval_dataset.remove_columns(["length"])
|
| 464 |
dataloader = super().get_eval_dataloader(eval_dataset)
|
| 465 |
self.data_collator = ( # pylint: disable=attribute-defined-outside-init
|
| 466 |
self.train_data_collator
|