Upload layer_9/config.yaml with huggingface_hub
Browse files- layer_9/config.yaml +18 -0
layer_9/config.yaml
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
dataset:
|
| 2 |
+
max_length: 128
|
| 3 |
+
name: monology/pile-uncopyrighted
|
| 4 |
+
split: train
|
| 5 |
+
model:
|
| 6 |
+
device: cuda
|
| 7 |
+
name: EleutherAI/pythia-410m
|
| 8 |
+
transcoding:
|
| 9 |
+
batch_size: 512
|
| 10 |
+
bias: true
|
| 11 |
+
debug: false
|
| 12 |
+
hidden_multiplier: 4
|
| 13 |
+
layer_idx: 9
|
| 14 |
+
learning_rate: 0.02
|
| 15 |
+
model_type: Bilinear
|
| 16 |
+
n_batches: 20
|
| 17 |
+
n_batches_full: 3000
|
| 18 |
+
optimizer_type: Muon
|