Hugging Face's logo

CodeIsAbstract
/
HybridTimeScaleModel_modded

Safetensors
hybrid_fourier_lm
Model card Files
xet
 Community
CodeIsAbstract commited on
Commit
aa742bd
·
verified ·
1 Parent(s): 5cc7317

Training in progress, step 200, checkpoint

Browse files
Files changed (10) hide show
  1. last-checkpoint/config.json +33 -0
  2. last-checkpoint/generation_config.json +9 -0
  3. last-checkpoint/model.safetensors +3 -0
  4. last-checkpoint/optimizer.pt +3 -0
  5. last-checkpoint/rng_state.pth +3 -0
  6. last-checkpoint/scheduler.pt +3 -0
  7. last-checkpoint/tokenizer.json +0 -0
  8. last-checkpoint/tokenizer_config.json +17 -0
  9. last-checkpoint/trainer_state.json +346 -0
  10. last-checkpoint/training_args.bin +3 -0
last-checkpoint/config.json ADDED
@@ -0,0 +1,33 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "architectures": [
3
+ "HybridFourierLM"
4
+ ],
5
+ "bos_token_id": 1,
6
+ "dropout": 0.05,
7
+ "dtype": "bfloat16",
8
+ "eos_token_id": 2,
9
+ "latent_dim": 768,
10
+ "layer_types": [
11
+ "linear",
12
+ "linear",
13
+ "linear",
14
+ "softmax",
15
+ "linear",
16
+ "linear",
17
+ "linear",
18
+ "softmax",
19
+ "linear",
20
+ "linear",
21
+ "linear",
22
+ "softmax"
23
+ ],
24
+ "model_type": "hybrid_fourier_lm",
25
+ "num_layers": 12,
26
+ "num_modes": 64,
27
+ "pad_token_id": 2,
28
+ "tie_word_embeddings": true,
29
+ "time_scale": 128.0,
30
+ "transformers_version": "5.13.1",
31
+ "use_cache": false,
32
+ "vocab_size": 32768
33
+ }
last-checkpoint/generation_config.json ADDED
@@ -0,0 +1,9 @@
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "_from_model_config": true,
3
+ "bos_token_id": 1,
4
+ "eos_token_id": 2,
5
+ "output_attentions": false,
6
+ "output_hidden_states": false,
7
+ "pad_token_id": 2,
8
+ "transformers_version": "5.13.1"
9
+ }
last-checkpoint/model.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:4f3ff6417f337444900c9d0ee3418cae9650f976bbe8cad9be7e0ebb946b00b8
3
+ size 329100200
last-checkpoint/optimizer.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c37d5c1333fa20846532da34840740f4df14612f69d38d91951e2a09c63b6c97
3
+ size 188845393
last-checkpoint/rng_state.pth ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:f5656667be1478f776aa5cbc3870cc2b7f43ab2e34c2fe808cae05f89d24863b
3
+ size 14645
last-checkpoint/scheduler.pt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:17980325c36d869e063d721782688706e0e30902ea32568d55af1c62cc6c3e6f
3
+ size 1465
last-checkpoint/tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
last-checkpoint/tokenizer_config.json ADDED
@@ -0,0 +1,17 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": false,
3
+ "backend": "tokenizers",
4
+ "bos_token": "<s>",
5
+ "clean_up_tokenization_spaces": false,
6
+ "eos_token": "</s>",
7
+ "is_local": false,
8
+ "legacy": false,
9
+ "local_files_only": false,
10
+ "model_max_length": 1000000000,
11
+ "pad_token": "</s>",
12
+ "sp_model_kwargs": {},
13
+ "spaces_between_special_tokens": false,
14
+ "tokenizer_class": "TokenizersBackend",
15
+ "unk_token": "<unk>",
16
+ "use_default_system_prompt": false
17
+ }
last-checkpoint/trainer_state.json ADDED
@@ -0,0 +1,346 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "best_global_step": null,
3
+ "best_metric": null,
4
+ "best_model_checkpoint": null,
5
+ "epoch": 0.06666666666666667,
6
+ "eval_steps": 50,
7
+ "global_step": 200,
8
+ "is_hyper_param_search": false,
9
+ "is_local_process_zero": true,
10
+ "is_world_process_zero": true,
11
+ "log_history": [
12
+ {
13
+ "epoch": 0.0016666666666666668,
14
+ "grad_norm": 54.850189208984375,
15
+ "learning_rate": 0.0002,
16
+ "loss": 6.845032501220703,
17
+ "step": 5
18
+ },
19
+ {
20
+ "epoch": 0.0033333333333333335,
21
+ "grad_norm": 55.685916900634766,
22
+ "learning_rate": 0.00045000000000000004,
23
+ "loss": 6.620301055908203,
24
+ "step": 10
25
+ },
26
+ {
27
+ "epoch": 0.005,
28
+ "grad_norm": 7.728530406951904,
29
+ "learning_rate": 0.0005,
30
+ "loss": 7.703640747070312,
31
+ "step": 15
32
+ },
33
+ {
34
+ "epoch": 0.006666666666666667,
35
+ "grad_norm": 7.862086296081543,
36
+ "learning_rate": 0.0005,
37
+ "loss": 7.1933952331542965,
38
+ "step": 20
39
+ },
40
+ {
41
+ "epoch": 0.008333333333333333,
42
+ "grad_norm": 41934556.0,
43
+ "learning_rate": 0.0005,
44
+ "loss": 7.290315246582031,
45
+ "step": 25
46
+ },
47
+ {
48
+ "epoch": 0.01,
49
+ "grad_norm": 32783185920.0,
50
+ "learning_rate": 0.0005,
51
+ "loss": 10.341773986816406,
52
+ "step": 30
53
+ },
54
+ {
55
+ "epoch": 0.011666666666666667,
56
+ "grad_norm": 67137892352.0,
57
+ "learning_rate": 0.0005,
58
+ "loss": 16.722955322265626,
59
+ "step": 35
60
+ },
61
+ {
62
+ "epoch": 0.013333333333333334,
63
+ "grad_norm": 107766048.0,
64
+ "learning_rate": 0.0005,
65
+ "loss": 18.834597778320312,
66
+ "step": 40
67
+ },
68
+ {
69
+ "epoch": 0.015,
70
+ "grad_norm": 750511.1875,
71
+ "learning_rate": 0.0005,
72
+ "loss": 16.470211791992188,
73
+ "step": 45
74
+ },
75
+ {
76
+ "epoch": 0.016666666666666666,
77
+ "grad_norm": 1690175.0,
78
+ "learning_rate": 0.0005,
79
+ "loss": 17.498741149902344,
80
+ "step": 50
81
+ },
82
+ {
83
+ "epoch": 0.016666666666666666,
84
+ "eval_loss": 9.509819984436035,
85
+ "eval_runtime": 217.3232,
86
+ "eval_samples_per_second": 1.371,
87
+ "eval_steps_per_second": 0.686,
88
+ "step": 50
89
+ },
90
+ {
91
+ "epoch": 0.018333333333333333,
92
+ "grad_norm": 44012.49609375,
93
+ "learning_rate": 0.0005,
94
+ "loss": 19.190980529785158,
95
+ "step": 55
96
+ },
97
+ {
98
+ "epoch": 0.02,
99
+ "grad_norm": 1957.0230712890625,
100
+ "learning_rate": 0.0005,
101
+ "loss": 20.642401123046874,
102
+ "step": 60
103
+ },
104
+ {
105
+ "epoch": 0.021666666666666667,
106
+ "grad_norm": 238.84210205078125,
107
+ "learning_rate": 0.0005,
108
+ "loss": 21.76719970703125,
109
+ "step": 65
110
+ },
111
+ {
112
+ "epoch": 0.023333333333333334,
113
+ "grad_norm": 71.29987335205078,
114
+ "learning_rate": 0.0005,
115
+ "loss": 20.405929565429688,
116
+ "step": 70
117
+ },
118
+ {
119
+ "epoch": 0.025,
120
+ "grad_norm": 105.72884368896484,
121
+ "learning_rate": 0.0005,
122
+ "loss": 19.047821044921875,
123
+ "step": 75
124
+ },
125
+ {
126
+ "epoch": 0.02666666666666667,
127
+ "grad_norm": 7008.24560546875,
128
+ "learning_rate": 0.0005,
129
+ "loss": 18.463946533203124,
130
+ "step": 80
131
+ },
132
+ {
133
+ "epoch": 0.028333333333333332,
134
+ "grad_norm": 171495.171875,
135
+ "learning_rate": 0.0005,
136
+ "loss": 18.29894561767578,
137
+ "step": 85
138
+ },
139
+ {
140
+ "epoch": 0.03,
141
+ "grad_norm": 122140.40625,
142
+ "learning_rate": 0.0005,
143
+ "loss": 18.62984619140625,
144
+ "step": 90
145
+ },
146
+ {
147
+ "epoch": 0.03166666666666667,
148
+ "grad_norm": 8799.4404296875,
149
+ "learning_rate": 0.0005,
150
+ "loss": 18.06275177001953,
151
+ "step": 95
152
+ },
153
+ {
154
+ "epoch": 0.03333333333333333,
155
+ "grad_norm": 435225.90625,
156
+ "learning_rate": 0.0005,
157
+ "loss": 17.608197021484376,
158
+ "step": 100
159
+ },
160
+ {
161
+ "epoch": 0.03333333333333333,
162
+ "eval_loss": 8.762983322143555,
163
+ "eval_runtime": 218.6047,
164
+ "eval_samples_per_second": 1.363,
165
+ "eval_steps_per_second": 0.682,
166
+ "step": 100
167
+ },
168
+ {
169
+ "epoch": 0.035,
170
+ "grad_norm": 10736.8583984375,
171
+ "learning_rate": 0.0005,
172
+ "loss": 17.602194213867186,
173
+ "step": 105
174
+ },
175
+ {
176
+ "epoch": 0.03666666666666667,
177
+ "grad_norm": 2109406.5,
178
+ "learning_rate": 0.0005,
179
+ "loss": 18.206829833984376,
180
+ "step": 110
181
+ },
182
+ {
183
+ "epoch": 0.03833333333333333,
184
+ "grad_norm": 311519.9375,
185
+ "learning_rate": 0.0005,
186
+ "loss": 18.991905212402344,
187
+ "step": 115
188
+ },
189
+ {
190
+ "epoch": 0.04,
191
+ "grad_norm": 295.97235107421875,
192
+ "learning_rate": 0.0005,
193
+ "loss": 19.67296447753906,
194
+ "step": 120
195
+ },
196
+ {
197
+ "epoch": 0.041666666666666664,
198
+ "grad_norm": 31418684.0,
199
+ "learning_rate": 0.0005,
200
+ "loss": 20.369107055664063,
201
+ "step": 125
202
+ },
203
+ {
204
+ "epoch": 0.043333333333333335,
205
+ "grad_norm": 3918581.0,
206
+ "learning_rate": 0.0005,
207
+ "loss": 20.751130676269533,
208
+ "step": 130
209
+ },
210
+ {
211
+ "epoch": 0.045,
212
+ "grad_norm": 13102.0244140625,
213
+ "learning_rate": 0.0005,
214
+ "loss": 21.04936828613281,
215
+ "step": 135
216
+ },
217
+ {
218
+ "epoch": 0.04666666666666667,
219
+ "grad_norm": 16994096.0,
220
+ "learning_rate": 0.0005,
221
+ "loss": 21.621359252929686,
222
+ "step": 140
223
+ },
224
+ {
225
+ "epoch": 0.04833333333333333,
226
+ "grad_norm": 2272.0400390625,
227
+ "learning_rate": 0.0005,
228
+ "loss": 22.08568878173828,
229
+ "step": 145
230
+ },
231
+ {
232
+ "epoch": 0.05,
233
+ "grad_norm": 4981.4296875,
234
+ "learning_rate": 0.0005,
235
+ "loss": 22.322303771972656,
236
+ "step": 150
237
+ },
238
+ {
239
+ "epoch": 0.05,
240
+ "eval_loss": 11.168928146362305,
241
+ "eval_runtime": 217.379,
242
+ "eval_samples_per_second": 1.371,
243
+ "eval_steps_per_second": 0.685,
244
+ "step": 150
245
+ },
246
+ {
247
+ "epoch": 0.051666666666666666,
248
+ "grad_norm": 2926.708251953125,
249
+ "learning_rate": 0.0005,
250
+ "loss": 22.440846252441407,
251
+ "step": 155
252
+ },
253
+ {
254
+ "epoch": 0.05333333333333334,
255
+ "grad_norm": 37812.78515625,
256
+ "learning_rate": 0.0005,
257
+ "loss": 22.036326599121093,
258
+ "step": 160
259
+ },
260
+ {
261
+ "epoch": 0.055,
262
+ "grad_norm": 638.6507568359375,
263
+ "learning_rate": 0.0005,
264
+ "loss": 20.635609436035157,
265
+ "step": 165
266
+ },
267
+ {
268
+ "epoch": 0.056666666666666664,
269
+ "grad_norm": 45.453582763671875,
270
+ "learning_rate": 0.0005,
271
+ "loss": 19.9438720703125,
272
+ "step": 170
273
+ },
274
+ {
275
+ "epoch": 0.058333333333333334,
276
+ "grad_norm": 122177.8359375,
277
+ "learning_rate": 0.0005,
278
+ "loss": 19.473664855957033,
279
+ "step": 175
280
+ },
281
+ {
282
+ "epoch": 0.06,
283
+ "grad_norm": 1840.748779296875,
284
+ "learning_rate": 0.0005,
285
+ "loss": 19.456393432617187,
286
+ "step": 180
287
+ },
288
+ {
289
+ "epoch": 0.06166666666666667,
290
+ "grad_norm": 130342.7265625,
291
+ "learning_rate": 0.0005,
292
+ "loss": 19.97138366699219,
293
+ "step": 185
294
+ },
295
+ {
296
+ "epoch": 0.06333333333333334,
297
+ "grad_norm": 26071.556640625,
298
+ "learning_rate": 0.0005,
299
+ "loss": 20.022232055664062,
300
+ "step": 190
301
+ },
302
+ {
303
+ "epoch": 0.065,
304
+ "grad_norm": 6111.51513671875,
305
+ "learning_rate": 0.0005,
306
+ "loss": 20.719598388671876,
307
+ "step": 195
308
+ },
309
+ {
310
+ "epoch": 0.06666666666666667,
311
+ "grad_norm": 387584.5,
312
+ "learning_rate": 0.0005,
313
+ "loss": 21.63736572265625,
314
+ "step": 200
315
+ },
316
+ {
317
+ "epoch": 0.06666666666666667,
318
+ "eval_loss": 11.059845924377441,
319
+ "eval_runtime": 216.5693,
320
+ "eval_samples_per_second": 1.376,
321
+ "eval_steps_per_second": 0.688,
322
+ "step": 200
323
+ }
324
+ ],
325
+ "logging_steps": 5,
326
+ "max_steps": 3000,
327
+ "num_input_tokens_seen": 0,
328
+ "num_train_epochs": 9223372036854775807,
329
+ "save_steps": 200,
330
+ "stateful_callbacks": {
331
+ "TrainerControl": {
332
+ "args": {
333
+ "should_epoch_stop": false,
334
+ "should_evaluate": false,
335
+ "should_log": false,
336
+ "should_save": true,
337
+ "should_training_stop": false
338
+ },
339
+ "attributes": {}
340
+ }
341
+ },
342
+ "total_flos": 2276052487372800.0,
343
+ "train_batch_size": 2,
344
+ "trial_name": null,
345
+ "trial_params": null
346
+ }
last-checkpoint/training_args.bin ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:56d4930d2f2e2d96034931eacf623f5cc07c816a8f7e9190dfff6ba278e2fb60
3
+ size 5201