Hugging Face's logo

assafbk
/
decimamba-130m-squad-doc-ret

Text Generation
Transformers
PyTorch
mamba
long context
Model card Files Community
1
jon12398 commited on
Commit
e8d7c5e
·
1 Parent(s): c4e3e56

updated model

Browse files
Files changed (4) hide show
  1. fintune_ssm_config.json +1 -1
  2. pytorch_model.bin +1 -1
  3. special_tokens_map.json +3 -21
  4. tokenizer_config.json +0 -202
fintune_ssm_config.json CHANGED
@@ -1 +1 @@
1
- {"output_dir": "/data2/assaf/mamba/outputs/models", "cache_dir": "/data2/hf_cache", "activate_logging": true, "wandb_dir": "/data2/assaf/wandb/mamba", "run_name_addon": "retrieve doc id, train full dataset, append query to each noise doc + query/doc flag, 11 train noise docs, inj policy=random_loc, batch_size=64, deci_layer=12, Lbase=2000", "record_debug_params": false, "recover_step": null, "eval_mode": false, "mamba_arch": "deci", "model_type": "mamba-130m", "use_finetuned_model": false, "load_cp": null, "clip_grad": true, "clip_grad_max_norm": 1, "seed": 123, "lr_sched_type": "const", "sampling_temperature": 1.2, "save_steps": 50, "eval_steps": 10, "grad_flow_steps": 100, "max_step": 200000, "epochs": 10, "model_device": "cuda:0", "dataset": "squad_retrieve", "train_set_size": 6144, "eval_set_size": 20, "eval_samples_to_log": 10, "log_eval_predictions_steps": 10, "eval_max_len": 10, "max_train_input_len": 20000, "enable_eos_token": false, "scrolls_evaluator_path": "/data1/assaf/datasets/scrolls/evaluator/dataset_evaluator.py", "niah_train_set_size": 6144, "niah_context_len_train": 2000, "niah_needle_depths_eval": [0, 0.25, 0.5, 0.75, 1], "niah_context_lens_eval": [128000, 144000, 176000], "ppl_test_context_len_train": 2000, "ppl_test_pred_len": 100, "multidoc_num_noise_docs_train": 11, "multidoc_num_noise_docs_eval": [0, 5, 10, 20, 40, 80, 120, 160, 200], "multidoc_noise_injection_policy": "random_loc", "activate_decimation": true, "decimation_type": "max_p", "decimation_k": 6, "min_decimating_layer": 12, "max_decimating_layer": 20, "decimating_layers": [12], "decimation_min_seq_len": 20, "decimation_max_p_L_base": 2000, "decimation_freeze_encoder": false, "find_deci_layer": false, "lr": 0.0001, "weight_decay": 0.1, "grad_accum_steps": 64, "activate_profiling": false, "deci_num_chunks": 1}
 
1
+ {"output_dir": "./output", "cache_dir": "/data2/hf_cache", "activate_logging": true, "wandb_dir": "./wandb", "run_name_addon": "", "record_debug_params": false, "recover_step": null, "eval_mode": false, "mamba_arch": "deci", "model_type": "mamba-130m", "use_finetuned_model": false, "load_cp": null, "clip_grad": true, "clip_grad_max_norm": 1, "seed": 345, "lr_sched_type": "const", "save_steps": 200, "eval_steps": 20, "grad_flow_steps": 10, "max_step": 20000, "epochs": 10, "model_device": "cuda:3", "dataset": "squad_retrieve", "train_set_size": 6144, "eval_set_size": 20, "eval_samples_to_log": 10, "log_eval_predictions_steps": 10, "eval_max_len": 10, "max_train_input_len": 20000, "niah_train_set_size": 6144, "niah_context_len_train": 2000, "niah_needle_depths_eval": [0, 0.25, 0.5, 0.75, 1], "niah_context_lens_eval": [1000, 2000, 4000, 8000, 16000, 32000, 64000, 128000], "ppl_test_context_len_train": 2000, "ppl_test_num_windows_per_context_len_eval": 10, "ppl_test_context_lens_eval": [1000, 2000, 4000, 10000, 20000, 30000, 40000, 50000, 60000], "ppl_test_pred_len": 100, "deci_num_chunks": 1, "multidoc_num_noise_docs_train": 11, "multidoc_num_noise_docs_eval": [10, 20, 40, 60, 80, 100, 120], "multidoc_noise_injection_policy": "random_loc", "activate_decimation": true, "decimation_type": "max_p", "decimation_beta": 0.5, "decimating_layers": [12], "decimation_min_seq_len": 20, "decimation_max_p_L_base": 2000, "find_deci_layer": false, "lr": 0.0001, "weight_decay": 0.1, "grad_accum_steps": 64}
pytorch_model.bin CHANGED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:8286bb1b6effdef5061a9a53e1229f77f8b20a77ff29510ad65d697d2d96055c
3
  size 671093846
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:0056818afe047c6fe1818bfc8794f5f36cd449dd403f7e7a8bd32b7ad27fa03e
3
  size 671093846
special_tokens_map.json CHANGED
@@ -1,23 +1,5 @@
1
  {
2
- "bos_token": {
3
- "content": "<|endoftext|>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "<|endoftext|>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<|endoftext|>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
  }
 
1
  {
2
+ "bos_token": "<|endoftext|>",
3
+ "eos_token": "<|endoftext|>",
4
+ "unk_token": "<|endoftext|>"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5
  }
tokenizer_config.json CHANGED
@@ -1,207 +1,5 @@
1
  {
2
  "add_prefix_space": false,
3
- "added_tokens_decoder": {
4
- "0": {
5
- "content": "<|endoftext|>",
6
- "lstrip": false,
7
- "normalized": false,
8
- "rstrip": false,
9
- "single_word": false,
10
- "special": true
11
- },
12
- "1": {
13
- "content": "<|padding|>",
14
- "lstrip": false,
15
- "normalized": false,
16
- "rstrip": false,
17
- "single_word": false,
18
- "special": true
19
- },
20
- "50254": {
21
- "content": " ",
22
- "lstrip": false,
23
- "normalized": true,
24
- "rstrip": false,
25
- "single_word": false,
26
- "special": false
27
- },
28
- "50255": {
29
- "content": " ",
30
- "lstrip": false,
31
- "normalized": true,
32
- "rstrip": false,
33
- "single_word": false,
34
- "special": false
35
- },
36
- "50256": {
37
- "content": " ",
38
- "lstrip": false,
39
- "normalized": true,
40
- "rstrip": false,
41
- "single_word": false,
42
- "special": false
43
- },
44
- "50257": {
45
- "content": " ",
46
- "lstrip": false,
47
- "normalized": true,
48
- "rstrip": false,
49
- "single_word": false,
50
- "special": false
51
- },
52
- "50258": {
53
- "content": " ",
54
- "lstrip": false,
55
- "normalized": true,
56
- "rstrip": false,
57
- "single_word": false,
58
- "special": false
59
- },
60
- "50259": {
61
- "content": " ",
62
- "lstrip": false,
63
- "normalized": true,
64
- "rstrip": false,
65
- "single_word": false,
66
- "special": false
67
- },
68
- "50260": {
69
- "content": " ",
70
- "lstrip": false,
71
- "normalized": true,
72
- "rstrip": false,
73
- "single_word": false,
74
- "special": false
75
- },
76
- "50261": {
77
- "content": " ",
78
- "lstrip": false,
79
- "normalized": true,
80
- "rstrip": false,
81
- "single_word": false,
82
- "special": false
83
- },
84
- "50262": {
85
- "content": " ",
86
- "lstrip": false,
87
- "normalized": true,
88
- "rstrip": false,
89
- "single_word": false,
90
- "special": false
91
- },
92
- "50263": {
93
- "content": " ",
94
- "lstrip": false,
95
- "normalized": true,
96
- "rstrip": false,
97
- "single_word": false,
98
- "special": false
99
- },
100
- "50264": {
101
- "content": " ",
102
- "lstrip": false,
103
- "normalized": true,
104
- "rstrip": false,
105
- "single_word": false,
106
- "special": false
107
- },
108
- "50265": {
109
- "content": " ",
110
- "lstrip": false,
111
- "normalized": true,
112
- "rstrip": false,
113
- "single_word": false,
114
- "special": false
115
- },
116
- "50266": {
117
- "content": " ",
118
- "lstrip": false,
119
- "normalized": true,
120
- "rstrip": false,
121
- "single_word": false,
122
- "special": false
123
- },
124
- "50267": {
125
- "content": " ",
126
- "lstrip": false,
127
- "normalized": true,
128
- "rstrip": false,
129
- "single_word": false,
130
- "special": false
131
- },
132
- "50268": {
133
- "content": " ",
134
- "lstrip": false,
135
- "normalized": true,
136
- "rstrip": false,
137
- "single_word": false,
138
- "special": false
139
- },
140
- "50269": {
141
- "content": " ",
142
- "lstrip": false,
143
- "normalized": true,
144
- "rstrip": false,
145
- "single_word": false,
146
- "special": false
147
- },
148
- "50270": {
149
- "content": " ",
150
- "lstrip": false,
151
- "normalized": true,
152
- "rstrip": false,
153
- "single_word": false,
154
- "special": false
155
- },
156
- "50271": {
157
- "content": " ",
158
- "lstrip": false,
159
- "normalized": true,
160
- "rstrip": false,
161
- "single_word": false,
162
- "special": false
163
- },
164
- "50272": {
165
- "content": " ",
166
- "lstrip": false,
167
- "normalized": true,
168
- "rstrip": false,
169
- "single_word": false,
170
- "special": false
171
- },
172
- "50273": {
173
- "content": " ",
174
- "lstrip": false,
175
- "normalized": true,
176
- "rstrip": false,
177
- "single_word": false,
178
- "special": false
179
- },
180
- "50274": {
181
- "content": " ",
182
- "lstrip": false,
183
- "normalized": true,
184
- "rstrip": false,
185
- "single_word": false,
186
- "special": false
187
- },
188
- "50275": {
189
- "content": " ",
190
- "lstrip": false,
191
- "normalized": true,
192
- "rstrip": false,
193
- "single_word": false,
194
- "special": false
195
- },
196
- "50276": {
197
- "content": " ",
198
- "lstrip": false,
199
- "normalized": true,
200
- "rstrip": false,
201
- "single_word": false,
202
- "special": false
203
- }
204
- },
205
  "bos_token": "<|endoftext|>",
206
  "clean_up_tokenization_spaces": true,
207
  "eos_token": "<|endoftext|>",
 
1
  {
2
  "add_prefix_space": false,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  "bos_token": "<|endoftext|>",
4
  "clean_up_tokenization_spaces": true,
5
  "eos_token": "<|endoftext|>",