Update config.json
Browse filesfrom vLLM logic:
per block intermediate size = int(2 * ffn_mult * n_embd / 3)
2 * 5.25 * 8192 / 2 = 43008
ceiling to nearest 512: n + k - (n % k)
43008 + 512 - (43008 % 512)
= 43520
- config.json +1 -1
config.json
CHANGED
@@ -1455,7 +1455,7 @@
|
|
1455 |
"hidden_act": "silu",
|
1456 |
"hidden_size": 8192,
|
1457 |
"initializer_range": 0.02,
|
1458 |
-
"intermediate_size":
|
1459 |
"max_position_embeddings": 131072,
|
1460 |
"mlp_bias": false,
|
1461 |
"model_type": "nemotron-nas",
|
|
|
1455 |
"hidden_act": "silu",
|
1456 |
"hidden_size": 8192,
|
1457 |
"initializer_range": 0.02,
|
1458 |
+
"intermediate_size": 43520,
|
1459 |
"max_position_embeddings": 131072,
|
1460 |
"mlp_bias": false,
|
1461 |
"model_type": "nemotron-nas",
|