@@ -3305,32 +3305,13 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
3305
3305
3306
3306
// out_proj
3307
3307
layer.ssm_out = create_tensor(tn(LLM_TENSOR_SSM_OUT, "weight", i), {d_inner, n_embd}, 0);
3308
-
3309
- layer.wq = nullptr;
3310
- layer.wk = nullptr;
3311
- layer.wv = nullptr;
3312
- layer.wo = nullptr;
3313
-
3314
3308
} else {
3315
3309
// Attention layers
3316
3310
3317
3311
layer.wq = create_tensor(tn(LLM_TENSOR_ATTN_Q, "weight", i), {n_embd, n_embd}, 0);
3318
3312
layer.wk = create_tensor(tn(LLM_TENSOR_ATTN_K, "weight", i), {n_embd, n_embd_gqa}, 0);
3319
3313
layer.wv = create_tensor(tn(LLM_TENSOR_ATTN_V, "weight", i), {n_embd, n_embd_gqa}, 0);
3320
3314
layer.wo = create_tensor(tn(LLM_TENSOR_ATTN_OUT, "weight", i), {n_embd, n_embd}, 0);
3321
-
3322
- layer.ssm_in = nullptr;
3323
- layer.ssm_conv1d = nullptr;
3324
- layer.ssm_conv1d_b = nullptr;
3325
- layer.ssm_x = nullptr;
3326
- layer.ssm_dt_norm = nullptr;
3327
- layer.ssm_dt = nullptr;
3328
- layer.ssm_dt_b = nullptr;
3329
- layer.ssm_b_norm = nullptr;
3330
- layer.ssm_c_norm = nullptr;
3331
- layer.ssm_a = nullptr;
3332
- layer.ssm_d = nullptr;
3333
- layer.ssm_out = nullptr;
3334
3315
}
3335
3316
3336
3317
layer.ffn_norm = create_tensor(tn(LLM_TENSOR_FFN_NORM, "weight", i), {n_embd}, 0);
@@ -3342,19 +3323,11 @@ bool llama_model::load_tensors(llama_model_loader & ml) {
3342
3323
layer.ffn_gate_exps = create_tensor(tn(LLM_TENSOR_FFN_GATE_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0);
3343
3324
layer.ffn_down_exps = create_tensor(tn(LLM_TENSOR_FFN_DOWN_EXPS, "weight", i), {n_ff, n_embd, n_expert}, 0);
3344
3325
layer.ffn_up_exps = create_tensor(tn(LLM_TENSOR_FFN_UP_EXPS, "weight", i), {n_embd, n_ff, n_expert}, 0);
3345
-
3346
- layer.ffn_gate = nullptr;
3347
- layer.ffn_down = nullptr;
3348
- layer.ffn_up = nullptr;
3349
3326
} else {
3350
3327
// FFN (no MoE)
3351
3328
layer.ffn_gate = create_tensor(tn(LLM_TENSOR_FFN_GATE, "weight", i), {n_embd, n_ff}, 0);
3352
3329
layer.ffn_down = create_tensor(tn(LLM_TENSOR_FFN_DOWN, "weight", i), {n_ff, n_embd}, 0);
3353
3330
layer.ffn_up = create_tensor(tn(LLM_TENSOR_FFN_UP, "weight", i), {n_embd, n_ff}, 0);
3354
-
3355
- layer.ffn_gate_exps = nullptr;
3356
- layer.ffn_down_exps = nullptr;
3357
- layer.ffn_up_exps = nullptr;
3358
3331
}
3359
3332
}
3360
3333
} break;
0 commit comments