Skip to content

Commit 3e7be3d

Browse files
KawrakowKawrakow
andauthored
Correct L4 rms_norm (ikawrakow#324)
Co-authored-by: Iwan Kawrakow <iwan.kawrakow@gmail.com>
1 parent 474435f commit 3e7be3d

File tree

1 file changed

+2
-2
lines changed

1 file changed

+2
-2
lines changed

src/llama.cpp

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9913,8 +9913,8 @@ struct llm_build_context {
99139913

99149914
if (model.arch == LLM_ARCH_LLAMA4 && use_rope && hparams.use_kq_norm) {
99159915
// Llama4TextL2Norm
9916-
Qcur = ggml_rms_norm(ctx0, Qcur, 1e-6);
9917-
Kcur = ggml_rms_norm(ctx0, Kcur, 1e-6);
9916+
Qcur = ggml_rms_norm(ctx0, Qcur, hparams.f_norm_rms_eps);
9917+
Kcur = ggml_rms_norm(ctx0, Kcur, hparams.f_norm_rms_eps);
99189918
cb(Qcur, "Qcur_normed", il);
99199919
cb(Kcur, "Kcur_normed", il);
99209920
}

0 commit comments

Comments
 (0)