We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent 16b73d4 commit 333eeedCopy full SHA for 333eeed
src/llama-graph.cpp
@@ -347,7 +347,7 @@ void llm_graph_input_attn_no_cache::set_input(const llama_ubatch * ubatch) {
347
const int64_t pos_diff = pos_j - pos_i;
348
349
if (hparams.use_alibi &&
350
- (pos_diff >= -half_n_swa && pos_diff <= half_n_swa)) {
+ (hparams.n_swa == 0 || (pos_diff >= -half_n_swa && pos_diff <= half_n_swa))) {
351
f = -std::abs(ubatch->pos[ti] - ubatch->pos[tj]);
352
} else {
353
f = 0.0f;
0 commit comments