Skip to content

Commit 3637576

Browse files
authored
server : disable speculative decoding for SWA models (#13970)
* server : use swa-full fo draft context ggml-ci * server : disable speculative decoding for SWA models
1 parent ea394d7 commit 3637576

File tree

1 file changed

+5
-3
lines changed

1 file changed

+5
-3
lines changed

tools/server/server.cpp

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2016,6 +2016,11 @@ struct server_context {
20162016
params_base.n_cache_reuse = 0;
20172017
SRV_WRN("%s\n", "cache_reuse is not supported by this context, it will be disabled");
20182018
}
2019+
2020+
if (!params_base.speculative.model.path.empty()) {
2021+
SRV_ERR("%s\n", "err: speculative decode is not supported by this context");
2022+
return false;
2023+
}
20192024
}
20202025

20212026
return true;
@@ -3203,9 +3208,7 @@ struct server_context {
32033208
}
32043209
} else {
32053210
// if we don't cache the prompt, we have to remove the entire KV cache
3206-
llama_kv_self_seq_rm(ctx, slot.id, 0, -1);
32073211
slot.n_past = 0;
3208-
slot.cache_tokens.clear(); // TODO: not needed, will be cleared later via "keep_first()"
32093212
}
32103213

32113214
if (slot.n_past > 0 && slot.n_past < (int) slot.cache_tokens.size()) {
@@ -3220,7 +3223,6 @@ struct server_context {
32203223
SLT_WRN(slot, "n_past = %d, cache_tokens.size() = %d, seq_id = %d, pos_min = %d, n_swa = %d\n", slot.n_past, (int) slot.cache_tokens.size(), slot.id, pos_min, n_swa);
32213224
SLT_WRN(slot, "forcing full prompt re-processing due to lack of cache data (likely due to SWA, see %s)\n",
32223225
"https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055");
3223-
llama_kv_self_seq_rm(ctx, slot.id, 0, -1);
32243226
slot.n_past = 0;
32253227
}
32263228
}

0 commit comments

Comments
 (0)