kv-cache : allow context shift for recurrent models

compilade · compilade · commit e94f3932f2db · 2025-05-02T19:29:23.000-04:00
diff --git a/src/llama-kv-cache.cpp b/src/llama-kv-cache.cpp
@@ -1938,7 +1938,8 @@ llama_pos llama_kv_cache_recurrent::get_pos_max() const {
 }
 
 bool llama_kv_cache_recurrent::get_can_shift() const {
-    return false;
+    // shifting is trivial, the recurrent states don't care about the absolute position
+    return true;
 }
 
 uint32_t llama_kv_cache_recurrent::cell_max() const {

Original file line number	Diff line number	Diff line change
`@@ -1938,7 +1938,8 @@ llama_pos llama_kv_cache_recurrent::get_pos_max() const {`
`1938`	`1938`	`}`
`1939`	`1939`
`1940`	`1940`	`bool llama_kv_cache_recurrent::get_can_shift() const {`
`1941`		`- return false;`
	`1941`	`+ // shifting is trivial, the recurrent states don't care about the absolute position`
	`1942`	`+ return true;`
`1942`	`1943`	`}`
`1943`	`1944`
`1944`	`1945`	`uint32_t llama_kv_cache_recurrent::cell_max() const {`