Skip to content

Commit 886da0a

Browse files
ggerganovggerganov
committed
kv-cache : prepare K/V buffers for separation
ggml-ci
1 parent 7b63a71 commit 886da0a

File tree

3 files changed

+127
-36
lines changed

3 files changed

+127
-36
lines changed

‎src/llama-hparams.cpp

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,46 @@ uint32_t llama_hparams::n_embd_v_gqa(uint32_t il) const {
6565
return n_embd_head_v * n_head_kv;
6666
}
6767

68+
bool llama_hparams::is_n_embd_k_gqa_variable() const {
69+
const uint32_t val = n_embd_k_gqa();
70+
for (uint32_t il = 0; il < n_layer; ++il) {
71+
if (val != n_embd_k_gqa(il)) {
72+
return true;
73+
}
74+
}
75+
76+
return false;
77+
}
78+
79+
bool llama_hparams::is_n_embd_v_gqa_variable() const {
80+
const uint32_t val = n_embd_v_gqa();
81+
for (uint32_t il = 0; il < n_layer; ++il) {
82+
if (val != n_embd_v_gqa(il)) {
83+
return true;
84+
}
85+
}
86+
87+
return false;
88+
}
89+
90+
uint32_t llama_hparams::n_embd_k_gqa_max() const {
91+
uint32_t val = n_embd_k_gqa();
92+
for (uint32_t il = 0; il < n_layer; ++il) {
93+
val = std::max(val, n_embd_k_gqa(il));
94+
}
95+
96+
return val;
97+
}
98+
99+
uint32_t llama_hparams::n_embd_v_gqa_max() const {
100+
uint32_t val = n_embd_v_gqa();
101+
for (uint32_t il = 0; il < n_layer; ++il) {
102+
val = std::max(val, n_embd_v_gqa(il));
103+
}
104+
105+
return val;
106+
}
107+
68108
uint32_t llama_hparams::n_embd_r() const {
69109
if (wkv_head_size != 0) {
70110
// for RWKV models

‎src/llama-hparams.h

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,14 @@ struct llama_hparams {
189189
// dimension of value embeddings across all k-v heads
190190
uint32_t n_embd_v_gqa(uint32_t il = 0) const;
191191

192+
// true if any layer has a different n_embd_k_gqa/n_embd_v_gqa
193+
bool is_n_embd_k_gqa_variable() const;
194+
bool is_n_embd_v_gqa_variable() const;
195+
196+
// return the maximum n_embd_k_gqa/n_embd_v_gqa across all layers
197+
uint32_t n_embd_k_gqa_max() const;
198+
uint32_t n_embd_v_gqa_max() const;
199+
192200
// dimension of the rolling state embeddings
193201
// corresponds to Mamba's conv_states size or RWKV's token_shift states size
194202
uint32_t n_embd_r() const;

‎src/llama-kv-cache-unified.cpp

Lines changed: 79 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -68,14 +68,21 @@ llama_kv_cache_unified::llama_kv_cache_unified(
6868

6969
cells.resize(kv_size);
7070

71+
// [TAG_V_CACHE_VARIABLE]
72+
if (v_trans && hparams.is_n_embd_v_gqa_variable()) {
73+
LLAMA_LOG_WARN("%s: the V embeddings have different sizes across layers and FA is not enabled - padding V cache to %d\n",
74+
__func__, hparams.n_embd_v_gqa_max());
75+
}
76+
7177
for (uint32_t il = 0; il < n_layer_cache; il++) {
7278
if (filter && !filter(il)) {
7379
LLAMA_LOG_DEBUG("%s: layer %3d: skipped\n", __func__, il);
7480
continue;
7581
}
7682

77-
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
78-
const uint32_t n_embd_v_gqa = hparams.n_embd_v_gqa(il);
83+
// [TAG_V_CACHE_VARIABLE]
84+
const uint32_t n_embd_k_gqa = hparams.n_embd_k_gqa(il);
85+
const uint32_t n_embd_v_gqa = !v_trans ? hparams.n_embd_v_gqa(il) : hparams.n_embd_v_gqa_max();
7986

8087
const char * dev_name = "CPU";
8188

@@ -98,8 +105,8 @@ llama_kv_cache_unified::llama_kv_cache_unified(
98105
ggml_tensor * k;
99106
ggml_tensor * v;
100107

101-
k = ggml_new_tensor_2d(ctx, type_k, n_embd_k_gqa, kv_size);
102-
v = ggml_new_tensor_2d(ctx, type_v, n_embd_v_gqa, kv_size);
108+
k = ggml_new_tensor_3d(ctx, type_k, n_embd_k_gqa, kv_size, 1);
109+
v = ggml_new_tensor_3d(ctx, type_v, n_embd_v_gqa, kv_size, 1);
103110

104111
ggml_format_name(k, "cache_k_l%d", il);
105112
ggml_format_name(v, "cache_v_l%d", il);
@@ -780,33 +787,47 @@ ggml_tensor * llama_kv_cache_unified::get_k(ggml_context * ctx, int32_t il, uint
780787

781788
auto * k = layers[ikv].k;
782789

783-
return ggml_view_3d(ctx, k,
784-
hparams.n_embd_head_k, hparams.n_head_kv(il), n_kv,
790+
const uint64_t kv_size = get_size();
791+
const uint64_t n_embd_k_gqa = k->ne[0];
792+
793+
assert(n_embd_k_gqa == hparams.n_embd_k_gqa(il));
794+
795+
return ggml_view_4d(ctx, k,
796+
hparams.n_embd_head_k, hparams.n_head_kv(il), n_kv, 1,
785797
ggml_row_size(k->type, hparams.n_embd_head_k),
786-
ggml_row_size(k->type, hparams.n_embd_k_gqa(il)),
787-
0);
798+
ggml_row_size(k->type, n_embd_k_gqa),
799+
ggml_row_size(k->type, n_embd_k_gqa*kv_size),
800+
ggml_row_size(k->type, n_embd_k_gqa*kv_size)*0);
788801
}
789802

790803
ggml_tensor * llama_kv_cache_unified::get_v(ggml_context * ctx, int32_t il, uint32_t n_kv) const {
791804
const int32_t ikv = map_layer_ids.at(il);
792805

793806
auto * v = layers[ikv].v;
794807

808+
const uint64_t kv_size = get_size();
809+
const uint64_t n_embd_v_gqa = v->ne[0];
810+
811+
// [TAG_V_CACHE_VARIABLE]
812+
assert(n_embd_v_gqa >= hparams.n_embd_v_gqa(il));
813+
795814
if (!v_trans) {
796815
// note: v->nb[1] <= v->nb[2]
797-
return ggml_view_3d(ctx, v,
798-
hparams.n_embd_head_v, hparams.n_head_kv(il), n_kv,
799-
ggml_row_size(v->type, hparams.n_embd_head_v), // v->nb[1]
800-
ggml_row_size(v->type, hparams.n_embd_v_gqa(il)), // v->nb[2]
801-
0);
816+
return ggml_view_4d(ctx, v,
817+
hparams.n_embd_head_v, hparams.n_head_kv(il), n_kv, 1,
818+
ggml_row_size(v->type, hparams.n_embd_head_v), // v->nb[1]
819+
ggml_row_size(v->type, n_embd_v_gqa), // v->nb[2]
820+
ggml_row_size(v->type, n_embd_v_gqa*kv_size), // v->nb[3]
821+
ggml_row_size(v->type, n_embd_v_gqa*kv_size)*0);
802822
}
803823

804824
// note: v->nb[1] > v->nb[2]
805-
return ggml_view_3d(ctx, v,
806-
n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v,
807-
ggml_row_size(v->type, v->ne[1]*hparams.n_embd_head_v), // v->nb[1]
808-
ggml_row_size(v->type, v->ne[1]), // v->nb[2]
809-
0);
825+
return ggml_view_4d(ctx, v,
826+
n_kv, hparams.n_head_kv(il), hparams.n_embd_head_v, 1,
827+
ggml_row_size(v->type, kv_size*hparams.n_embd_head_v), // v->nb[1]
828+
ggml_row_size(v->type, kv_size), // v->nb[2]
829+
ggml_row_size(v->type, kv_size*n_embd_v_gqa), // v->nb[3]
830+
ggml_row_size(v->type, kv_size*n_embd_v_gqa)*0);
810831
}
811832

812833
ggml_tensor * llama_kv_cache_unified::cpy_k(ggml_context * ctx, ggml_tensor * k_cur, ggml_tensor * k_idxs, int32_t il, const slot_info & sinfo) const {
@@ -820,6 +841,10 @@ ggml_tensor * llama_kv_cache_unified::cpy_k(ggml_context * ctx, ggml_tensor * k_
820841
k_cur = ggml_reshape_2d(ctx, k_cur, k->ne[0], n_tokens);
821842

822843
if (k_idxs && supports_set_rows) {
844+
if (k->ne[2] > 1) {
845+
k = ggml_reshape_2d(ctx, k, k->ne[0], k->ne[1]*k->ne[2]);
846+
}
847+
823848
return ggml_set_rows(ctx, k, k_cur, k_idxs);
824849
}
825850

@@ -838,31 +863,30 @@ ggml_tensor * llama_kv_cache_unified::cpy_v(ggml_context * ctx, ggml_tensor * v_
838863

839864
auto * v = layers[ikv].v;
840865

841-
const int64_t n_embd_v_gqa = v->ne[0];
842-
const int64_t n_tokens = v_cur->ne[2];
866+
const int64_t n_embd_v_gqa = v_cur->ne[0]*v_cur->ne[1];
867+
const int64_t n_tokens = v_cur->ne[2];
843868

844869
v_cur = ggml_reshape_2d(ctx, v_cur, n_embd_v_gqa, n_tokens);
845870

846871
if (v_idxs && supports_set_rows) {
847872
if (!v_trans) {
873+
if (v->ne[2] > 1) {
874+
v = ggml_reshape_2d(ctx, v, v->ne[0], v->ne[1]*v->ne[2]);
875+
}
876+
848877
return ggml_set_rows(ctx, v, v_cur, v_idxs);
849878
}
850879

851-
// the row becomes a single element
852-
ggml_tensor * v_view = ggml_reshape_3d(ctx, v, 1, v->ne[1], v->ne[0]);
880+
// [TAG_V_CACHE_VARIABLE]
881+
if (n_embd_v_gqa < v->ne[0]) {
882+
v_cur = ggml_pad(ctx, v_cur, v->ne[0] - n_embd_v_gqa, 0, 0, 0);
883+
}
853884

854-
// note: the V cache is transposed when not using flash attention
855-
v_cur = ggml_permute(ctx, ggml_reshape_3d(ctx, v_cur, v_cur->ne[0], 1, v_cur->ne[1]), 2, 0, 1, 3);
885+
// the row becomes a single element
886+
ggml_tensor * v_view = ggml_reshape_2d(ctx, v, 1, v->ne[0]*v->ne[1]*v->ne[2]);
856887

857-
// note: we can be more explicit here at the cost of extra cont
858-
// however, above we take advantage that a row of single element is always continuous regardless of the row stride
859-
//v_cur = ggml_transpose(ctx, v_cur);
860-
//v_cur = ggml_cont_3d(ctx, v_cur, 1, v_cur->ne[0], v_cur->ne[1]);
888+
v_cur = ggml_reshape_2d(ctx, v_cur, 1, v_cur->ne[0]*v_cur->ne[1]);
861889

862-
// we broadcast the KV indices n_embd_v_gqa times
863-
// v [1, n_kv, n_embd_v_gqa]
864-
// v_cur [1, n_tokens, n_embd_v_gqa]
865-
// v_idxs [n_tokens, 1, 1]
866890
return ggml_set_rows(ctx, v_view, v_cur, v_idxs);
867891
}
868892

@@ -899,7 +923,13 @@ ggml_tensor * llama_kv_cache_unified::build_input_k_idxs(ggml_context * ctx, con
899923
ggml_tensor * llama_kv_cache_unified::build_input_v_idxs(ggml_context * ctx, const llama_ubatch & ubatch) const {
900924
const uint32_t n_tokens = ubatch.n_tokens;
901925

902-
ggml_tensor * v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens);
926+
ggml_tensor * v_idxs;
927+
928+
if (!v_trans) {
929+
v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens);
930+
} else {
931+
v_idxs = ggml_new_tensor_1d(ctx, GGML_TYPE_I64, n_tokens*hparams.n_embd_v_gqa_max());
932+
}
903933

904934
ggml_set_input(v_idxs);
905935

@@ -916,7 +946,7 @@ void llama_kv_cache_unified::set_input_k_idxs(ggml_tensor * dst, const llama_uba
916946
GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
917947
int64_t * data = (int64_t *) dst->data;
918948

919-
for (int64_t i = 0; i < n_tokens; ++i) {
949+
for (uint32_t i = 0; i < n_tokens; ++i) {
920950
data[i] = sinfo.idxs.at(i);
921951
}
922952
}
@@ -931,8 +961,21 @@ void llama_kv_cache_unified::set_input_v_idxs(ggml_tensor * dst, const llama_uba
931961
GGML_ASSERT(ggml_backend_buffer_is_host(dst->buffer));
932962
int64_t * data = (int64_t *) dst->data;
933963

934-
for (int64_t i = 0; i < n_tokens; ++i) {
935-
data[i] = sinfo.idxs.at(i);
964+
if (!v_trans) {
965+
for (uint32_t i = 0; i < n_tokens; ++i) {
966+
data[i] = sinfo.idxs.at(i);
967+
}
968+
} else {
969+
// note: the V cache is transposed when not using flash attention
970+
const int64_t kv_size = get_size();
971+
972+
const int64_t n_embd_v_gqa = hparams.n_embd_v_gqa_max();
973+
974+
for (uint32_t i = 0; i < n_tokens; ++i) {
975+
for (uint32_t j = 0; j < n_embd_v_gqa; ++j) {
976+
data[i*n_embd_v_gqa + j] = j*kv_size + sinfo.idxs.at(i);
977+
}
978+
}
936979
}
937980
}
938981

0 commit comments

Comments
 (0)