Skip to content

Commit 02cd957

Browse files
ggerganovggerganov
committed
cont : remove the parameter and the sched resets
ggml-ci
1 parent ffc7634 commit 02cd957

File tree

6 files changed

+12
-57
lines changed

6 files changed

+12
-57
lines changed

‎common/arg.cpp

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1464,14 +1464,6 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
14641464
params.swa_full = true;
14651465
}
14661466
).set_env("LLAMA_ARG_SWA_FULL"));
1467-
add_opt(common_arg(
1468-
{"--graph-reuse", "-gr"},
1469-
string_format("reuse previous compute graphs when possible (default: %s)"
1470-
"[(more info)](https://github.com/ggml-org/llama.cpp/pull/14482)", params.graph_reuse ? "true" : "false"),
1471-
[](common_params & params) {
1472-
params.graph_reuse = true;
1473-
}
1474-
).set_env("LLAMA_ARG_GRAPH_REUSE"));
14751467
add_opt(common_arg(
14761468
{"--no-context-shift"},
14771469
string_format("disables context shift on infinite text generation (default: %s)", params.ctx_shift ? "disabled" : "enabled"),

‎common/common.cpp

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1157,7 +1157,6 @@ struct llama_context_params common_context_params_to_llama(const common_params &
11571157
cparams.no_perf = params.no_perf;
11581158
cparams.op_offload = !params.no_op_offload;
11591159
cparams.swa_full = params.swa_full;
1160-
cparams.graph_reuse = params.graph_reuse;
11611160

11621161
cparams.type_k = params.cache_type_k;
11631162
cparams.type_v = params.cache_type_v;

‎common/common.h

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -330,7 +330,6 @@ struct common_params {
330330
bool no_perf = false; // disable performance metrics
331331
bool ctx_shift = true; // context shift on inifinite text generation
332332
bool swa_full = false; // use full-size SWA cache (https://github.com/ggml-org/llama.cpp/pull/13194#issuecomment-2868343055)
333-
bool graph_reuse = false; // reuse previous compute graphs when possible
334333

335334
bool input_prefix_bos = false; // prefix BOS to user inputs, preceding input_prefix
336335
bool use_mmap = true; // use mmap for faster loads

‎src/llama-context.cpp

Lines changed: 8 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -718,7 +718,14 @@ llm_graph_result_i * llama_context::process_ubatch(const llama_ubatch & ubatch,
718718
}
719719
}
720720

721-
res->set_inputs(&ubatch);
721+
// set the input data for the input tensors
722+
{
723+
//const auto t_start_us = ggml_time_us();
724+
725+
res->set_inputs(&ubatch);
726+
727+
//LLAMA_LOG_INFO("graph set inputs time: %.3f ms\n", (ggml_time_us() - t_start_us)/1000.0);
728+
}
722729

723730
const auto status = graph_compute(res->get_gf(), ubatch.n_tokens > 1);
724731
if (status != GGML_STATUS_SUCCESS) {
@@ -852,12 +859,6 @@ int llama_context::encode(const llama_batch & batch_inp) {
852859
}
853860
}
854861

855-
// Reset state for the next token before backend sync, to allow the CPU activities in the reset to
856-
// overlap with device computation.
857-
if (!cparams.graph_reuse) {
858-
ggml_backend_sched_reset(sched.get());
859-
}
860-
861862
// TODO: hacky solution
862863
if (model.arch == LLM_ARCH_T5 && t_embd) {
863864
//cross.t_embd = t_embd;
@@ -1197,12 +1198,6 @@ int llama_context::decode(const llama_batch & batch_inp) {
11971198
// wait for the computation to finish (automatically done when obtaining the model output)
11981199
//synchronize();
11991200

1200-
// Reset state for the next token before backend sync, to allow the CPU activities in the reset to
1201-
// overlap with device computation.
1202-
if (!cparams.graph_reuse) {
1203-
ggml_backend_sched_reset(sched.get());
1204-
}
1205-
12061201
return 0;
12071202
}
12081203

‎src/llama-kv-cache-unified.cpp

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -963,6 +963,7 @@ void llama_kv_cache_unified::set_input_kq_mask(ggml_tensor * dst, const llama_ub
963963
// xxxxx-----
964964
// xxxxx-----
965965
// To visualize the mask, see https://github.com/ggml-org/llama.cpp/pull/12615
966+
// TODO: optimize this section
966967
for (uint32_t h = 0; h < 1; ++h) {
967968
for (uint32_t i = 0; i < n_tokens; ++i) {
968969
const llama_seq_id seq_id = ubatch->seq_id[i][0];

‎tools/llama-bench/llama-bench.cpp

Lines changed: 3 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -261,7 +261,6 @@ struct cmd_params {
261261
std::vector<bool> use_mmap;
262262
std::vector<bool> embeddings;
263263
std::vector<bool> no_op_offload;
264-
std::vector<bool> graph_reuse;
265264
ggml_numa_strategy numa;
266265
int reps;
267266
ggml_sched_priority prio;
@@ -299,7 +298,6 @@ static const cmd_params cmd_params_defaults = {
299298
/* use_mmap */ { true },
300299
/* embeddings */ { false },
301300
/* no_op_offload */ { false },
302-
/* graph_reuse */ { false },
303301
/* numa */ GGML_NUMA_STRATEGY_DISABLED,
304302
/* reps */ 5,
305303
/* prio */ GGML_SCHED_PRIO_NORMAL,
@@ -379,7 +377,6 @@ static void print_usage(int /* argc */, char ** argv) {
379377
printf(" -ot --override-tensors <tensor name pattern>=<buffer type>;...\n");
380378
printf(" (default: disabled)\n");
381379
printf(" -nopo, --no-op-offload <0|1> (default: 0)\n");
382-
printf(" -gr, --graph-reuse <0|1> (default: 0)\n");
383380
printf("\n");
384381
printf(
385382
"Multiple values can be given for each parameter by separating them with ','\n"
@@ -623,13 +620,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
623620
}
624621
auto p = string_split<bool>(argv[i], split_delim);
625622
params.no_kv_offload.insert(params.no_kv_offload.end(), p.begin(), p.end());
626-
} else if (arg == "-gr" || arg == "--graph-reuse") {
627-
if (++i >= argc) {
628-
invalid_param = true;
629-
break;
630-
}
631-
auto p = string_split<bool>(argv[i], split_delim);
632-
params.graph_reuse.insert(params.graph_reuse.end(), p.begin(), p.end());
633623
} else if (arg == "--numa") {
634624
if (++i >= argc) {
635625
invalid_param = true;
@@ -895,9 +885,6 @@ static cmd_params parse_cmd_params(int argc, char ** argv) {
895885
if (params.no_op_offload.empty()) {
896886
params.no_op_offload = cmd_params_defaults.no_op_offload;
897887
}
898-
if (params.graph_reuse.empty()) {
899-
params.graph_reuse = cmd_params_defaults.graph_reuse;
900-
}
901888
if (params.n_threads.empty()) {
902889
params.n_threads = cmd_params_defaults.n_threads;
903890
}
@@ -939,7 +926,6 @@ struct cmd_params_instance {
939926
bool use_mmap;
940927
bool embeddings;
941928
bool no_op_offload;
942-
bool graph_reuse;
943929

944930
llama_model_params to_llama_mparams() const {
945931
llama_model_params mparams = llama_model_default_params();
@@ -1012,7 +998,6 @@ struct cmd_params_instance {
1012998
cparams.embeddings = embeddings;
1013999
cparams.op_offload = !no_op_offload;
10141000
cparams.swa_full = false;
1015-
cparams.graph_reuse = graph_reuse;
10161001

10171002
return cparams;
10181003
}
@@ -1033,7 +1018,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
10331018
for (const auto & mmp : params.use_mmap)
10341019
for (const auto & embd : params.embeddings)
10351020
for (const auto & nopo : params.no_op_offload)
1036-
for (const auto & gr : params.graph_reuse)
10371021
for (const auto & nb : params.n_batch)
10381022
for (const auto & nub : params.n_ubatch)
10391023
for (const auto & tk : params.type_k)
@@ -1075,7 +1059,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
10751059
/* .use_mmap = */ mmp,
10761060
/* .embeddings = */ embd,
10771061
/* .no_op_offload= */ nopo,
1078-
/* .graph_reuse = */ gr,
10791062
};
10801063
instances.push_back(instance);
10811064
}
@@ -1109,7 +1092,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
11091092
/* .use_mmap = */ mmp,
11101093
/* .embeddings = */ embd,
11111094
/* .no_op_offload= */ nopo,
1112-
/* .graph_reuse = */ gr,
11131095
};
11141096
instances.push_back(instance);
11151097
}
@@ -1143,7 +1125,6 @@ static std::vector<cmd_params_instance> get_cmd_params_instances(const cmd_param
11431125
/* .use_mmap = */ mmp,
11441126
/* .embeddings = */ embd,
11451127
/* .no_op_offload= */ nopo,
1146-
/* .graph_reuse = */ gr,
11471128
};
11481129
instances.push_back(instance);
11491130
}
@@ -1181,7 +1162,6 @@ struct test {
11811162
bool use_mmap;
11821163
bool embeddings;
11831164
bool no_op_offload;
1184-
bool graph_reuse;
11851165
int n_prompt;
11861166
int n_gen;
11871167
int n_depth;
@@ -1217,7 +1197,6 @@ struct test {
12171197
use_mmap = inst.use_mmap;
12181198
embeddings = inst.embeddings;
12191199
no_op_offload = inst.no_op_offload;
1220-
graph_reuse = inst.graph_reuse;
12211200
n_prompt = inst.n_prompt;
12221201
n_gen = inst.n_gen;
12231202
n_depth = inst.n_depth;
@@ -1264,8 +1243,8 @@ struct test {
12641243
"cpu_mask", "cpu_strict", "poll", "type_k", "type_v", "n_gpu_layers",
12651244
"split_mode", "main_gpu", "no_kv_offload", "flash_attn", "tensor_split", "tensor_buft_overrides",
12661245
"defrag_thold",
1267-
"use_mmap", "embeddings", "no_op_offload", "graph_reuse", "n_prompt", "n_gen", "n_depth",
1268-
"test_time", "avg_ns", "stddev_ns", "avg_ts", "stddev_ts",
1246+
"use_mmap", "embeddings", "no_op_offload", "n_prompt", "n_gen", "n_depth", "test_time",
1247+
"avg_ns", "stddev_ns", "avg_ts", "stddev_ts",
12691248
};
12701249
return fields;
12711250
}
@@ -1280,7 +1259,7 @@ struct test {
12801259
return INT;
12811260
}
12821261
if (field == "f16_kv" || field == "no_kv_offload" || field == "cpu_strict" || field == "flash_attn" ||
1283-
field == "use_mmap" || field == "embeddings" || field == "graph_reuse") {
1262+
field == "use_mmap" || field == "embeddings") {
12841263
return BOOL;
12851264
}
12861265
if (field == "avg_ts" || field == "stddev_ts" || field == "defrag_thold") {
@@ -1354,7 +1333,6 @@ struct test {
13541333
std::to_string(use_mmap),
13551334
std::to_string(embeddings),
13561335
std::to_string(no_op_offload),
1357-
std::to_string(graph_reuse),
13581336
std::to_string(n_prompt),
13591337
std::to_string(n_gen),
13601338
std::to_string(n_depth),
@@ -1540,9 +1518,6 @@ struct markdown_printer : public printer {
15401518
if (field == "no_op_offload") {
15411519
return 4;
15421520
}
1543-
if (field == "graph_reuse") {
1544-
return 4;
1545-
}
15461521

15471522
int width = std::max((int) field.length(), 10);
15481523

@@ -1577,9 +1552,6 @@ struct markdown_printer : public printer {
15771552
if (field == "no_op_offload") {
15781553
return "nopo";
15791554
}
1580-
if (field == "graph_reuse") {
1581-
return "gr";
1582-
}
15831555
if (field == "tensor_split") {
15841556
return "ts";
15851557
}
@@ -1654,9 +1626,6 @@ struct markdown_printer : public printer {
16541626
if (params.no_op_offload.size() > 1 || params.no_op_offload != cmd_params_defaults.no_op_offload) {
16551627
fields.emplace_back("no_op_offload");
16561628
}
1657-
if (params.graph_reuse.size() > 1 || params.graph_reuse != cmd_params_defaults.graph_reuse) {
1658-
fields.emplace_back("graph_reuse");
1659-
}
16601629
fields.emplace_back("test");
16611630
fields.emplace_back("t/s");
16621631

0 commit comments

Comments
 (0)