Skip to content

Commit 11f1859

Browse files
Artefact2Artefact2
committed
common, server : add top-a sampler
This improves compatibility with clients like AI Horde.
1 parent 5207b3f commit 11f1859

File tree

7 files changed

+41
-5
lines changed

7 files changed

+41
-5
lines changed

‎common/common.cpp

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,12 @@ bool gpt_params_parse_ex(int argc, char ** argv, gpt_params & params) {
360360
break;
361361
}
362362
sparams.min_p = std::stof(argv[i]);
363+
} else if (arg == "--top-a") {
364+
if (++i >= argc) {
365+
invalid_param = true;
366+
break;
367+
}
368+
sparams.top_a = std::stof(argv[i]);
363369
} else if (arg == "--temp") {
364370
if (++i >= argc) {
365371
invalid_param = true;
@@ -970,6 +976,7 @@ void gpt_print_usage(int /*argc*/, char ** argv, const gpt_params & params) {
970976
printf(" --top-k N top-k sampling (default: %d, 0 = disabled)\n", sparams.top_k);
971977
printf(" --top-p N top-p sampling (default: %.1f, 1.0 = disabled)\n", (double)sparams.top_p);
972978
printf(" --min-p N min-p sampling (default: %.1f, 0.0 = disabled)\n", (double)sparams.min_p);
979+
printf(" --top-a N top-a sampling (default: %.1f, 0.0 = disabled)\n", (double)sparams.top_a);
973980
printf(" --tfs N tail free sampling, parameter z (default: %.1f, 1.0 = disabled)\n", (double)sparams.tfs_z);
974981
printf(" --typical N locally typical sampling, parameter p (default: %.1f, 1.0 = disabled)\n", (double)sparams.typical_p);
975982
printf(" --repeat-last-n N last n tokens to consider for penalize (default: %d, 0 = disabled, -1 = ctx_size)\n", sparams.penalty_last_n);
@@ -1140,6 +1147,7 @@ std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::
11401147
{"top_p", llama_sampler_type::TOP_P},
11411148
{"typical_p", llama_sampler_type::TYPICAL_P},
11421149
{"min_p", llama_sampler_type::MIN_P},
1150+
{"top_a", llama_sampler_type::TOP_A},
11431151
{"tfs_z", llama_sampler_type::TFS_Z},
11441152
{"temperature", llama_sampler_type::TEMPERATURE}
11451153
};
@@ -1153,6 +1161,7 @@ std::vector<llama_sampler_type> sampler_types_from_names(const std::vector<std::
11531161
{"typical-p", llama_sampler_type::TYPICAL_P},
11541162
{"typical", llama_sampler_type::TYPICAL_P},
11551163
{"min-p", llama_sampler_type::MIN_P},
1164+
{"top-a", llama_sampler_type::TOP_A},
11561165
{"tfs-z", llama_sampler_type::TFS_Z},
11571166
{"tfs", llama_sampler_type::TFS_Z},
11581167
{"temp", llama_sampler_type::TEMPERATURE}
@@ -1188,6 +1197,7 @@ std::vector<llama_sampler_type> sampler_types_from_chars(const std::string & nam
11881197
{'p', llama_sampler_type::TOP_P},
11891198
{'y', llama_sampler_type::TYPICAL_P},
11901199
{'m', llama_sampler_type::MIN_P},
1200+
{'a', llama_sampler_type::TOP_A},
11911201
{'f', llama_sampler_type::TFS_Z},
11921202
{'t', llama_sampler_type::TEMPERATURE}
11931203
};
@@ -1210,6 +1220,7 @@ std::string sampler_type_to_name_string(llama_sampler_type sampler_type) {
12101220
case llama_sampler_type::TYPICAL_P: return "typical_p";
12111221
case llama_sampler_type::TOP_P: return "top_p";
12121222
case llama_sampler_type::MIN_P: return "min_p";
1223+
case llama_sampler_type::TOP_A: return "top_a";
12131224
case llama_sampler_type::TEMPERATURE: return "temperature";
12141225
default : return "";
12151226
}
@@ -1755,6 +1766,7 @@ void dump_non_result_info_yaml(FILE * stream, const gpt_params & params, const l
17551766
fprintf(stream, "top_k: %d # default: 40\n", sparams.top_k);
17561767
fprintf(stream, "top_p: %f # default: 0.95\n", sparams.top_p);
17571768
fprintf(stream, "min_p: %f # default: 0.0\n", sparams.min_p);
1769+
fprintf(stream, "top_a: %f # default: 0.0\n", sparams.top_a);
17581770
fprintf(stream, "typical_p: %f # default: 1.0\n", sparams.typical_p);
17591771
fprintf(stream, "verbose_prompt: %s # default: false\n", params.verbose_prompt ? "true" : "false");
17601772
fprintf(stream, "display_prompt: %s # default: true\n", params.display_prompt ? "true" : "false");

‎common/sampling.cpp

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,10 +91,10 @@ std::string llama_sampling_print(const llama_sampling_params & params) {
9191

9292
snprintf(result, sizeof(result),
9393
"\trepeat_last_n = %d, repeat_penalty = %.3f, frequency_penalty = %.3f, presence_penalty = %.3f\n"
94-
"\ttop_k = %d, tfs_z = %.3f, top_p = %.3f, min_p = %.3f, typical_p = %.3f, temp = %.3f\n"
94+
"\ttop_k = %d, tfs_z = %.3f, top_p = %.3f, min_p = %.3f, top_a = %.3f, typical_p = %.3f, temp = %.3f\n"
9595
"\tmirostat = %d, mirostat_lr = %.3f, mirostat_ent = %.3f",
9696
params.penalty_last_n, params.penalty_repeat, params.penalty_freq, params.penalty_present,
97-
params.top_k, params.tfs_z, params.top_p, params.min_p, params.typical_p, params.temp,
97+
params.top_k, params.tfs_z, params.top_p, params.min_p, params.top_a, params.typical_p, params.temp,
9898
params.mirostat, params.mirostat_eta, params.mirostat_tau);
9999

100100
return std::string(result);
@@ -128,6 +128,7 @@ static void sampler_queue(
128128
const int32_t top_k = params.top_k;
129129
const float top_p = params.top_p;
130130
const float min_p = params.min_p;
131+
const float top_a = params.top_a;
131132
const float tfs_z = params.tfs_z;
132133
const float typical_p = params.typical_p;
133134
const std::vector<llama_sampler_type> & samplers_sequence = params.samplers_sequence;
@@ -139,6 +140,7 @@ static void sampler_queue(
139140
case llama_sampler_type::TYPICAL_P: llama_sample_typical (ctx_main, &cur_p, typical_p, min_keep); break;
140141
case llama_sampler_type::TOP_P : llama_sample_top_p (ctx_main, &cur_p, top_p, min_keep); break;
141142
case llama_sampler_type::MIN_P : llama_sample_min_p (ctx_main, &cur_p, min_p, min_keep); break;
143+
case llama_sampler_type::TOP_A : llama_sample_top_a (ctx_main, &cur_p, top_a, min_keep); break;
142144
case llama_sampler_type::TEMPERATURE:
143145
if (dynatemp_range > 0) {
144146
float dynatemp_min = std::max(0.0f, temp - dynatemp_range);

‎common/sampling.h

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,7 @@ enum class llama_sampler_type : char {
1313
TOP_K = 'k',
1414
TOP_P = 'p',
1515
MIN_P = 'm',
16+
TOP_A = 'a',
1617
TFS_Z = 'f',
1718
TYPICAL_P = 'y',
1819
TEMPERATURE = 't'
@@ -26,6 +27,7 @@ typedef struct llama_sampling_params {
2627
int32_t top_k = 40; // <= 0 to use vocab size
2728
float top_p = 0.95f; // 1.0 = disabled
2829
float min_p = 0.05f; // 0.0 = disabled
30+
float top_a = 0.00f; // 0.0 = disabled
2931
float tfs_z = 1.00f; // 1.0 = disabled
3032
float typical_p = 1.00f; // 1.0 = disabled
3133
float temp = 0.80f; // <= 0.0 to sample greedily, 0.0 to not output probabilities
@@ -46,6 +48,7 @@ typedef struct llama_sampling_params {
4648
llama_sampler_type::TYPICAL_P,
4749
llama_sampler_type::TOP_P,
4850
llama_sampler_type::MIN_P,
51+
llama_sampler_type::TOP_A,
4952
llama_sampler_type::TEMPERATURE
5053
};
5154

‎examples/server/README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -158,6 +158,8 @@ node index.js
158158

159159
`min_p`: The minimum probability for a token to be considered, relative to the probability of the most likely token (default: 0.05).
160160

161+
`top_a`: Limit the next token selection to a subset of tokens with a probability above a*P^2, where P is the most probable token (default: 0.0, 0.0 = disabled).
162+
161163
`n_predict`: Set the maximum number of tokens to predict when generating text. **Note:** May exceed the set limit slightly if the last token is a partial multibyte character. When 0, no tokens will be generated but the prompt is evaluated into the cache. (default: -1, -1 = infinity).
162164

163165
`n_keep`: Specify the number of tokens from the prompt to retain when the context size is exceeded and tokens need to be discarded.

‎examples/server/server.cpp

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -531,6 +531,7 @@ struct llama_server_context
531531
slot->sparams.top_k = json_value(data, "top_k", default_sparams.top_k);
532532
slot->sparams.top_p = json_value(data, "top_p", default_sparams.top_p);
533533
slot->sparams.min_p = json_value(data, "min_p", default_sparams.min_p);
534+
slot->sparams.top_a = json_value(data, "top_a", default_sparams.top_a);
534535
slot->sparams.tfs_z = json_value(data, "tfs_z", default_sparams.tfs_z);
535536
slot->sparams.typical_p = json_value(data, "typical_p", default_sparams.typical_p);
536537
slot->sparams.temp = json_value(data, "temperature", default_sparams.temp);
@@ -1075,6 +1076,7 @@ struct llama_server_context
10751076
{"top_k", slot.sparams.top_k},
10761077
{"top_p", slot.sparams.top_p},
10771078
{"min_p", slot.sparams.min_p},
1079+
{"top_a", slot.sparams.top_a},
10781080
{"tfs_z", slot.sparams.tfs_z},
10791081
{"typical_p", slot.sparams.typical_p},
10801082
{"repeat_last_n", slot.sparams.penalty_last_n},

‎llama.cpp

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -9362,7 +9362,7 @@ void llama_sample_top_p(struct llama_context * ctx, llama_token_data_array * can
93629362
}
93639363
}
93649364

9365-
void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
9365+
static void llama_sample_min_p_pow(struct llama_context * ctx, llama_token_data_array * candidates, float p, float pow, size_t min_keep) {
93669366
if (p <= 0.0f || !candidates->size) {
93679367
return;
93689368
}
@@ -9379,7 +9379,7 @@ void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * can
93799379
for (size_t i = 0; i < candidates->size; ++i) {
93809380
max_logit = std::max(max_logit, candidates->data[i].logit);
93819381
}
9382-
const float min_logit = max_logit + logf(p); // min logit for p_i >= p * p_max
9382+
const float min_logit = max_logit + logf(p) * pow; // min logit for p_i >= p * p_max^pow
93839383

93849384
for (size_t i = 0; i < candidates->size; ++i) {
93859385
if (candidates->data[i].logit >= min_logit) {
@@ -9405,7 +9405,7 @@ void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * can
94059405
candidates->sorted = true;
94069406
}
94079407

9408-
const float min_logit = candidates->data[0].logit + logf(p); // min logit for p_i >= p * p_max
9408+
const float min_logit = candidates->data[0].logit + logf(p) * pow; // min logit for p_i >= p * p_max^pow
94099409
size_t i = 1; // first token always matches
94109410

94119411
for (; i < candidates->size; ++i) {
@@ -9423,6 +9423,14 @@ void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * can
94239423
}
94249424
}
94259425

9426+
void llama_sample_min_p(struct llama_context * ctx, llama_token_data_array * candidates, float p, size_t min_keep) {
9427+
llama_sample_min_p_pow(ctx, candidates, p, 1.f, min_keep);
9428+
}
9429+
9430+
void llama_sample_top_a(struct llama_context * ctx, llama_token_data_array * candidates, float a, size_t min_keep) {
9431+
llama_sample_min_p_pow(ctx, candidates, a, 2.f, min_keep);
9432+
}
9433+
94269434
void llama_sample_tail_free(struct llama_context * ctx, llama_token_data_array * candidates, float z, size_t min_keep) {
94279435
if (z >= 1.0f || candidates->size <= 2) {
94289436
return;

‎llama.h

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -798,6 +798,13 @@ extern "C" {
798798
float p,
799799
size_t min_keep);
800800

801+
/// @details Top-A sampling as described in https://github.com/BlinkDL/RWKV-LM/tree/4cb363e5aa31978d801a47bc89d28e927ab6912e#the-top-a-sampling-method
802+
LLAMA_API void llama_sample_top_a(
803+
struct llama_context * ctx,
804+
llama_token_data_array * candidates,
805+
float a,
806+
size_t min_keep);
807+
801808
/// @details Tail Free Sampling described in https://www.trentonbricken.com/Tail-Free-Sampling/.
802809
LLAMA_API void llama_sample_tail_free(
803810
struct llama_context * ctx,

0 commit comments

Comments
 (0)