few shot 评测基准

[Dyw08]

您的工作太优秀了，但是我有个问题，就是有一些基准是使用few shot的，例如，mmlu，mmlu-pro 这类基准，我使用本地下载的数据集，那我怎么确定我是否使用上了few shot呢？期待您的回复。
这是我输出的log文件。
2025-10-15 08:07:11,449 - evalscope - INFO - Dump task config to ./outputs/20251015_080701/configs/task_config_283902.yaml
2025-10-15 08:07:11,512 - evalscope - INFO - {
"model": "/20251001-SFT-stage-1-3B/checkpoint-120000",
"model_id": "checkpoint-120000",
"model_args": {},
"model_task": "text_generation",
"template_type": null,
"chat_template": null,
"datasets": [
"mmlu_pro"
],
"dataset_args": {
"mmlu_pro": {
"name": "mmlu_pro",
"dataset_id": "/data0/eval/MMLU-Pro-main/MMLU-Pro-data/data_my",
"model_adapter": "generation",
"output_types": [
"multiple_choice_logits",
"generation"
],
"subset_list": [
"computer science",
"math",
"chemistry",
"engineering",
"law",
"biology",
"health",
"physics",
"business",
"philosophy",
"economics",
"other",
"psychology",
"history"
],
"metric_list": [
"AverageAccuracy"
],
"few_shot_num": 5,
"few_shot_random": false,
"train_split": "validation",
"eval_split": "test",
"prompt_template": "The following are multiple choice questions (with answers) about {subset_name}. Think step by step and then finish your answer with "the answer is (X)" where X is the correct letter choice.\n{query}",
"system_prompt": null,
"query_template": null,
"pretty_name": "MMLU-Pro",
"description": "MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options.",
"tags": [
"MCQ",
"Knowledge"
],
"filters": {
"remove_until": ""
},
"extra_params": {}
}
},
"dataset_dir": "/root/.cache/modelscope/hub/datasets",
"dataset_hub": "modelscope",
"generation_config": {
"max_tokens": 4096,
"temperature": 0,
"n": 1
},
"eval_type": "service",
"eval_backend": "Native",
"eval_config": null,
"stage": "all",
"limit": null,
"eval_batch_size": 64,
"mem_cache": false,
"use_cache": null,
"work_dir": "./outputs/20251015_080701",
"outputs": null,
"ignore_errors": false,
"debug": false,
"dry_run": false,
"seed": 42,
"api_url": "http://localhost:8801/v1/chat/completions",
"api_key": "EMPTY",
"timeout": 60000,
"stream": true,
"judge_strategy": "auto",
"judge_worker_num": 1,
"judge_model_args": {},
"analysis_report": false
}
2025-10-15 08:07:11,512 - evalscope - INFO - Start evaluating on dataset /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my
2025-10-15 08:07:11,540 - evalscope - INFO - Loading dataset from local disk: /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my
2025-10-15 08:07:11,983 - evalscope - INFO - Loading dataset: dataset_name: /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my > subsets: ['default']
2025-10-15 08:07:15,212 - evalscope - INFO - Use settings: > few_shot_num: 5, > few_shot_split: validation, > target_eval_split: test

我怎么确定使用了few shot呢？

[Yunnglin]

这个打印的config里面有配置，你看这里：

"metric_list": [
"AverageAccuracy"
],
"few_shot_num": 5, # 这里
"few_shot_random": false,
"train_split": "validation",

[Dyw08]

那这个配置是使用了few shot还是没用使用呢？

[Dyw08]

我输入的数据只传test文件，他的喝这个confog还是这样的
2025-10-15 09:26:27,170 - evalscope - INFO - Loading model /data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000 ...
2025-10-15 09:26:44,608 - evalscope - WARNING - Got local model dir: /data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000
2025-10-15 09:26:44,609 - evalscope - INFO - Updating generation config ...
2025-10-15 09:26:44,885 - evalscope - INFO - Dump task config to ./outputs/20251015_092621/configs/task_config_692153.yaml
2025-10-15 09:26:44,941 - evalscope - INFO - {
"model": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000",
"model_id": "checkpoint-120000",
"model_args": {
"revision": "master",
"precision": "torch.float16"
},
"model_task": "text_generation",
"template_type": null,
"chat_template": null,
"datasets": [
"mmlu_pro"
],
"dataset_args": {
"mmlu_pro": {
"name": "mmlu_pro",
"dataset_id": "/data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my/test-00000-of-00001.json",
"model_adapter": "generation",
"output_types": [
"multiple_choice_logits",
"generation"
],
"subset_list": [
"computer science",
"math",
"chemistry",
"engineering",
"law",
"biology",
"health",
"physics",
"business",
"philosophy",
"economics",
"other",
"psychology",
"history"
],
"metric_list": [
"AverageAccuracy"
],
"few_shot_num": 5,
"few_shot_random": false,
"train_split": "validation",
"eval_split": "test",
"prompt_template": "The following are multiple choice questions (with answers) about {subset_name}. Think step by step and then finish your answer with "the answer is (X)" where X is the correct letter choice.\n{query}",
"system_prompt": null,
"query_template": null,
"pretty_name": "MMLU-Pro",
"description": "MMLU-Pro is a benchmark for evaluating language models on multiple-choice questions across various subjects. It includes questions from different domains, where the model must select the correct answer from given options.",
"tags": [
"MCQ",
"Knowledge"
],
"filters": {
"remove_until": ""
},
"extra_params": {}
}
},
"dataset_dir": "/root/.cache/modelscope/hub/datasets",
"dataset_hub": "modelscope",
"generation_config": {
"max_tokens": 4096,
"temperature": 0,
"n": 1
},
"eval_type": "checkpoint",
"eval_backend": "Native",
"eval_config": null,
"stage": "all",
"limit": null,
"eval_batch_size": 64,
"mem_cache": false,
"use_cache": null,
"work_dir": "./outputs/20251015_092621",
"outputs": null,
"ignore_errors": false,
"debug": false,
"dry_run": false,
"seed": 42,
"api_url": null,
"api_key": "EMPTY",
"timeout": 60000,
"stream": true,
"judge_strategy": "auto",
"judge_worker_num": 1,
"judge_model_args": {},
"analysis_report": false
}
2025-10-15 09:26:44,942 - evalscope - INFO - Start evaluating on dataset /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my/test-00000-of-00001.json
2025-10-15 09:26:44,997 - evalscope - INFO - Loading dataset from local disk: /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my/test-00000-of-00001.json
2025-10-15 09:26:45,371 - evalscope - INFO - Loading dataset: dataset_name: /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my/test-00000-of-00001.json > subsets: ['default']
2025-10-15 09:26:49,822 - evalscope - INFO - Use settings: > few_shot_num: 5, > few_shot_split: validation, > target_eval_split: test
按道理来说，这个只传了test文件，不应该是使用few shot

[Yunnglin]

看代码，你使用的不是最新版本，建议更新evalscope到1.1.0版本。新版本加上--debug参数可以看到请求的prompt内容，从而确定few shot是否生效；此外新版本如果few shot设置不正确会报错。

[Dyw08]

是的，新版本的config文件中 输出
2025-10-15 13:05:36 - evalscope - INFO: Loading dataset /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my from local > subset: default > split: test ...
2025-10-15 13:05:38 - evalscope - INFO: Loading dataset /data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my from local > subset: default > split: validation ... 确实使用了few shot。
但是我想问下 这个旧版本的few shot怎么配置才能使用呢？因为旧版本的生成回复的jsonl文件里面有 [{"finish_reason": "length", 有这个字段，说明模型输出截断了，我想保留这个字段。但是新版本中这个字段去掉了，期待您的回复！

[Yunnglin]

新版的结果中也有 "stop_reason": "stop" 字段，你可以看一下

[Dyw08]

我看到了 但是我有个问题，我的max_tokens: 32176 已经设置成32k，但是我的结果中还是有很多的"stop_reason": "max_tokens" ，这是什么原因呢？

[Yunnglin]

你可以先看下模型输出是否真的有32k这么长，如果不是的话，请提供一下完整的评测命令，会进一步看看是否有bug

[Dyw08]

{"id": "a5499130", "content": 。。。。。。。。Perhaps I need to use the exact fraction.\n\nFrom earlier:\n\ne", "source": "generate", "metadata": null, "internal": null, "role": "assistant", "tool_calls": null, "model": "/checkpoint-120000"}, "stop_reason": "max_tokens", "logprobs": null}], "usage": {"input_tokens": 1491, "output_tokens": 2048, "total_tokens": 3539, "input_tokens_cache_write": null, "input_tokens_cache_read": null, "reasoning_tokens": null}, "time": null, "metadata": null, "error": null},
这是我结果的某条数据，输出突然截断，这个total_tokens 显示才3539，但是我评测命令的cogfig文件，

• mmlu_pro
  debug: false
  eval_backend: Native
  eval_batch_size: 64
  eval_config: null
  eval_type: openai_api
  generation_config:
  batch_size: 64
  max_tokens: 32176
  n: 1
  stream: true
  temperature: 0.0
  timeout: 60000
  ignore_errors: false
  judge_model_args: {}
  judge_strategy: auto
  judge_worker_num: 1
  limit: null 评测中使用的32k，这是什么原因呢？

[Yunnglin]

1. 你使用的模型API是什么，是完全兼容openqi api格式的吗
2. 请提供一下完整评测命令
   我这边复现不了这个问题

[Dyw08]

我使用了的一个qwen2.5 经过sft的一个模型，完全兼容openai api格式，
评测的脚本
task_cfg = TaskConfig(
model="/checkpoint-120000",
#api_key='EMPTY',
eval_type=EvalType.SERVICE,
api_url='http://localhost:8802/v1/chat/completions', # vLLM服务地址
datasets=[
'mmlu_pro',
],
dataset_args={
'mmlu_pro': {
"local_path": "/data0/train-aitech/projects/yingwei5/eval/MMLU-Pro-main/MMLU-Pro-data/data_my",
'filters': {'remove_until': ''} # 思考模式，评测的时候过滤掉思考的内容
}
},
eval_batch_size=64,
generation_config={
'max_tokens': 32176,
'temperature': 0,
'n': 1,
},
timeout=60000,
stream=True,
)
vllm 启动命令 python -m vllm.entrypoints.openai.api_server
--host 0.0.0.0
--port 8801
--model /checkpoint-120000
--tensor-parallel-size 2
--trust-remote-code 您看这配置有啥问题吗

[Dyw08]

我发现ifeval 这个基准也是，设置的32k，total_tokens 不到32k就截断了，"stop_reason": "max_tokens"而且也是这个原因

[Yunnglin]

1. 从这个config看起来没有问题，你贴一下具体的log，例如outputs/20251xxx/logs/eval_log.log
2. 你观察一下vLLM中打印的log，看接受到的请求是否包含'max_tokens': 32176这个参数，如果有这个参数那可能是模型服务配置有问题；如果这个值是2048，需要再排查evalscope的问题

[Dyw08]

您是在太厉害了，确实是vllm启动配置有问题，实在太感谢了！
但是我又遇到了另一个问题就是，我只使用单卡的时候，没有启动vllm，直接运行脚本，脚本max token还是32k，"finish_reason": "stop" 显示停止了，但是我的输出语句还是截断了。这是这条数据：
[{"index": 0, "message": {"role": "assistant", "content": "First,。。。。。。。That must be a typo. Perhaps (H) is WPA3.\n\nIn the user message, it says: "(H): WPA2" but that can't be right because (G) is WPA2. Let's check the original text.\n\nUser said: "Options are: (A): WPA (B): WPA2-Enterprise (C): WEP (D): WEP+ (E): WPA-PSK (F): WEP2 ("}, "finish_reason": "stop"}], "created": 1760608087, "usage": {"prompt_tokens": 1469, "completion_tokens": 2048, "total_tokens": 3517}, "model_spec": {"model_id": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000", "device_map": "cuda", "torch_dtype": "auto"}, "answer_id": "answer-a3004f0cf7e03328a0b39a73c27467aa", "subset_name": "computer science", "raw_input": {"question_id": 10368, "question": "Which among them has the strongest wireless security?", "options": ["WPA", "WPA2-Enterprise", "WEP", "WEP+", "WPA-PSK", "WEP2", "WPA2", "WPA3", "WPA4", "WPA2-PSK"], "answer": "H", "answer_index": 7, "cot_content": "", "category": "computer science", "src": "ori_mmlu-computer_security"}, "index": 12}
但是我启动vllm服务，使用2卡进行评测，这个数据输出就是正常的，
"choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "First, the question。。。。。。。is stronger, it is not yet a standard in most environments, so WPA3 is the most appropriate choice for the strongest security available.\n\nAnswer: (H)", "role": "assistant"}}], "created": 1760610485, "model": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000", "object": "chat.completion", "usage": {"completion_tokens": 2726, "prompt_tokens": 1469, "total_tokens": 4195}, "model_spec": {"api_url": "http://localhost:8802/v1/chat/completions", "model_id": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000", "api_key": "EMPTY"}, "answer_id": "answer-be678307953b66a376c12c0112d90ff8", "subset_name": "computer science", "raw_input": {"question_id": 10368, "question": "Which among them has the strongest wireless security?", "options": ["WPA", "WPA2-Enterprise", "WEP", "WEP+", "WPA-PSK", "WEP2", "WPA2", "WPA3", "WPA4", "WPA2-PSK"], "answer": "H", "answer_index": 7, "cot_content": "", "category": "computer science", "src": "ori_mmlu-computer_security"}, "index": 12}
期待您的回复！

[Yunnglin]

您是在太厉害了，确实是vllm启动配置有问题，实在太感谢了！ 但是我又遇到了另一个问题就是，我只使用单卡的时候，没有启动vllm，直接运行脚本，脚本max token还是32k，"finish_reason": "stop" 显示停止了，但是我的输出语句还是截断了。这是这条数据： [{"index": 0, "message": {"role": "assistant", "content": "First,。。。。。。。That must be a typo. Perhaps (H) is WPA3.\n\nIn the user message, it says: "(H): WPA2" but that can't be right because (G) is WPA2. Let's check the original text.\n\nUser said: "Options are: (A): WPA (B): WPA2-Enterprise (C): WEP (D): WEP+ (E): WPA-PSK (F): WEP2 ("}, "finish_reason": "stop"}], "created": 1760608087, "usage": {"prompt_tokens": 1469, "completion_tokens": 2048, "total_tokens": 3517}, "model_spec": {"model_id": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000", "device_map": "cuda", "torch_dtype": "auto"}, "answer_id": "answer-a3004f0cf7e03328a0b39a73c27467aa", "subset_name": "computer science", "raw_input": {"question_id": 10368, "question": "Which among them has the strongest wireless security?", "options": ["WPA", "WPA2-Enterprise", "WEP", "WEP+", "WPA-PSK", "WEP2", "WPA2", "WPA3", "WPA4", "WPA2-PSK"], "answer": "H", "answer_index": 7, "cot_content": "", "category": "computer science", "src": "ori_mmlu-computer_security"}, "index": 12} 但是我启动vllm服务，使用2卡进行评测，这个数据输出就是正常的， "choices": [{"finish_reason": "stop", "index": 0, "message": {"content": "First, the question。。。。。。。is stronger, it is not yet a standard in most environments, so WPA3 is the most appropriate choice for the strongest security available.\n\nAnswer: (H)", "role": "assistant"}}], "created": 1760610485, "model": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000", "object": "chat.completion", "usage": {"completion_tokens": 2726, "prompt_tokens": 1469, "total_tokens": 4195}, "model_spec": {"api_url": "http://localhost:8802/v1/chat/completions", "model_id": "/data0/train-aitech/projects/xusen1/LLaMA-Factory/exps-20251001-SFT-stage-1-3B/ckpts/20251001-SFT-stage-1-3B/checkpoint-120000", "api_key": "EMPTY"}, "answer_id": "answer-be678307953b66a376c12c0112d90ff8", "subset_name": "computer science", "raw_input": {"question_id": 10368, "question": "Which among them has the strongest wireless security?", "options": ["WPA", "WPA2-Enterprise", "WEP", "WEP+", "WPA-PSK", "WEP2", "WPA2", "WPA3", "WPA4", "WPA2-PSK"], "answer": "H", "answer_index": 7, "cot_content": "", "category": "computer science", "src": "ori_mmlu-computer_security"}, "index": 12} 期待您的回复！

你这里贴的内容不是新版evalscope的结果，使用最新版再试试呢