-
Notifications
You must be signed in to change notification settings - Fork 553
/
Copy patheval_modelscope_datasets.py
112 lines (110 loc) · 6.33 KB
/
eval_modelscope_datasets.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
# export DATASET_SOURCE='ModelScope' # before run this script
from datasets import Dataset, DatasetDict
from mmengine.config import read_base
from tqdm import tqdm
with read_base():
from opencompass.configs.datasets.agieval.agieval_gen import \
agieval_datasets as agieval_v2_datasets # ok
from opencompass.configs.datasets.agieval.agieval_gen_a0c741 import \
agieval_datasets as agieval_v1_datasets # ok
from opencompass.configs.datasets.ARC_c.ARC_c_clean_ppl import \
ARC_c_datasets as ARC_c_clean_datasets # ok
from opencompass.configs.datasets.ARC_c.ARC_c_gen import \
ARC_c_datasets # ok
from opencompass.configs.datasets.ARC_e.ARC_e_gen import \
ARC_e_datasets # ok
from opencompass.configs.datasets.bbh.bbh_gen import bbh_datasets
from opencompass.configs.datasets.ceval.ceval_clean_ppl import \
ceval_datasets as ceval_clean_datasets # ok
from opencompass.configs.datasets.ceval.ceval_gen import \
ceval_datasets # ok
from opencompass.configs.datasets.CLUE_afqmc.CLUE_afqmc_gen import \
afqmc_datasets # ok
from opencompass.configs.datasets.CLUE_cmnli.CLUE_cmnli_gen import \
cmnli_datasets # ok
from opencompass.configs.datasets.CLUE_cmnli.CLUE_cmnli_ppl import \
cmnli_datasets as cmnli_ppl_datasets # ok
from opencompass.configs.datasets.CLUE_ocnli.CLUE_ocnli_gen import \
ocnli_datasets # ok
from opencompass.configs.datasets.cmmlu.cmmlu_gen import \
cmmlu_datasets # ok
from opencompass.configs.datasets.commonsenseqa.commonsenseqa_gen import \
commonsenseqa_datasets # 额外处理gpt
from opencompass.configs.datasets.GaokaoBench.GaokaoBench_gen import \
GaokaoBench_datasets # ok
from opencompass.configs.datasets.GaokaoBench.GaokaoBench_mixed import \
GaokaoBench_datasets as GaokaoBench_mixed_datasets # ok
from opencompass.configs.datasets.GaokaoBench.GaokaoBench_no_subjective_gen_4c31db import \
GaokaoBench_datasets as GaokaoBench_no_subjective_datasets # ok
from opencompass.configs.datasets.gsm8k.gsm8k_gen import \
gsm8k_datasets # ok
from opencompass.configs.datasets.hellaswag.hellaswag_10shot_gen_e42710 import \
hellaswag_datasets as hellaswag_ice_datasets # ok
from opencompass.configs.datasets.hellaswag.hellaswag_clean_ppl import \
hellaswag_datasets as hellaswag_clean_datasets # ok
from opencompass.configs.datasets.hellaswag.hellaswag_gen import \
hellaswag_datasets as hellaswag_v2_datasets # ok
from opencompass.configs.datasets.hellaswag.hellaswag_ppl_9dbb12 import \
hellaswag_datasets as hellaswag_v1_datasets # ok
from opencompass.configs.datasets.hellaswag.hellaswag_ppl_a6e128 import \
hellaswag_datasets as hellaswag_v3_datasets # ok
from opencompass.configs.datasets.humaneval.humaneval_gen import \
humaneval_datasets # ok
from opencompass.configs.datasets.humaneval.humaneval_repeat10_gen_8e312c import \
humaneval_datasets as humaneval_repeat10_datasets # ok
from opencompass.configs.datasets.lambada.lambada_gen import \
lambada_datasets # ok
from opencompass.configs.datasets.lcsts.lcsts_gen import \
lcsts_datasets # ok
from opencompass.configs.datasets.math.math_gen import math_datasets # ok
from opencompass.configs.datasets.mbpp.mbpp_gen import \
mbpp_datasets as mbpp_v1_datasets # ok
from opencompass.configs.datasets.mbpp.mbpp_passk_gen_830460 import \
mbpp_datasets as mbpp_v2_datasets # ok
from opencompass.configs.datasets.mbpp.sanitized_mbpp_gen_830460 import \
sanitized_mbpp_datasets # ok
from opencompass.configs.datasets.mmlu.mmlu_clean_ppl import \
mmlu_datasets as mmlu_clean_datasets # ok
from opencompass.configs.datasets.mmlu.mmlu_gen import mmlu_datasets # ok
from opencompass.configs.datasets.nq.nq_gen import nq_datasets # ok
from opencompass.configs.datasets.obqa.obqa_gen import obqa_datasets # ok
from opencompass.configs.datasets.obqa.obqa_ppl_6aac9e import \
obqa_datasets as obqa_ppl_datasets # ok
from opencompass.configs.datasets.piqa.piqa_gen import \
piqa_datasets as piqa_v2_datasets # ok
from opencompass.configs.datasets.piqa.piqa_ppl import \
piqa_datasets as piqa_v1_datasets # ok
from opencompass.configs.datasets.piqa.piqa_ppl_0cfff2 import \
piqa_datasets as piqa_v3_datasets # ok
from opencompass.configs.datasets.race.race_ppl import race_datasets # ok
from opencompass.configs.datasets.siqa.siqa_gen import \
siqa_datasets as siqa_v2_datasets # ok
from opencompass.configs.datasets.siqa.siqa_gen_18632c import \
siqa_datasets as siqa_v3_datasets # ok
from opencompass.configs.datasets.siqa.siqa_ppl_42bc6e import \
siqa_datasets as siqa_ppl_datasets # ok
from opencompass.configs.datasets.storycloze.storycloze_gen import \
storycloze_datasets # ok
from opencompass.configs.datasets.storycloze.storycloze_ppl import \
storycloze_datasets as storycloze_ppl_datasets # ok
from opencompass.configs.datasets.strategyqa.strategyqa_gen import \
strategyqa_datasets
from opencompass.configs.datasets.summedits.summedits_gen import \
summedits_datasets as summedits_v2_datasets # ok
from opencompass.configs.datasets.triviaqa.triviaqa_gen import \
triviaqa_datasets # ok
from opencompass.configs.datasets.triviaqa.triviaqa_wiki_1shot_gen_20a989 import \
triviaqa_datasets as triviaqa_wiki_1shot_datasets # ok
from opencompass.configs.datasets.tydiqa.tydiqa_gen import \
tydiqa_datasets # ok
from opencompass.configs.datasets.winogrande.winogrande_5shot_ll_252f01 import \
winogrande_datasets as winogrande_5shot_ll_datasets # ok
from opencompass.configs.datasets.winogrande.winogrande_gen import \
winogrande_datasets
from opencompass.configs.datasets.winogrande.winogrande_ll import \
winogrande_datasets as winogrande_ll_datasets # ok
from opencompass.configs.datasets.Xsum.Xsum_gen import Xsum_datasets
from opencompass.configs.models.opt.hf_opt_125m import models
datasets = sum((v for k, v in locals().items() if k.endswith('_datasets')), [])
for d in datasets:
d['reader_cfg'].update({'train_range': '[0:5]', 'test_range': '[0:5]'})