File size: 1,807 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 |
from opencompass.openicl.icl_prompt_template import PromptTemplate
from opencompass.openicl.icl_retriever import ZeroRetriever
from opencompass.openicl.icl_inferencer import PPLInferencer
from opencompass.openicl.icl_evaluator import AccEvaluator
from opencompass.datasets import AnliDataset
anli_datasets = []
for _split in ['R1', 'R2', 'R3']:
anli_reader_cfg = dict(
input_columns=["context", "hypothesis"],
output_column="label",
)
anli_infer_cfg = dict(
prompt_template=dict(
type=PromptTemplate,
template={
"A":
dict(round=[
dict(role="HUMAN", prompt="{context}\n{hypothesis}\What is the relation between the two sentences?"),
dict(role="BOT", prompt="Contradiction"),
]),
"B":
dict(round=[
dict(role="HUMAN", prompt="{context}\n{hypothesis}\What is the relation between the two sentences?"),
dict(role="BOT", prompt="Entailment"),
]),
"C":
dict(round=[
dict(role="HUMAN", prompt="{context}\n{hypothesis}\What is the relation between the two sentences?"),
dict(role="BOT", prompt="Neutral"),
]),
},
),
retriever=dict(type=ZeroRetriever),
inferencer=dict(type=PPLInferencer),
)
anli_eval_cfg = dict(evaluator=dict(type=AccEvaluator), )
anli_datasets.append(
dict(
type=AnliDataset,
abbr=f"anli-{_split}",
path=f"data/anli/anli_v1.0/{_split}/dev.jsonl",
reader_cfg=anli_reader_cfg,
infer_cfg=anli_infer_cfg,
eval_cfg=anli_eval_cfg,
)
)
|