File size: 2,213 Bytes
256a159 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 |
from ..utils.function_utils import compute_f1_two_sets
from ..utils.rc_f1 import CJRCEvaluator
"""
task: event detection
metric: F1 score
事件检测
"""
option_list = ["支付/给付", "欺骗", "搜查/扣押", "要求/请求", "卖出", "买入", "获利", "拘捕", "鉴定", "同意/接受", "供述", "联络", "帮助/救助", "租用/借用", "受伤", "伪造", "卖淫", "伤害人身", "赔偿", "归还/偿还"]
def compute_sjjc(data_dict):
"""
Compute the F1-score
The sjjc task covers 20 event types.
A question may involve one or more event types.
Given a list of event types from both the ground truth and the prediction, we compute the F1-score between
these two lists.
"""
score_list, abstentions = [], 0
for example in data_dict:
question, prediction, answer = example["origin_prompt"], example["prediction"], example["refr"]
answers = answer.split(";")
prediction_list =[]
for option in option_list:
if option in prediction:
prediction_list.append(option)
if len(prediction_list) == 0:
abstentions += 1
gt_set = set(answers)
pred_set = set(prediction_list)
score = compute_f1_two_sets(gt_set, pred_set)
score_list.append(score)
f1_score_average = sum(score_list) / len(score_list)
return {"score": f1_score_average, "abstention_rate": abstentions/len(data_dict)}
"""
task: trigger word extraction
metric: F1 score
触发词抽取
"""
def compute_cfcy(data_dict):
scores = 0
for example in data_dict:
question, prediction, answer = example["origin_prompt"], example["prediction"], example["refr"]
answers = answer.split(";")
predictions = prediction.split(";")
intersected = [CJRCEvaluator.compute_f1(r, h) for r, h in zip(answers, predictions)]
prec = sum(intersected) / len(predictions) if len(predictions) > 0 else 0
rec = sum(intersected) / len(answers) if len(answers) > 0 else 0
# print(prec, rec, intersected)
scores += 2 * prec * rec / (prec + rec + 1e-10)
f1_score_average = scores / len(data_dict)
return {"score": f1_score_average}
|