|
|
|
OVERALL_INFO = ["Model", "Overall"] |
|
|
|
risk_topic_1_columns = [ |
|
"Crimes and Illegal Activities", |
|
"Cybersecurity", |
|
"Data Privacy", |
|
"Ethics and Morality", |
|
"Physical and Mental Health", |
|
"Hate Speech", |
|
"Extremism", |
|
"Inappropriate Suggestions" |
|
] |
|
risk_topic_1_columns = [item.lower() for item in risk_topic_1_columns] |
|
|
|
attack_columns = [ |
|
"Adaptive Attack", |
|
"Positive Induction", |
|
"Reverse Induction", |
|
"Code Injection", |
|
"Instruction Jailbreak", |
|
"Goal Hijacking", |
|
"Instruction Encryption", |
|
"DeepInception", |
|
"In-Context Attack", |
|
"Chain of Utterances", |
|
"Compositional Instructions" |
|
] |
|
attack_columns = [item.lower() for item in attack_columns] |
|
|
|
XLSX_DIR = "./file//results.xlsx" |
|
|
|
LEADERBOARD_INTRODUCTION = """# 🏆 S-Eval Leaderboard |
|
## 🔔 Updates |
|
📣 [2025/03/30]: 🎉 Our paper has been accepted by ISSTA 2025. To meet evaluation needs under different budgets, we partition the benchmark into four scales: [Small](https://github.com/IS2Lab/S-Eval/tree/main/s_eval/small) (1,000 Base and 10,000 Attack in each language), [Medium](https://github.com/IS2Lab/S-Eval/tree/main/s_eval/medium) (3,000 Base and 30,000 Attack in each language), [Large](https://github.com/IS2Lab/S-Eval/tree/main/s_eval/large) (5,000 Base and 50,000 Attack in each language) and [Full](https://github.com/IS2Lab/S-Eval/tree/main/s_eval/full) (10,000 Base and 100,000 Attack in each language), comprehensively considering the balance and harmfulness of data. |
|
|
|
📣 [2024/10/25]: We release all 20,000 base risk prompts and 200,000 corresponding attack prompts ([Version-0.1.2](https://github.com/IS2Lab/S-Eval)). We also update [🏆 LeaderBoard](https://huggingface.co/spaces/IS2Lab/S-Eval) with new evaluation results including GPT-4 and other models. |
|
🎉 S-Eval has achieved about **7,000** total views and about **2,000** total downloads across multiple platforms. 🎉 |
|
|
|
📣 [2024/06/17]: We further release 10,000 base risk prompts and 100,000 corresponding attack prompts ([Version-0.1.1](https://github.com/IS2Lab/S-Eval)). If you require automatic safety evaluations, please feel free to submit a request via [Issues](https://huggingface.co/spaces/IS2Lab/S-Eval/discussions) or contact us by [Email](mailto:[email protected]). |
|
|
|
📣 [2024/05/31]: We release 20,000 corresponding attack prompts. |
|
|
|
📣 [2024/05/23]: We publish our [paper](https://arxiv.org/abs/2405.14191) and first release 2,000 base risk prompts. You can download the benchmark from our [project](https://github.com/IS2Lab/S-Eval), the [HuggingFace Dataset](https://huggingface.co/datasets/IS2Lab/S-Eval). |
|
|
|
### ❗️ Note |
|
Due to the limited machine resource, please refresh the page if a connection timeout error occurs. |
|
|
|
You can get more detailed information from our [Project](https://github.com/IS2Lab/S-Eval) and [Paper](https://arxiv.org/abs/2405.14191). |
|
""" |
|
|
|
SELECT_SET_INTRO = ( |
|
"Select whether Chinese or English results should be shown." |
|
) |
|
|
|
TABLE_INTRODUCTION_1 = """In the table below, we summarize the safety scores (%) of differnet models on Base Risk Prompt Set.""" |
|
TABLE_INTRODUCTION_2 = """In the table below, we summarize the attack success rates (%) of the instruction attacks in Attack Prompt Set on different models""" |
|
|
|
|
|
LEADERBORAD_INFO = """ |
|
S-Eval is designed to be a new comprehensive, multi-dimensional and open-ended safety evaluation benchmark. So far, S-Eval has 220,000 evaluation prompts in total (and is still in active expansion), including 20,000 base risk prompts (10,000 in Chinese and 10,000 in English) and 200,000 *corresponding* attack prompts derived from 10 popular adversarial instruction attacks. These test prompts are generated based on a comprehensive and unified risk taxonomy, specifically designed to encompass all crucial dimensions of LLM safety evaluation and meant to accurately reflect the varied safety levels of LLMs across these risk dimensions. |
|
More details on the construction of the test suite including model-based test generation, selection and the expert critique LLM can be found in our [paper](https://arxiv.org/abs/2405.14191). |
|
""" |
|
|
|
|
|
CITATION_BUTTON_LABEL = "If our work is useful for your own, you can cite us with the following BibTex entry:" |
|
|
|
CITATION_BUTTON_TEXT = r""" |
|
@article{yuan2024seval, |
|
title={S-Eval: Towards Automated and Comprehensive Safety Evaluation for Large Language Models}, |
|
author={Xiaohan Yuan and Jinfeng Li and Dongxia Wang and Yuefeng Chen and Xiaofeng Mao and Longtao Huang and Jialuo Chen and Hui Xue and Xiaoxia Liu and Wenhai Wang and Kui Ren and Jingyi Wang}, |
|
journal={arXiv preprint arXiv:2405.14191}, |
|
year={2024} |
|
} |
|
""" |
|
|
