|
|
--- |
|
|
license: mit |
|
|
language: |
|
|
- en |
|
|
pipeline_tag: text-classification |
|
|
library_name: fasttext |
|
|
tags: |
|
|
- code |
|
|
--- |
|
|
|
|
|
|
|
|
## Code classifier |
|
|
|
|
|
We use `code-classifier` to retrieve math-related content from `fineweb-edu`, `dclm`, ... to upsample code-related content |
|
|
|
|
|
```python |
|
|
import json |
|
|
import os |
|
|
import time |
|
|
from concurrent.futures import ProcessPoolExecutor, wait, ALL_COMPLETED |
|
|
from time import sleep |
|
|
|
|
|
import fasttext |
|
|
import numpy as np |
|
|
import pandas as pd |
|
|
import pyarrow.parquet as pq |
|
|
from tqdm import tqdm |
|
|
|
|
|
|
|
|
def print_error(value): |
|
|
print("error: ", value) |
|
|
|
|
|
|
|
|
def data_process(index, file, saved_dir): |
|
|
|
|
|
try: |
|
|
model_path = "code_classifier.bin" |
|
|
model = fasttext.load_model(model_path) |
|
|
|
|
|
# saved_dir: fineweb-edu/data/CC...-code |
|
|
filename = file.split('/')[-1].replace('.parquet', '.jsonl') |
|
|
path90 = os.path.join(saved_dir, "09_10", filename) |
|
|
if os.path.exists(path90): |
|
|
print("exist", path90, flush=True) |
|
|
return |
|
|
|
|
|
# avoid reading at the same time |
|
|
sleep(index * 3) |
|
|
os.makedirs(saved_dir, exist_ok=True) |
|
|
label_list = [] |
|
|
s67_list = [] |
|
|
s78_list = [] |
|
|
s89_list = [] |
|
|
s90_list = [] |
|
|
|
|
|
st = time.time() |
|
|
print("reading parquet", file, flush=True) |
|
|
df = pd.read_parquet(file) |
|
|
ed = time.time() |
|
|
print("read parquet time: ", ed - st, flush=True) |
|
|
for _, row_orginal in tqdm( |
|
|
df.iterrows(), |
|
|
total=len(df), |
|
|
position=index, |
|
|
desc=filename, |
|
|
): |
|
|
row = row_orginal.to_dict() |
|
|
text = row['text'].encode('unicode_escape').decode('utf-8') |
|
|
|
|
|
pred = model.predict(text) |
|
|
label, score = pred[0][0], pred[1][0] |
|
|
label_list.append(pred) |
|
|
if label == '__label__positive': |
|
|
if 0.6 <= score < 0.7: |
|
|
s67_list.append(row) |
|
|
elif 0.7 <= score < 0.8: |
|
|
s78_list.append(row) |
|
|
elif 0.8 <= score < 0.9: |
|
|
s89_list.append(row) |
|
|
elif 0.9 <= score <= 1.0: |
|
|
s90_list.append(row) |
|
|
else: |
|
|
continue |
|
|
except Exception as e: |
|
|
print_error(e) |
|
|
return None |
|
|
|
|
|
os.makedirs(os.path.join(saved_dir, "labeled"), exist_ok=True) |
|
|
|
|
|
print("writing to file", flush=True) |
|
|
|
|
|
with open( |
|
|
os.path.join(saved_dir, "labeled", |
|
|
filename.replace('.jsonl', '.txt')), 'w') as f: |
|
|
f.write("\n".join(str(pred) for pred in label_list)) |
|
|
|
|
|
for dir_name in ["06_07", "07_08", "08_09", "09_10"]: |
|
|
os.makedirs(os.path.join(saved_dir, dir_name), exist_ok=True) |
|
|
|
|
|
with open(os.path.join(saved_dir, "06_07", filename), 'w') as f: |
|
|
f.write("\n".join(json.dumps(line_now) for line_now in s67_list)) |
|
|
|
|
|
with open(os.path.join(saved_dir, "07_08", filename), 'w') as f: |
|
|
f.write("\n".join(json.dumps(line_now) for line_now in s78_list)) |
|
|
|
|
|
with open(os.path.join(saved_dir, "08_09", filename), 'w') as f: |
|
|
f.write("\n".join(json.dumps(line_now) for line_now in s89_list)) |
|
|
|
|
|
with open(os.path.join(saved_dir, "09_10", filename), 'w') as f: |
|
|
f.write("\n".join(json.dumps(line_now) for line_now in s90_list)) |
|
|
|
|
|
return None |
|
|
|
|
|
if __name__ == '__main__': |
|
|
|
|
|
num_process = 8 |
|
|
start_time = time.time() |
|
|
file_paths = [] |
|
|
base = "fineweb-edu" |
|
|
|
|
|
for file_name in [ |
|
|
'CC-MAIN-2020-16', 'CC-MAIN-2022-05', 'CC-MAIN-2022-40', |
|
|
'CC-MAIN-2020-24', 'CC-MAIN-2020-34', 'CC-MAIN-2021-39', |
|
|
'CC-MAIN-2020-29', 'CC-MAIN-2023-23', 'CC-MAIN-2022-49', |
|
|
'CC-MAIN-2021-43', 'CC-MAIN-2023-14', 'CC-MAIN-2020-50', |
|
|
'CC-MAIN-2021-25', 'CC-MAIN-2021-10', 'CC-MAIN-2021-49', |
|
|
'CC-MAIN-2020-40', 'CC-MAIN-2020-45', 'CC-MAIN-2022-21', |
|
|
'CC-MAIN-2021-04', 'CC-MAIN-2023-40', 'CC-MAIN-2021-17', |
|
|
'CC-MAIN-2021-21', 'CC-MAIN-2022-27', 'CC-MAIN-2022-33', |
|
|
'CC-MAIN-2023-06', 'CC-MAIN-2023-50', 'CC-MAIN-2020-05', |
|
|
'CC-MAIN-2021-31','CC-MAIN-2020-10' |
|
|
]: |
|
|
|
|
|
print("Walking:", file_name) |
|
|
original_file_path = base + file_name |
|
|
code_dir = original_file_path + "-code" |
|
|
for root, dirs, files in os.walk(original_file_path): |
|
|
for file in files: |
|
|
if file.endswith(".parquet"): # 只处理Parquet文件 |
|
|
file_path = os.path.abspath(os.path.join(root, file)) |
|
|
saved_dir = code_dir + "/" + file_path.split("/")[-1][:-8] |
|
|
file_paths.append((file_path, saved_dir)) |
|
|
|
|
|
print("total file paths", len(file_paths)) |
|
|
num_process = min(num_process, len(file_paths)) |
|
|
print("num_process", num_process) |
|
|
|
|
|
futures = [] |
|
|
with ProcessPoolExecutor(num_process) as executor: |
|
|
for index, (file_path, saved_dir) in enumerate(file_paths): |
|
|
futures.append( |
|
|
executor.submit(data_process, index % num_process, file_path, |
|
|
saved_dir)) |
|
|
done, not_done = wait(futures, return_when=ALL_COMPLETED) |
|
|
|
|
|
end_time = time.time() |
|
|
|
|
|
# 计算并打印所用时间 |
|
|
elapsed_time = end_time - start_time |
|
|
print(f"Time taken: {elapsed_time} seconds") |
|
|
print("=" * 100) |
|
|
``` |
|
|
|
|
|
|
|
|
## Related resources |
|
|
|
|
|
- [Math classifier](https://huggingface.co/yulan-team/math-classifier) |
|
|
- [Code classifier](https://huggingface.co/yulan-team/code-classifier) |
|
|
- [Reasoning classifier](https://huggingface.co/yulan-team/reasoning-classifier) |
|
|
|
|
|
|
|
|
--- |
|
|
|
|
|
## Contributing |
|
|
|
|
|
We welcome any form of contribution, including feedback on model bad cases, feature suggestions, and example contributions. You can do so by submitting an [issue](https://github.com/RUC-GSAI/YuLan-Mini/issues). |
|
|
|
|
|
## The Team |
|
|
|
|
|
YuLan-Mini is developed and maintained by [AI Box, Renmin University of China](http://aibox.ruc.edu.cn/). |
|
|
|
|
|
## License |
|
|
|
|
|
- The code in this repository, the model weights, and optimizer states are released under the [MIT License](./LICENSE). |
|
|
- Policies regarding the use of model weights, intermediate optimizer states, and training data will be announced in future updates. |
|
|
- Limitations: Despite our efforts to mitigate safety concerns and encourage the generation of ethical and lawful text, the probabilistic nature of language models may still lead to unexpected outputs. For instance, responses might contain bias, discrimination, or other harmful content. Please refrain from disseminating such content. We are not liable for any consequences arising from the spread of harmful information. |
|
|
|
|
|
## Citation |
|
|
|
|
|
If you find YuLan-Mini helpful for your research or development, please cite [our technical report](https://arxiv.org/abs/2412.17743): |
|
|
|
|
|
|
|
|
``` |
|
|
@article{hu2024yulan, |
|
|
title={YuLan-Mini: An Open Data-efficient Language Model}, |
|
|
author={Hu, Yiwen and Song, Huatong and Deng, Jia and Wang, Jiapeng and Chen, Jie and Zhou, Kun and Zhu, Yutao and Jiang, Jinhao and Dong, Zican and Zhao, Wayne Xin and others}, |
|
|
journal={arXiv preprint arXiv:2412.17743}, |
|
|
year={2024} |
|
|
} |
|
|
``` |
|
|
|
