File size: 3,464 Bytes
b79c056 5acee80 b79c056 0ab3e57 b79c056 0ab3e57 d6e90a6 b79c056 9aa7185 b79c056 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 |
# -*- coding: utf-8 -*-
import gradio as gr
import operator
import torch
import os
from transformers import BertTokenizer, BertForMaskedLM
# 使用私有模型和分詞器
model_name_or_path = "DeepLearning101/Corrector101zhTW"
# auth_token = os.getenv("Corrector101zhTW") # 從環境變量中獲取 token
# tokenizer = BertTokenizer.from_pretrained(model_name_or_path, use_auth_token=auth_token)
# model = BertForMaskedLM.from_pretrained(model_name_or_path, use_auth_token=auth_token)
# tokenizer = BertTokenizer.from_pretrained(model_name_or_path)
# model = BertForMaskedLM.from_pretrained(model_name_or_path)
model_name_or_path = "DeepLearning101/Corrector101zhTW"
try:
tokenizer = BertTokenizer.from_pretrained(model_name_or_path)
model = BertForMaskedLM.from_pretrained(model_name_or_path)
except Exception as e:
def ai_text(text):
with torch.no_grad():
outputs = model(**tokenizer([text], padding=True, return_tensors='pt'))
def to_highlight(corrected_sent, errs):
output = [{"entity": "糾錯", "word": err[1], "start": err[2], "end": err[3]} for i, err in
enumerate(errs)]
return {"text": corrected_sent, "entities": output}
def get_errors(corrected_text, origin_text):
sub_details = []
for i, ori_char in enumerate(origin_text):
if ori_char in [' ', '“', '”', '‘', '’', '琊', '\n', '…', '—', '擤']:
# add unk word
corrected_text = corrected_text[:i] + ori_char + corrected_text[i:]
continue
if i >= len(corrected_text):
continue
if ori_char != corrected_text[i]:
if ori_char.lower() == corrected_text[i]:
# pass english upper char
corrected_text = corrected_text[:i] + ori_char + corrected_text[i + 1:]
continue
sub_details.append((ori_char, corrected_text[i], i, i + 1))
sub_details = sorted(sub_details, key=operator.itemgetter(2))
return corrected_text, sub_details
_text = tokenizer.decode(torch.argmax(outputs.logits[0], dim=-1), skip_special_tokens=True).replace(' ', '')
corrected_text = _text[:len(text)]
corrected_text, details = get_errors(corrected_text, text)
print(text, ' => ', corrected_text, details)
return corrected_text + ' ' + str(details)
if __name__ == '__main__':
examples = [
['你究輸入利的手機門號跟生分證就可以了。'],
['這裡是客服中新，很高性為您服物，請問金天有什麼須要幫忙'],
['因為我們這邊是按天術比例計蒜給您的，其實不會有態大的穎響。也就是您用前面的資非的廢率來做計算'],
['我來看以下，他的時價是多少？起實您就可以直皆就不用到門事'],
['因為你現在月富是六九九嘛，我幫擬減衣百塊，兒且也不會江速'],
]
inputs=[gr.Textbox(lines=2, label="欲校正的文字")],
outputs=[gr.Textbox(lines=2, label="修正後的文字")],
gr.Interface(
inputs='text',
outputs='text',
title="客服ASR文本AI糾錯系統",
description="""
<a href="https://www.twman.org" target='_blank'>TonTon Huang Ph.D. @ 2024/04 </a><br>
輸入ASR文本，糾正同音字/詞錯誤<br>
Masked Language Model (MLM) as correction BERT
""", examples=examples
).launch() |