Spaces:
Running
on
Zero
Running
on
Zero
File size: 1,984 Bytes
b96e750 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 |
# Copyright (c) 2024 Amphion.
#
# This source code is licensed under the MIT license found in the
# LICENSE file in the root directory of this source tree.
import re
"""
Text clean time
"""
rep_map = {
"：": ",",
"；": ",",
"，": ",",
"。": ".",
"！": "!",
"？": "?",
"\n": ".",
"·": ",",
"、": ",",
"...": ".",
"…": ".",
"$": ".",
"“": "",
"”": "",
"‘": "",
"’": "",
"（": "",
"）": "",
"(": "",
")": "",
"《": "",
"》": "",
"【": "",
"】": "",
"[": "",
"]": "",
"—": "",
"～": "-",
"~": "-",
"「": "",
"」": "",
"¿": "",
"¡": "",
}
def collapse_whitespace(text):
# Regular expression matching whitespace:
_whitespace_re = re.compile(r"\s+")
return re.sub(_whitespace_re, " ", text).strip()
def remove_punctuation_at_begin(text):
return re.sub(r"^[,.!?]+", "", text)
def remove_aux_symbols(text):
text = re.sub(r"[\<\>\(\)\[\]\"\«\»]+", "", text)
return text
def replace_symbols(text):
text = text.replace(";", ",")
text = text.replace("-", " ")
text = text.replace(":", ",")
return text
def replace_punctuation(text):
pattern = re.compile("|".join(re.escape(p) for p in rep_map.keys()))
replaced_text = pattern.sub(lambda x: rep_map[x.group()], text)
return replaced_text
def text_normalize(text):
text = replace_punctuation(text)
text = replace_symbols(text)
text = remove_aux_symbols(text)
text = remove_punctuation_at_begin(text)
text = collapse_whitespace(text)
text = re.sub(r"([^\.,!\?\-…])$", r"\1", text)
return text
def german_to_ipa(text, text_tokenizer):
if type(text) == str:
text = text_normalize(text)
phonemes = text_tokenizer(text)
return phonemes
else:
for i, t in enumerate(text):
text[i] = text_normalize(t)
return text_tokenizer(text)
|