Spaces:
Sleeping
Sleeping
File size: 1,870 Bytes
1ce1659 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 |
from transformers import pipeline
ner_pipeline = pipeline("ner")
def extract_entities(text):
output = ner_pipeline(text)
words = extract_words(output)
words = combine_subwords(words)
# extract word in each entity and assign to a list of entities, connect words if there is no space between them
entities = []
for entity in words:
if entity not in entities:
entities.append(entity)
return entities
def extract_words(entities):
"""
Extracts the words from a list of entities.
Args:
entities: A list of entities.
Returns:
A list of words extracted from the entities.
"""
words = []
for entity in entities:
words.append(entity["word"])
return words
def combine_subwords(word_list):
"""
Combines subwords (indicated by "##") with the preceding word in a list.
Args:
word_list: A list of words, where subwords are prefixed with "##".
Returns:
A new list with subwords combined with their preceding words.
"""
result = []
i = 0
while i < len(word_list):
if word_list[i].startswith("##"):
result[-1] += word_list[i][2:] # Remove "##" and append to the previous word
elif i < len(word_list) - 2 and word_list[i + 1] == "-": # Combine hyphenated words
result.append(word_list[i] + word_list[i + 1] + word_list[i + 2])
i += 2 # Skip the next two words
else:
result.append(word_list[i])
i += 1
return result
if __name__ == "__main__":
text = "The Saudi authorities, I am told, are currently working flat out" \
"to collate everything they have on the Magdeburg market suspect," \
"Taleb al-Abdulmohsen, and to share it with Germany's ongoing" \
"investigation"
print(extract_entities(text)) |