File size: 1,834 Bytes
da88570 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 |
from wtpsplit import SaT
from transformers import pipeline
import json
from src.nlp.playground.textclassification import classify_paragraph
from src.persistence.db import *
uri = f"mongodb+srv://event_data_extraction_application:[email protected]/?retryWrites=true&w=majority&appName=Cluster0"
client = MongoClient(uri)
db = client.event_data
unsorted_urls = db.unsorted_urls
event_urls = db.event_urls
label_description = "Veranstaltungsbeschreibung"
sat = SaT("sat-12l-sm")
text_class = [
label_description,
"Titel",
"Sonstiges",
]
classifier = pipeline("zero-shot-classification",
model="facebook/bart-large-mnli")
hypothesis_template = "Der Text ist {}."
count = 10
for event in event_urls.find():
if "data" in event:
text = event["data"]
if text !="":
print("Original:")
print(text)
segments = sat.split(text, do_paragraph_segmentation=True,paragraph_threshold=0.1)
# print(segments)
title = "",
description = ""
for sequence in segments:
sequence = " ".join(sequence)
# print(sequence)
if sequence!="":
## Since monolingual model,its sensitive to hypothesis template. This can be experimented
print("SEGMENTATION:")
print(sequence)
# predictions = classifier(sequence, text_class, hypothesis_template=hypothesis_template)
predictions = classify_paragraph(sequence)
print("Labels:")
print(predictions)
print("\n")
print("Beschreibung:")
print(description)
count-=1
if count==0:
break |