Spaces:
Runtime error
Runtime error
File size: 4,327 Bytes
eb67193 a76c1ab eb67193 a76c1ab eb67193 a76c1ab eb67193 a76c1ab eb67193 a76c1ab eb67193 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 |
import pandas as pd
import requests
import isort
import black
import flair
import time
from bs4 import BeautifulSoup
import re
import numpy as np
from flair.data import Sentence
from flair.models import SequenceTagger
from transformers import AutoTokenizer, AutoModelForQuestionAnswering, pipeline
import string
URL = "https://www.formula1.com/content/fom-website/en/latest/all.xml"
def get_xml(url):
# xpath is only for formula1
# use urllib.parse to check for formula1.com website or other news
xml = pd.read_xml(url,xpath='channel/item')
# care taken to only consider results where there are more words not a single word quotes
def extract_quote(string):
# Use the re.findall function to extract the quoted text
results = re.findall(r'[“\"](.*?)[”\"]', string)
quotes = []
for result in results:
split_result = result.split()
if len(split_result) >3:
quotes.append(result)
return quotes
def get_names(text):
# # load the NER tagger
tagger = SequenceTagger.load('ner')
sentence = Sentence(text)
tagger.predict(sentence)
names = []
for label in sentence.get_labels('ner'):
if label.value == "PER":
names.append(f"{label.data_point.text}")
# convert to a set to remove some of the repetitions
names = list(set(names))
return names
def get_text(new_articles_df):
"""
quotes outputs a list of quotes
"""
dfs_dict = {}
for article in tqdm(new_articles_df.iterrows()):
link = article[1]["guid"]
request = requests.get(link)
soup = BeautifulSoup(request.content, "html.parser")
# class_ below will be different for different websites
s = soup.find("div", class_="col-lg-8 col-xl-7 offset-xl-1 f1-article--content")
lines = s.find_all("p")
text_content = pd.DataFrame(data={"text": []})
for i, line in enumerate(lines):
df = pd.DataFrame(data={"text": [line.text]})
text_content = pd.concat([text_content, df], ignore_index=True)
strongs = s.find_all("strong")
strong_content = pd.DataFrame(data={"text": []})
for i, strong in enumerate(strongs):
if i > 0:
df = pd.DataFrame(data={"text": [strong.text]})
strong_content = pd.concat([strong_content, df], ignore_index=True)
# df has content
df = text_content[~text_content["text"].isin(strong_content["text"])].reset_index(
drop=True
)
# df["quote"] = df["text"].apply(lambda row: extract_quote(row))
# # combine all rows into context
context = ""
for i,row in df.iterrows():
context += f" {row['text']}"
quotes = extract_quote(context)
# to save some time not computing unnecessary NER
if len(quotes) != 0:
speakers = get_names(context)
else:
speakers = ()
dfs_dict[link] = {'context':context, 'quotes':quotes, 'speakers':speakers}
return dfs_dict
def load_speaker_model():
model_name = f"microsoft/deberta-v2-large"
tokenizer = AutoTokenizer.from_pretrained(model_name)
model = AutoModelForQuestionAnswering.from_pretrained(model_name)
question_answerer = pipeline("question-answering", model=model, tokenizer=tokenizer)
return question_answerer
def remove_punctuations(text):
modified_text = "".join([character for character in text if character not in string.punctuation])
modified_text = modified_text.lstrip(" ")
modified_text = modified_text.rstrip(" ")
return modified_text
def check_updates(every=300):
while True:
time.sleep(every)
latest_xml = get_xml()
if ~previous_xml.equals(latest_xml):
print('New articles found')
new_articles_df = latest_xml[~latest_xml["guid"].isin(previous_xml["guid"])]
# loops through new articles and gets the necessary text, quotes and speakers
dfs_dict = get_text(new_articles_df)
else:
print('No New article is found')
|