src/application/text/helper.py

from collections import Counter
import re
import string
from sklearn.feature_extraction.text import TfidfVectorizer
from nltk.tokenize import word_tokenize
from nltk.util import ngrams


def clean_text(text):
    """Doc cleaning"""
    punctuations = r"""!"#$%&'()*+-/:;<=>?@[\]^_`{|}~"""  # not include , and . due to number
    # Lowering text
    text = text.lower()
    
    # Removing punctuation
    text = "".join([c for c in text if c not in punctuations])
    
    # Removing whitespace and newlines
    text = re.sub(r'\s+',' ',text)
    
    text.replace("£", " * ")
    
    words = text.split()
    text = ' '.join(words[:18])  # Join the first 18 words back into a string
    
    return text

def remove_punctuation(text):
    """Remove punctuation from a given text."""
    punctuation_without_dot = string.punctuation.replace(".", "")
    translator = str.maketrans('', '', punctuation_without_dot)
    return text.translate(translator)

def get_keywords(text, num_keywords=5):
    """Return top k keywords from a doc using TF-IDF method"""
    
    # Create a TF-IDF Vectorizer
    vectorizer = TfidfVectorizer(stop_words='english')
    
    # Fit and transform the text
    tfidf_matrix = vectorizer.fit_transform([text])
    
    # Get feature names (words)
    feature_names = vectorizer.get_feature_names_out()
    
    # Get TF-IDF scores
    tfidf_scores = tfidf_matrix.toarray()[0]
    
    # Sort words by TF-IDF score
    word_scores = list(zip(feature_names, tfidf_scores))
    word_scores.sort(key=lambda x: x[1], reverse=True)
    
    # Return top keywords
    return [word for word, score in word_scores[:num_keywords]]

"""
# Example usage
text = "Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to natural intelligence displayed by animals including humans. Leading AI textbooks define the field as the study of "intelligent agents": any system that perceives its environment and takes actions that maximize its chance of achieving its goals. Some popular accounts use the term "artificial intelligence" to describe machines that mimic "cognitive" functions that humans associate with the human mind, such as "learning" and "problem solving", however this definition is rejected by major AI researchers."
print(f"\n# Input text:\n'{text}'")
print("\n----------------------\n") 

keywords = get_keywords(text)
print("# Top keywords:", keywords)
print("\n----------------------\n")
"""

def get_important_sentences(paragraph: str, keywords: list[str], num_sentences: int = 3) -> list[str]:
    """
    Selects important sentences from a given paragraph based on a list of keywords.

    Args:
        paragraph (str): The input paragraph.
        keywords (list[str]): List of important keywords.
        num_sentences (int): Number of sentences to return (default is 3).

    Returns:
        list: A list of important sentences.
    """
    # Clean and split the paragraph into sentences
    sentences = [s.strip() for s in re.split(r'(?<=[.!?])\s+', paragraph) if s.strip()]
    
    # Calculate the importance score for each sentence
    sentence_scores = []
    for sentence in sentences:
        processed_sentence = clean_text(sentence)
        score = 0
        words = processed_sentence.lower().split()
        word_count = Counter(words)
        
        for keyword in keywords:
            if keyword.lower() in word_count:
                score += word_count[keyword.lower()]
        
        sentence_scores.append((sentence, score))
    
    # Sort sentences by their scores in descending order
    sentence_scores.sort(key=lambda x: x[1], reverse=True)
    
    # Return the top N sentences
    return [sentence for sentence, score in sentence_scores[:num_sentences]]

"""# Example usage
keywords = get_keywords(paragraph)
important_sentences = get_important_sentences(paragraph, keywords)

print("# Important sentences:")
for i, sentence in enumerate(important_sentences, 1):
    print(f"{i}. {sentence}")
print("\n----------------------\n") 
"""

def extract_important_phrases(paragraph: str, keywords: list[str], phrase_length: int = 5) -> list[str]:
    """
    Extracts important phrases from a given paragraph based on a list of keywords.
    Phrase length is auto-determined, and overlapped parts are less than 20%.

    Args:
        paragraph (str): The input paragraph.
        keywords (list[str]): List of important keywords.
        phrase_length (int): The length of phrases to extract (default is 5 words).

    Returns:
        list: A list of important phrases.
    """
    # Tokenize the paragraph into words
    words = word_tokenize(paragraph.lower())
    
    # Determine phrase length (between 3 and 7 words)
    phrase_length = min(max(len(words) // 10, 5), 7)
    
    # Generate n-grams (phrases) from the paragraph
    phrases = list(ngrams(words, phrase_length))
    
    important_phrases = []
    used_indices = set()
    
    for i, phrase in enumerate(phrases):
        # Check if the phrase contains any keyword
        if any(keyword.lower() in phrase for keyword in keywords):
            # Check overlap with previously selected phrases
            if not any(abs(i - j) < phrase_length * 0.8 for j in used_indices):
                important_phrases.append(clean_text(" ".join(phrase)))
                used_indices.add(i)
    
    return important_phrases

"""# Example usage
keywords = get_keywords(paragraph)
important_phrases = extract_important_phrases(paragraph, keywords)

print("# Important phrases:")
for i, phrase in enumerate(important_phrases[:5], 1):  # Print top 5 phrases
    print(f"{i}. {phrase}")"""