first-commit

.gitignore

@@ -0,0 +1,2 @@
+.DS_Store
+.idea/

.streamlit/config.toml

@@ -0,0 +1,6 @@
+[theme]
+primaryColor="#f58442"
+backgroundColor="#FFFFFF"
+secondaryBackgroundColor="#F0F2F6"
+textColor="#262730"
+font="sans serif"

app.py

@@ -0,0 +1,254 @@
+import pandas as pd
+import streamlit as st
+from annotated_text import annotated_text
+from streamlit_option_menu import option_menu
+from sentiment_analysis import SentimentAnalysis
+from keyword_extraction import KeywordExtractor
+from part_of_speech_tagging import POSTagging
+from emotion_detection import EmotionDetection
+from named_entity_recognition import NamedEntityRecognition
+
+hide_streamlit_style = """
+            <style>
+            #MainMenu {visibility: hidden;}
+            footer {visibility: hidden;}
+            </style>
+            """
+st.markdown(hide_streamlit_style, unsafe_allow_html=True)
+
+
+@st.cache(allow_output_mutation=True)
+def load_sentiment_model():
+    return SentimentAnalysis()
+
+@st.cache(allow_output_mutation=True)
+def load_keyword_model():
+    return KeywordExtractor()
+
+@st.cache(allow_output_mutation=True)
+def load_pos_model():
+    return POSTagging()
+
+@st.cache(allow_output_mutation=True)
+def load_emotion_model():
+    return EmotionDetection()
+
+@st.cache(allow_output_mutation=True)
+def load_ner_model():
+    return NamedEntityRecognition()
+
+
+sentiment_analyzer = load_sentiment_model()
+keyword_extractor = load_keyword_model()
+pos_tagger = load_pos_model()
+emotion_detector = load_emotion_model()
+ner = load_ner_model()
+
+example_text = "This is example text that contains both names of organizations like Hugging Face and cities like New York, all while portraying an upbeat attitude."
+
+with st.sidebar:
+    page = option_menu(menu_title='Menu',
+                       menu_icon="robot",
+                       options=["Welcome!",
+                                "Sentiment Analysis",
+                                "Keyword Extraction",
+                                "Part of Speech Tagging",
+                                "Emotion Detection",
+                                "Named Entity Recognition"],
+                       icons=["house-door",
+                              "chat-dots",
+                              "key",
+                              "tag",
+                              "emoji-heart-eyes",
+                              "building"],
+                       default_index=0
+                       )
+
+st.title('Open-source NLP')
+
+if page == "Welcome!":
+    st.header('Welcome!')
+
+    st.markdown("![Alt Text](https://media.giphy.com/media/2fEvoZ9tajMxq/giphy.gif)")
+    st.write(
+        """
+     
+     
+        """
+    )
+
+    st.subheader("Quickstart")
+    st.write(
+        """
+        Replace the example text below and flip through the pages in the menu to perform NLP tasks on-demand!
+        Feel free to use the example text for a test run. 
+        """
+    )
+
+    text = st.text_area("Paste text here", value=example_text)
+
+    st.subheader("Introduction")
+    st.write("""
+        Hello! This application is a celebration of open-source and the power that programmers have been granted today
+        by those who give back to the community. This tool was constructed using Streamlit, Huggingface Transformers, 
+        Transformers-Interpret, NLTK, Spacy, amongst other open-source Python libraries and models. 
+        
+        Utilizing this tool you will be able to perform a multitude of Natural Language Processing Tasks on a range of
+        different tasks. All you need to do is paste your input, select your task, and hit the start button! 
+        
+        * This application currently supports:
+            * Sentiment Analysis
+            * Keyword Extraction
+            * Part of Speech Tagging
+            * Emotion Detection
+            * Named Entity Recognition
+            
+        More features may be added in the future including article/tweet/youtube input, improved text annotation, model quality improvements, 
+        depending on community feedback. Please reach out to me at [email protected] or at my Linkedin page listed 
+        below if you have ideas or suggestions for improvement.
+        
+        If you would like to contribute yourself, feel free to fork the Github repository listed below and submit a merge request.
+        """
+    )
+    st.subheader("Notes")
+    st.write(
+        """
+        * This dashboard was constructed by myself, but every resource used is open-source! If you are interested in my other works you can view them here:
+        
+           [Project Github](https://github.com/MiesnerJacob/Multi-task-NLP-dashboard)
+           
+           [Jacob Miesner's Github](https://github.com/MiesnerJacob)
+           
+           [Jacob Miesner's Linkedin](https://www.linkedin.com/in/jacob-miesner-885050125/)
+           
+           [Jacob Miesner's Website](https://www.jacobmiesner.com)
+              
+        * The prediction justification for some of the tasks are printed as the model views them. For this reason the text may contain special tokens like [CLS] or [SEP] or even hashtags splitting words. If you are are familiar with language models you will recognize these, if you do not have prior experience with language models you can ignore these characters.  
+        """
+    )
+
+elif page == "Sentiment Analysis":
+    st.header('Sentiment Analysis')
+    st.markdown("![Alt Text](https://media.giphy.com/media/XIqCQx02E1U9W/giphy.gif)")
+    st.write(
+        """
+
+
+        """
+    )
+
+    text = st.text_area("Paste text here", value=example_text)
+
+    if st.button('🔥 Run!'):
+        with st.spinner("Loading..."):
+            preds, html = sentiment_analyzer.run(text)
+            st.success('All done!')
+            st.write("")
+            st.subheader("Sentiment Predictions")
+            st.bar_chart(data=preds, width=0, height=0, use_container_width=True)
+            st.write("")
+            st.subheader("Sentiment Justification")
+            raw_html = html._repr_html_()
+            st.components.v1.html(raw_html, height=500)
+
+elif page == "Keyword Extraction":
+    st.header('Keyword Extraction')
+    st.markdown("![Alt Text](https://media.giphy.com/media/xT9C25UNTwfZuk85WP/giphy-downsized-large.gif)")
+    st.write(
+        """
+
+
+        """
+    )
+
+    text = st.text_area("Paste text here", value=example_text)
+
+    max_keywords = st.slider('# of Keywords Max Limit', min_value=1, max_value=10, value=5, step=1)
+
+    if st.button('🔥 Run!'):
+        with st.spinner("Loading..."):
+            annotation, keywords = keyword_extractor.generate(text, max_keywords)
+            st.success('All done!')
+
+        if annotation:
+            st.subheader("Keyword Annotation")
+            st.write("")
+            annotated_text(*annotation)
+            st.text("")
+
+        st.subheader("Extracted Keywords")
+        st.write("")
+        df = pd.DataFrame(keywords, columns=['Extracted Keywords'])
+        csv = df.to_csv(index=False).encode('utf-8')
+        st.download_button('Download Keywords to CSV', csv, file_name='news_intelligence_keywords.csv')
+
+        data_table = st.table(df)
+
+elif page == "Part of Speech Tagging":
+    st.header('Part of Speech Tagging')
+    st.markdown("![Alt Text](https://media.giphy.com/media/WoWm8YzFQJg5i/giphy.gif)")
+    st.write(
+        """
+
+
+        """
+    )
+
+    text = st.text_area("Paste text here", value=example_text)
+
+    if st.button('🔥 Run!'):
+        with st.spinner("Loading..."):
+            preds = pos_tagger.classify(text)
+            st.success('All done!')
+            st.write("")
+            st.subheader("Part of Speech tags")
+            annotated_text(*preds)
+            st.write("")
+            st.components.v1.iframe('https://www.ling.upenn.edu/courses/Fall_2003/ling001/penn_treebank_pos.html', height=1000)
+
+elif page == "Emotion Detection":
+    st.header('Emotion Detection')
+    st.markdown("![Alt Text](https://media.giphy.com/media/fU8X6ozSszyEw/giphy.gif)")
+    st.write(
+        """
+
+
+        """
+    )
+
+    text = st.text_area("Paste text here", value=example_text)
+
+    if st.button('🔥 Run!'):
+        with st.spinner("Loading..."):
+            preds, html = emotion_detector.run(text)
+            st.success('All done!')
+            st.write("")
+            st.subheader("Emotion Predictions")
+            st.bar_chart(data=preds, width=0, height=0, use_container_width=True)
+            raw_html = html._repr_html_()
+            st.write("")
+            st.subheader("Emotion Justification")
+            st.components.v1.html(raw_html, height=500)
+
+elif page == "Named Entity Recognition":
+    st.header('Named Entity Recognition')
+    st.markdown("![Alt Text](https://media.giphy.com/media/lxO8wdWdu4tig/giphy.gif)")
+    st.write(
+        """
+
+
+        """
+    )
+
+    text = st.text_area("Paste text here", value=example_text)
+
+    if st.button('🔥 Run!'):
+        with st.spinner("Loading..."):
+            preds, ner_annotation = ner.classify(text)
+            st.success('All done!')
+            st.write("")
+            st.subheader("NER Predictions")
+            annotated_text(*ner_annotation)
+            st.write("")
+            st.subheader("NER Prediction Metadata")
+            st.write(preds)

emotion_detection.py

@@ -0,0 +1,74 @@
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers_interpret import SequenceClassificationExplainer
+import torch
+import pandas as pd
+
+
+class EmotionDetection:
+    """
+    Emotion Detection on text data.
+
+    Attributes:
+        tokenizer: An instance of Hugging Face Tokenizer
+        model: An instance of Hugging Face Model
+        explainer: An instance of SequenceClassificationExplainer from Transformers interpret
+    """
+
+    def __init__(self):
+        hub_location = 'cardiffnlp/twitter-roberta-base-emotion'
+        self.tokenizer = AutoTokenizer.from_pretrained(hub_location)
+        self.model = AutoModelForSequenceClassification.from_pretrained(hub_location)
+        self.explainer = SequenceClassificationExplainer(self.model, self.tokenizer)
+
+    def justify(self, text):
+        """
+        Get html annotation for displaying emotion justification over text.
+
+        Parameters:
+            text (str): The user input string to emotion justification
+
+        Returns:
+            html (hmtl): html object for plotting emotion prediction justification
+        """
+
+        word_attributions = self.explainer(text)
+        html = self.explainer.visualize("example.html")
+
+        return html
+
+    def classify(self, text):
+        """
+        Recognize Emotion in text.
+
+        Parameters:
+            text (str): The user input string to perform emotion classification on
+
+        Returns:
+            predictions (str): The predicted probabilities for emotion classes
+        """
+
+        tokens = self.tokenizer.encode_plus(text, add_special_tokens=False, return_tensors='pt')
+        outputs = self.model(**tokens)
+        probs = torch.nn.functional.softmax(outputs[0], dim=-1)
+        probs = probs.mean(dim=0).detach().numpy()
+        labels = list(self.model.config.id2label.values())
+        preds = pd.Series(probs, index=labels, name='Predicted Probability')
+
+        return preds
+
+    def run(self, text):
+        """
+        Classify and Justify Emotion in text.
+
+        Parameters:
+            text (str): The user input string to perform emotion classification on
+
+        Returns:
+            predictions (str): The predicted probabilities for emotion classes
+            html (hmtl): html object for plotting emotion prediction justification
+        """
+
+        preds = self.classify(text)
+        html = self.justify(text)
+
+        return preds, html

keyword_extraction.py

@@ -0,0 +1,158 @@
+import nltk
+import pytextrank
+import re
+from operator import itemgetter
+import en_core_web_sm
+
+
+class KeywordExtractor:
+    """
+    Keyword Extraction on text data
+
+    Attributes:
+        nlp: An instance English pipeline optimized for CPU for spacy
+    """
+
+    def __init__(self):
+        self.nlp = en_core_web_sm.load()
+        self.nlp.add_pipe("textrank")
+
+    def get_keywords(self, text, max_keywords):
+        """
+        Extract keywords from text.
+
+        Parameters:
+            text (str): The user input string to extract keywords from
+
+        Returns:
+            kws (list): list of extracted keywords
+        """
+
+        doc = self.nlp(text)
+
+        kws = [i.text for i in doc._.phrases[:max_keywords]]
+
+        return kws
+
+    def get_keyword_indices(self, kws, text):
+        """
+        Extract keywords from text.
+
+        Parameters:
+            kws (list): list of extracted keywords
+            text (str): The user input string to extract keywords from
+
+        Returns:
+            keyword_indices (list): list of indices for keyword boundaries in text
+        """
+
+        keyword_indices = []
+        for s in kws:
+            indices = [[m.start(), m.end()] for m in re.finditer(re.escape(s), text)]
+            keyword_indices.extend(indices)
+
+        return keyword_indices
+
+    def merge_overlapping_indices(self, keyword_indices):
+        """
+        Merge overlapping keyword indices.
+
+        Parameters:
+            keyword_indices (list): list of indices for keyword boundaries in text
+
+        Returns:
+            keyword_indices (list): list of indices for keyword boundaries in with overlapping combined
+        """
+
+        # Sort the array on the basis of start values of intervals.
+        keyword_indices.sort()
+
+        stack = []
+        # insert first interval into stack
+        stack.append(keyword_indices[0])
+        for i in keyword_indices[1:]:
+            # Check for overlapping interval,
+            # if interval overlap
+            if (stack[-1][0] <= i[0] <= stack[-1][-1]) or (stack[-1][-1] == i[0]-1):
+                stack[-1][-1] = max(stack[-1][-1], i[-1])
+            else:
+                stack.append(i)
+        return stack
+
+    def merge_until_finished(self, keyword_indices):
+        """
+        Loop until no overlapping keyword indices left.
+
+        Parameters:
+            keyword_indices (list): list of indices for keyword boundaries in text
+
+        Returns:
+            keyword_indices (list): list of indices for keyword boundaries in with overlapping combined
+        """
+
+        len_indices = 0
+        while True:
+            # Merge overlapping indices
+            merged = self.merge_overlapping_indices(keyword_indices)
+            # Check to see if merging reduced number of annotation indices
+            # If merging did not reduce list return final indicies
+            if len_indices == len(merged):
+                out_indices = sorted(merged, key=itemgetter(0))
+                return out_indices
+            else:
+                len_indices = len(merged)
+
+    def get_annotation(self, text, keyword_indices):
+        """
+        Create text annotation for extracted keywords.
+
+        Parameters:
+            keyword_indices (list): list of indices for keyword boundaries in text
+
+        Returns:
+            annotation (list): list of tuples for generating html
+        """
+
+        # Turn list to numpy array
+        arr = list(text)
+
+        # Loop through indices in list and insert delimeters
+        for idx in sorted(keyword_indices, reverse=True):
+            arr.insert(idx[0], "<kw>")
+            arr.insert(idx[1]+1, "<!kw> <kw>")
+
+        # join array
+        joined_annotation = ''.join(arr)
+
+        # split array on delimeter
+        split = joined_annotation.split('<kw>')
+
+        # Create annotation for keywords in text
+        annotation = [(x.replace('<!kw> ', ''), "KEY", "#26aaef") if "<!kw>" in x else x for x in split]
+
+        return annotation
+
+    def generate(self, text, max_keywords):
+        """
+        Create text annotation for extracted keywords.
+
+        Parameters:
+            text (str): The user input string to extract keywords from
+            max_keywords (int): Limit on number of keywords to generate
+
+        Returns:
+            annotation (list): list of tuples for generating html
+            kws (list): list of extracted keywords
+        """
+
+        kws = self.get_keywords(text, max_keywords)
+
+        indices = list(self.get_keyword_indices(kws, text))
+        if indices:
+            indices_merged = self.merge_until_finished(indices)
+            annotation = self.get_annotation(text, indices_merged)
+        else:
+            annotation = None
+
+        return annotation, kws
+

named_entity_recognition.py

@@ -0,0 +1,65 @@
+from transformers import AutoTokenizer, AutoModelForTokenClassification
+from transformers import pipeline
+
+
+class NamedEntityRecognition:
+    """
+    Named Entity Recognition on text data.
+
+    Attributes:
+        tokenizer: An instance of Hugging Face Tokenizer
+        model: An instance of Hugging Face Model
+        nlp: An instance of Hugging Face Named Entity Recognition pipeline
+    """
+
+    def __init__(self):
+        tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
+        model = AutoModelForTokenClassification.from_pretrained("xlm-roberta-large-finetuned-conll03-english")
+        self.nlp = pipeline("ner", model=model, tokenizer=tokenizer, grouped_entities=True)
+
+    def get_annotation(self, preds, text):
+        """
+        Get html annotation for displaying entities over text.
+
+        Parameters:
+            preds (dict): List of entities and their associated metadata
+            text (str): The user input string to generate entity tags for
+
+        Returns:
+            final_annotation (list): List of tuples to pass to text annotation html creator
+        """
+
+        splits = [0]
+        entities = {}
+        for i in preds:
+            splits.append(i['start'])
+            splits.append(i['end'])
+            entities[i['word']] = i['entity_group']
+
+        # Exclude bad preds
+        exclude = ['', '.', '. ', ' ']
+        for x in exclude:
+            if x in entities.keys():
+                entities.pop(x)
+
+        parts = [text[i:j] for i, j in zip(splits, splits[1:] + [None])]
+
+        final_annotation = [(x, entities[x], "") if x in entities.keys() else x for x in parts]
+
+        return final_annotation
+
+    def classify(self, text):
+        """
+        Recognize Named Entities in text.
+
+        Parameters:
+            text (str): The user input string to generate entity tags for
+
+        Returns:
+            predictions (str): The user input string to generate entity tags for
+            ner_annotation (str): The user input string to generate entity tags for
+        """
+
+        preds = self.nlp(text)
+        ner_annotation = self.get_annotation(preds, text)
+        return preds, ner_annotation

part_of_speech_tagging.py

@@ -0,0 +1,26 @@
+import nltk
+from nltk.tokenize import word_tokenize
+nltk.download('punkt')
+nltk.download('averaged_perceptron_tagger')
+
+
+class POSTagging:
+    """Part of Speech Tagging on text data"""
+
+    def __init__(self):
+       pass
+
+    def classify(self, text):
+        """
+        Generate Part of Speech tags.
+
+        Parameters:
+            text (str): The user input string to generate tags for
+
+        Returns:
+            predictions (list): list of tuples containing words and their respective tags
+        """
+
+        text = word_tokenize(text)
+        predictions = nltk.pos_tag(text)
+        return predictions

requirements.txt

@@ -0,0 +1,12 @@
+scikit-learn
+tensorflow==2.5.0
+tensorflow-hub==0.12.0
+nltk==3.5
+typing-extensions==3.7.4.3
+streamlit-option-menu==0.3.2
+st-annotated-text==3.0.0
+transformers-interpret==0.7.2
+htbuilder==0.6.0
+pytextrank==3.2.3
+spacy==3.8.3
+en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_core_web_sm-3.8.0/en_core_web_sm-3.8.0-py3-none-any.whl

sentiment_analysis.py

@@ -0,0 +1,84 @@
+from transformers import AutoTokenizer, AutoModelForSequenceClassification
+from transformers_interpret import SequenceClassificationExplainer
+import torch
+import pandas as pd
+
+
+class SentimentAnalysis:
+    """
+    Sentiment on text data.
+
+    Attributes:
+        tokenizer: An instance of Hugging Face Tokenizer
+        model: An instance of Hugging Face Model
+        explainer: An instance of SequenceClassificationExplainer from Transformers interpret
+    """
+
+    def __init__(self):
+        # Load Tokenizer & Model
+        hub_location = 'cardiffnlp/twitter-roberta-base-sentiment'
+        self.tokenizer = AutoTokenizer.from_pretrained(hub_location)
+        self.model = AutoModelForSequenceClassification.from_pretrained(hub_location)
+
+        # Change model labels in config
+        self.model.config.id2label[0] = "Negative"
+        self.model.config.id2label[1] = "Neutral"
+        self.model.config.id2label[2] = "Positive"
+        self.model.config.label2id["Negative"] = self.model.config.label2id.pop("LABEL_0")
+        self.model.config.label2id["Neutral"] = self.model.config.label2id.pop("LABEL_1")
+        self.model.config.label2id["Positive"] = self.model.config.label2id.pop("LABEL_2")
+
+        # Instantiate explainer
+        self.explainer = SequenceClassificationExplainer(self.model, self.tokenizer)
+
+    def justify(self, text):
+        """
+        Get html annotation for displaying sentiment justification over text.
+
+        Parameters:
+            text (str): The user input string to sentiment justification
+
+        Returns:
+            html (hmtl): html object for plotting sentiment prediction justification
+        """
+
+        word_attributions = self.explainer(text)
+        html = self.explainer.visualize("example.html")
+
+        return html
+
+    def classify(self, text):
+        """
+        Recognize Sentiment in text.
+
+        Parameters:
+            text (str): The user input string to perform sentiment classification on
+
+        Returns:
+            predictions (str): The predicted probabilities for sentiment classes
+        """
+
+        tokens = self.tokenizer.encode_plus(text, add_special_tokens=False, return_tensors='pt')
+        outputs = self.model(**tokens)
+        probs = torch.nn.functional.softmax(outputs[0], dim=-1)
+        probs = probs.mean(dim=0).detach().numpy()
+        predictions = pd.Series(probs, index=["Negative", "Neutral", "Positive"], name='Predicted Probability')
+
+        return predictions
+
+    def run(self, text):
+        """
+        Classify and Justify Sentiment in text.
+
+        Parameters:
+            text (str): The user input string to perform sentiment classification on
+
+        Returns:
+            predictions (str): The predicted probabilities for sentiment classes
+            html (hmtl): html object for plotting sentiment prediction justification
+        """
+
+        predictions = self.classify(text)
+        html = self.justify(text)
+
+        return predictions, html