first commit

.gitignore

@@ -1,3 +1,5 @@
 WhatsApp*
 /chat-analyser/*
-chat-analyser
+chat-analyser
+.DS_Store
+__pycache__

app.py

@@ -0,0 +1,85 @@
+import streamlit as st
+import preprocessor
+import matplotlib.pyplot as plt
+
+
+st.sidebar.title('Whatsapp Chat Analyzer')
+st.sidebar.write('Upload your whatsapp chat file to analyze the chat')
+st.sidebar.write('The chat file should be a .txt file exported from whatsapp')
+uploaded_file = st.sidebar.file_uploader('Choose a file')
+
+if uploaded_file is not None:
+    chat_data = uploaded_file.getvalue().decode('utf-8')
+    df = preprocessor.preprocess_data(chat_data)
+    # st.write(df)
+
+    #unique users dropdown menu to select user
+    unique_users = df['user'].unique()
+    unique_users = list(unique_users)
+    unique_users.sort()
+    unique_users.insert(0, 'All')
+    selected_user = st.sidebar.selectbox('Select a user', unique_users)
+    st.sidebar.write(selected_user)
+
+    #fetch stats
+    user_df, total_messages, media_messages, links, emojis, total_words = preprocessor.fetch_stats(selected_user, df)
+    st.title(f'User: {selected_user}')
+    # st.write(user_df)
+    #display below data side by side 
+    col1, col2, col3 = st.columns(3)
+    with col1:
+        st.write('Total messages:', total_messages)
+        st.write('Media messages:', media_messages)
+    with col2:
+        st.write('Links shared:', links)
+        st.write('Emojis shared:', emojis)
+    with col3:
+        st.write('Total words:', total_words)
+
+    #busiest users (users with most messages)
+    with st.expander('Busiest users'):
+        if selected_user == 'All':
+            busiest_users, plot = preprocessor.busiest_users(df)
+            st.write('Busiest users:')
+            st.write(busiest_users)
+            st.pyplot(plot)
+
+    with st.expander('View word cloud and most common words'):
+        st.pyplot(preprocessor.word_cloud(df, selected_user))
+
+    with st.expander('Most common words'):
+        st.write('Most common words')
+        temp_df, plot = preprocessor.most_common_words(selected_user, df)
+        st.write(temp_df)
+        st.pyplot(plot)
+        st.write("Most common emojis")
+        temp_df = preprocessor.most_common_emojis(selected_user, df)
+        st.write(temp_df)
+
+    with st.expander("Activity over time"):
+        col1, col2 = st.columns(2)
+        with col1: #left
+            _, plot = preprocessor.daily_timeline(selected_user, df)
+            st.pyplot(plot)
+            _, plot = preprocessor.weekday_activity_map(selected_user, df)
+            st.pyplot(plot)
+        with col2: #right
+            _, plot = preprocessor.monthly_timeline(selected_user, df)
+            st.pyplot(plot)
+            _, plot = preprocessor.month_activity_map(selected_user, df)
+            st.pyplot(plot)
+        
+        st.write("Messages sent by hour")
+        _, plot = preprocessor.hour_activity_map(selected_user, df)
+        st.pyplot(plot)
+
+        st.write("weekly heatmap")
+        plot = preprocessor.activity_heatmap(selected_user, df)
+        st.pyplot(plot)
+
+
+    
+
+
+
+

preprocessor.py

@@ -0,0 +1,230 @@
+from turtle import back
+import pandas as pd
+import re
+import matplotlib.pyplot as plt
+from wordcloud import WordCloud
+from collections import Counter
+import emoji
+import numpy as np
+import seaborn as sns
+
+def preprocess_data(chat_data):
+    date_time_pattern = r'\d{1,2}/\d{1,2}/\d{2,4},\s\d{1,2}:\d{2}\s[APap][mM]\s-\s'
+    messages = re.split(date_time_pattern, chat_data)[1:]
+    date_times = re.findall(date_time_pattern, chat_data)
+    df = pd.DataFrame({'date_time': date_times, 'message': messages})
+    df['date_time'] = df['date_time'].str.replace('\u202f', ' ')
+    df['date_time'] = pd.to_datetime(df['date_time'], format='%m/%d/%y, %I:%M %p - ')
+    #separate users and their corresponding messages
+    users = []
+    messages = []
+    for message in df['message']:
+        entry = re.split('([\w\W]+?):\s', message)
+        if len(entry) > 1:
+            users.append(entry[1])
+            messages.append(entry[2])
+        else:
+            users.append('group_notification')
+            messages.append(entry[0])
+    df['user'] = users
+    df['message'] = messages
+    # extract date month year hour and minute from date_time
+    df['date'] = df['date_time'].dt.date
+    df['time'] = df['date_time'].dt.time
+    df['hour'] = df['date_time'].dt.hour
+    df['minute'] = df['date_time'].dt.minute
+    df['day'] = df['date_time'].dt.day
+    df['month'] = df['date_time'].dt.month
+    df['year'] = df['date_time'].dt.year
+    df['weekday'] = df['date_time'].dt.weekday
+    df['weekday_en'] = df['weekday'].map({0: 'Mon', 1: 'Tue', 2: 'Wed', 3: 'Thu', 4: 'Fri', 5: 'Sat', 6: 'Sun'})
+    df['month_en'] = df['month'].map({1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec'})
+    # df['only_date'] = df['date_time'].dt.date
+    df.sample(20)
+    return df
+
+
+def fetch_stats(selected_user, df):
+    if selected_user == 'All':
+        user_df = df
+    else:
+        user_df = df[df['user'] == selected_user]
+    total_messages = user_df.shape[0]
+    media_messages = user_df[user_df['message'] == '<Media omitted>\n']
+    media_messages = media_messages.shape[0]
+    links = user_df[user_df['message'].str.contains('http')]
+    print(links)
+    links = links.shape[0]
+    emojis = user_df[user_df['message'].str.contains('[\U0001F600-\U0001F650]')].shape[0]
+    words = user_df['message'].apply(lambda x: len(x.split()))
+    total_words = words.sum()
+    return user_df, total_messages, media_messages, links, emojis, total_words
+
+def busiest_users(df):
+    user_message_counts = df['user'].value_counts().reset_index()
+    user_message_counts.columns = ['user', 'message_count']
+    user_message_counts['percentage'] = (user_message_counts['message_count'] / user_message_counts['message_count'].sum()) * 100
+    user_message_counts['percentage'] = user_message_counts['percentage'].round(2)
+    user_message_counts = user_message_counts.sort_values(by='message_count', ascending=False)
+    busiest_users = user_message_counts
+    plt.bar(busiest_users.user, busiest_users.message_count)
+    plt.xticks(rotation=45)
+    plt.title('Busiest users')
+    plt.xlabel('Users')
+    plt.ylabel('Total messages')
+    return busiest_users, plt
+
+def word_cloud(df, selected_user):
+    if selected_user == 'All':
+        user_df = df
+    else:
+        user_df = df[df['user'] == selected_user]
+
+    f = open('stop_hinglish.txt','r')
+    stop_words = f.read()
+
+    temp = user_df[user_df['user'] != 'group_notification']
+    temp = temp[temp['message'] != '<Media omitted>\n']
+    temp = temp[temp['message'] != 'This message was deleted\n']
+    temp = temp[temp['message'] != 'You deleted this message\n']
+
+    words = []
+    for message in temp['message']:
+        for word in message.lower().split():
+            if word not in stop_words:
+                words.append(word)
+    words = ' '.join(words)
+
+    wordcloud = WordCloud(width=800, height=400, random_state=21, max_font_size=110, background_color='white')
+    wordcloud = wordcloud.generate(words)
+    plt.figure(figsize=(20, 10))
+    plt.imshow(wordcloud, interpolation="bilinear")
+    plt.axis('off')
+    return plt
+
+def most_common_words(selected_user, df):
+    f = open('stop_hinglish.txt','r')
+    stop_words = f.read()
+
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+
+    temp = df[df['user'] != 'group_notification']
+    temp = temp[temp['message'] != '<Media omitted>\n']
+    temp = temp[temp['message'] != 'This message was deleted\n']
+    temp = temp[temp['message'] != 'You deleted this message\n']
+
+    words = []
+    for message in temp['message']:
+        for word in message.lower().split():
+            if word not in stop_words:
+                words.append(word)
+
+    most_common_df = pd.DataFrame(Counter(words).most_common(50))
+    most_common_df.columns = ['word', 'word_count']
+    fig, ax = plt.subplots(figsize=(10, 15))
+    ax.barh(most_common_df.word, most_common_df.word_count, height=0.8)
+    ax.set_xlabel('Word count')
+    ax.set_ylabel('Words')
+    ax.set_title('Most common words')
+    return most_common_df, fig
+
+def most_common_emojis(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+    emojis = []
+    for message in df['message']:
+        emojis.extend([c for c in message if c in emoji.EMOJI_DATA])
+
+    emoji_df = pd.DataFrame(Counter(emojis).most_common(len(Counter(emojis))))
+    emoji_df.columns = ['emoji', 'emoji_count']
+    return emoji_df
+
+def monthly_timeline(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+
+    monthly_timeline = df.groupby(['year', 'month_en']).count()['message'].reset_index()
+    monthly_timeline['month_year'] = monthly_timeline['month_en'] + ' ' + monthly_timeline['year'].astype(str)
+
+    fig, ax = plt.subplots(figsize=(10, 5))
+    ax.plot(monthly_timeline['month_year'], monthly_timeline['message'])
+    ax.set_title('Messages sent over Months')
+    ax.set_xlabel('Month-Year')
+    ax.set_ylabel('Total messages')
+    return monthly_timeline, fig
+
+def daily_timeline(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+
+    daily_timeline = df.groupby('date').count()['message'].reset_index()
+    daily_timeline.columns = ['date', 'message_count']
+
+    fig, ax = plt.subplots(figsize=(10, 5))
+    ax.plot(daily_timeline['date'], daily_timeline['message_count'])
+    ax.set_title('Messages sent over time')
+    ax.set_xlabel('Date')
+    ax.set_ylabel('Total messages')
+    return daily_timeline, fig
+
+def weekday_activity_map(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+
+    week_df = df['weekday_en'].value_counts().reset_index()
+    week_df.columns = ['weekday', 'message_count']
+    fig, ax = plt.subplots(figsize=(10, 5))
+    ax.bar(week_df['weekday'], week_df['message_count'])
+    ax.set_title('Messages sent per weekday')
+    ax.set_xlabel('Weekday')
+    ax.set_ylabel('Total messages')
+    return None, fig
+
+def month_activity_map(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+
+    month_df = df['month_en'].value_counts().reset_index()
+    month_df.columns = ['month', 'message_count']
+    fig, ax = plt.subplots(figsize=(10, 5))
+    ax.bar(month_df['month'], month_df['message_count'])
+    ax.set_title('Messages sent per month')
+    ax.set_xlabel('Month')
+    ax.set_ylabel('Total messages')
+
+    return month_df, fig
+
+def hour_activity_map(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+
+    # Count the number of messages per hour
+    hour_counts = df['hour'].value_counts().sort_index()
+    # Convert hours to radians
+    hours = np.arange(24)
+    radians = 2 * np.pi * (hours / 24)
+
+    # Create a polar plot
+    fig, ax = plt.subplots(figsize=(10, 10), subplot_kw={'projection': 'polar'})
+    ax.bar(radians, hour_counts, width=0.3, bottom=0.2)
+    ax.set_theta_direction(-1)  # Clockwise
+    ax.set_theta_offset(np.pi / 2.0)  # Start from top
+    ax.set_xticks(radians)
+    ax.set_xticklabels(hours)
+    ax.set_yticklabels([])
+    ax.set_title('Busiest Hours of the Day', va='bottom')
+
+    return hour_counts, fig
+
+def activity_heatmap(selected_user, df):
+    if selected_user != 'All':
+        df = df[df['user'] == selected_user]
+    
+    heatmap_data = df.pivot_table(index='weekday_en', columns='hour', values='message', aggfunc='count', fill_value=0)
+    plt.figure(figsize=(10, 6))
+    sns.heatmap(heatmap_data, cmap='gray_r', linewidths=.5, fmt='d')
+    plt.title('Activity Heatmap')
+    plt.xlabel('Hour of Day')
+    plt.ylabel('Day of Week')
+    return plt

stop_hinglish.txt

@@ -0,0 +1,1070 @@
+.
+..
+...
+?
+-
+--
+1
+2
+3
+4
+5
+6
+7
+8
+9
+0
+a
+aadi
+aaj
+aap
+aapne
+aata
+aati
+aaya
+aaye
+ab
+abbe
+abbey
+abe
+abhi
+able
+about
+above
+accha
+according
+accordingly
+acha
+achcha
+across
+actually
+after
+afterwards
+again
+against
+agar
+ain
+aint
+ain't
+aisa
+aise
+aisi
+alag
+all
+allow
+allows
+almost
+alone
+along
+already
+also
+although
+always
+am
+among
+amongst
+an
+and
+andar
+another
+any
+anybody
+anyhow
+anyone
+anything
+anyway
+anyways
+anywhere
+ap
+apan
+apart
+apna
+apnaa
+apne
+apni
+appear
+are
+aren
+arent
+aren't
+around
+arre
+as
+aside
+ask
+asking
+at
+aur
+avum
+aya
+aye
+baad
+baar
+bad
+bahut
+bana
+banae
+banai
+banao
+banaya
+banaye
+banayi
+banda
+bande
+bandi
+bane
+bani
+bas
+bata
+batao
+bc
+be
+became
+because
+become
+becomes
+becoming
+been
+before
+beforehand
+behind
+being
+below
+beside
+besides
+best
+better
+between
+beyond
+bhai
+bheetar
+bhi
+bhitar
+bht
+bilkul
+bohot
+bol
+bola
+bole
+boli
+bolo
+bolta
+bolte
+bolti
+both
+brief
+bro
+btw
+but
+by
+came
+can
+cannot
+cant
+can't
+cause
+causes
+certain
+certainly
+chahiye
+chaiye
+chal
+chalega
+chhaiye
+clearly
+c'mon
+com
+come
+comes
+could
+couldn
+couldnt
+couldn't
+d
+de
+dede
+dega
+degi
+dekh
+dekha
+dekhe
+dekhi
+dekho
+denge
+dhang
+di
+did
+didn
+didnt
+didn't
+dijiye
+diya
+diyaa
+diye
+diyo
+do
+does
+doesn
+doesnt
+doesn't
+doing
+done
+dono
+dont
+don't
+doosra
+doosre
+down
+downwards
+dude
+dunga
+dungi
+during
+dusra
+dusre
+dusri
+dvaara
+dvara
+dwaara
+dwara
+each
+edu
+eg
+eight
+either
+ek
+else
+elsewhere
+enough
+etc
+even
+ever
+every
+everybody
+everyone
+everything
+everywhere
+ex
+exactly
+example
+except
+far
+few
+fifth
+fir
+first
+five
+followed
+following
+follows
+for
+forth
+four
+from
+further
+furthermore
+gaya
+gaye
+gayi
+get
+gets
+getting
+ghar
+given
+gives
+go
+goes
+going
+gone
+good
+got
+gotten
+greetings
+guys
+haan
+had
+hadd
+hadn
+hadnt
+hadn't
+hai
+hain
+hamara
+hamare
+hamari
+hamne
+han
+happens
+har
+hardly
+has
+hasn
+hasnt
+hasn't
+have
+haven
+havent
+haven't
+having
+he
+hello
+help
+hence
+her
+here
+hereafter
+hereby
+herein
+here's
+hereupon
+hers
+herself
+he's
+hi
+him
+himself
+his
+hither
+hm
+hmm
+ho
+hoga
+hoge
+hogi
+hona
+honaa
+hone
+honge
+hongi
+honi
+hopefully
+hota
+hotaa
+hote
+hoti
+how
+howbeit
+however
+hoyenge
+hoyengi
+hu
+hua
+hue
+huh
+hui
+hum
+humein
+humne
+hun
+huye
+huyi
+i
+i'd
+idk
+ie
+if
+i'll
+i'm
+imo
+in
+inasmuch
+inc
+inhe
+inhi
+inho
+inka
+inkaa
+inke
+inki
+inn
+inner
+inse
+insofar
+into
+inward
+is
+ise
+isi
+iska
+iskaa
+iske
+iski
+isme
+isn
+isne
+isnt
+isn't
+iss
+isse
+issi
+isski
+it
+it'd
+it'll
+itna
+itne
+itni
+itno
+its
+it's
+itself
+ityaadi
+ityadi
+i've
+ja
+jaa
+jab
+jabh
+jaha
+jahaan
+jahan
+jaisa
+jaise
+jaisi
+jata
+jayega
+jidhar
+jin
+jinhe
+jinhi
+jinho
+jinhone
+jinka
+jinke
+jinki
+jinn
+jis
+jise
+jiska
+jiske
+jiski
+jisme
+jiss
+jisse
+jitna
+jitne
+jitni
+jo
+just
+jyaada
+jyada
+k
+ka
+kaafi
+kab
+kabhi
+kafi
+kaha
+kahaa
+kahaan
+kahan
+kahi
+kahin
+kahte
+kaisa
+kaise
+kaisi
+kal
+kam
+kar
+kara
+kare
+karega
+karegi
+karen
+karenge
+kari
+karke
+karna
+karne
+karni
+karo
+karta
+karte
+karti
+karu
+karun
+karunga
+karungi
+kaun
+kaunsa
+kayi
+kch
+ke
+keep
+keeps
+keh
+kehte
+kept
+khud
+ki
+kin
+kine
+kinhe
+kinho
+kinka
+kinke
+kinki
+kinko
+kinn
+kino
+kis
+kise
+kisi
+kiska
+kiske
+kiski
+kisko
+kisliye
+kisne
+kitna
+kitne
+kitni
+kitno
+kiya
+kiye
+know
+known
+knows
+ko
+koi
+kon
+konsa
+koyi
+krna
+krne
+kuch
+kuchch
+kuchh
+kul
+kull
+kya
+kyaa
+kyu
+kyuki
+kyun
+kyunki
+lagta
+lagte
+lagti
+last
+lately
+later
+le
+least
+lekar
+lekin
+less
+lest
+let
+let's
+li
+like
+liked
+likely
+little
+liya
+liye
+ll
+lo
+log
+logon
+lol
+look
+looking
+looks
+ltd
+lunga
+m
+maan
+maana
+maane
+maani
+maano
+magar
+mai
+main
+maine
+mainly
+mana
+mane
+mani
+mano
+many
+mat
+may
+maybe
+me
+mean
+meanwhile
+mein
+mera
+mere
+merely
+meri
+might
+mightn
+mightnt
+mightn't
+mil
+mjhe
+more
+moreover
+most
+mostly
+much
+mujhe
+must
+mustn
+mustnt
+mustn't
+my
+myself
+na
+naa
+naah
+nahi
+nahin
+nai
+name
+namely
+nd
+ne
+near
+nearly
+necessary
+neeche
+need
+needn
+neednt
+needn't
+needs
+neither
+never
+nevertheless
+new
+next
+nhi
+nine
+no
+nobody
+non
+none
+noone
+nope
+nor
+normally
+not
+nothing
+novel
+now
+nowhere
+o
+obviously
+of
+off
+often
+oh
+ok
+okay
+old
+on
+once
+one
+ones
+only
+onto
+or
+other
+others
+otherwise
+ought
+our
+ours
+ourselves
+out
+outside
+over
+overall
+own
+par
+pata
+pe
+pehla
+pehle
+pehli
+people
+per
+perhaps
+phla
+phle
+phli
+placed
+please
+plus
+poora
+poori
+provides
+pura
+puri
+q
+que
+quite
+raha
+rahaa
+rahe
+rahi
+rakh
+rakha
+rakhe
+rakhen
+rakhi
+rakho
+rather
+re
+really
+reasonably
+regarding
+regardless
+regards
+rehte
+rha
+rhaa
+rhe
+rhi
+ri
+right
+s
+sa
+saara
+saare
+saath
+sab
+sabhi
+sabse
+sahi
+said
+sakta
+saktaa
+sakte
+sakti
+same
+sang
+sara
+sath
+saw
+say
+saying
+says
+se
+second
+secondly
+see
+seeing
+seem
+seemed
+seeming
+seems
+seen
+self
+selves
+sensible
+sent
+serious
+seriously
+seven
+several
+shall
+shan
+shant
+shan't
+she
+she's
+should
+shouldn
+shouldnt
+shouldn't
+should've
+si
+sir
+sir.
+since
+six
+so
+soch
+some
+somebody
+somehow
+someone
+something
+sometime
+sometimes
+somewhat
+somewhere
+soon
+still
+sub
+such
+sup
+sure
+t
+tab
+tabh
+tak
+take
+taken
+tarah
+teen
+teeno
+teesra
+teesre
+teesri
+tell
+tends
+tera
+tere
+teri
+th
+tha
+than
+thank
+thanks
+thanx
+that
+that'll
+thats
+that's
+the
+theek
+their
+theirs
+them
+themselves
+then
+thence
+there
+thereafter
+thereby
+therefore
+therein
+theres
+there's
+thereupon
+these
+they
+they'd
+they'll
+they're
+they've
+thi
+thik
+thing
+think
+thinking
+third
+this
+tho
+thoda
+thodi
+thorough
+thoroughly
+those
+though
+thought
+three
+through
+throughout
+thru
+thus
+tjhe
+to
+together
+toh
+too
+took
+toward
+towards
+tried
+tries
+true
+truly
+try
+trying
+tu
+tujhe
+tum
+tumhara
+tumhare
+tumhari
+tune
+twice
+two
+um
+umm
+un
+under
+unhe
+unhi
+unho
+unhone
+unka
+unkaa
+unke
+unki
+unko
+unless
+unlikely
+unn
+unse
+until
+unto
+up
+upar
+upon
+us
+use
+used
+useful
+uses
+usi
+using
+uska
+uske
+usne
+uss
+usse
+ussi
+usually
+vaala
+vaale
+vaali
+vahaan
+vahan
+vahi
+vahin
+vaisa
+vaise
+vaisi
+vala
+vale
+vali
+various
+ve
+very
+via
+viz
+vo
+waala
+waale
+waali
+wagaira
+wagairah
+wagerah
+waha
+wahaan
+wahan
+wahi
+wahin
+waisa
+waise
+waisi
+wala
+wale
+wali
+want
+wants
+was
+wasn
+wasnt
+wasn't
+way
+we
+we'd
+well
+we'll
+went
+were
+we're
+weren
+werent
+weren't
+we've
+what
+whatever
+what's
+when
+whence
+whenever
+where
+whereafter
+whereas
+whereby
+wherein
+where's
+whereupon
+wherever
+whether
+which
+while
+who
+whoever
+whole
+whom
+who's
+whose
+why
+will
+willing
+with
+within
+without
+wo
+woh
+wohi
+won
+wont
+won't
+would
+wouldn
+wouldnt
+wouldn't
+y
+ya
+yadi
+yah
+yaha
+yahaan
+yahan
+yahi
+yahin
+ye
+yeah
+yeh
+yehi
+yes
+yet
+you
+you'd
+you'll
+your
+you're
+yours
+yourself
+yourselves
+you've
+yup
+,
+kia
+diya
+dia
+=
+{
++
+:
+;
+.
++=
+??
+<
+>
+}