init

app.py

@@ -0,0 +1,431 @@
+import sys
+
+from pathlib import Path
+import string
+import random
+import torch
+import numpy as np
+import pickle
+
+import gradio as gr
+import pandas as pd
+from scipy.special import softmax
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+import hydra
+from omegaconf import open_dict, DictConfig
+import matplotlib.pyplot as plt
+import matplotlib
+from matplotlib.patches import Patch
+sns.set()
+sns.set_style("darkgrid")
+
+from utils.data import *
+from utils.metrics import *
+
+
+
+def user_interface(Ufile, Pfile, Sfile=None, job_meta_file=None, user_meta_file=None, user_groups=None):
+    recdata = Data(Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups)
+
+
+    def calculate_user_item_metrics(res, S, U, k=10):
+        # get rec
+        m, n = res.shape
+        if not torch.is_tensor(res):
+            res = torch.from_numpy(res)
+        if not torch.is_tensor(U):
+            U = torch.from_numpy(U)
+        _, rec = torch.topk(res, k, dim=1)
+        rec_onehot = slow_onehot(rec, res)
+        # rec_onehot = F.one_hot(rec, num_classes=n).sum(1).float()
+        try:
+            rec_per_job = rec_onehot.sum(axis=0).numpy()
+        except:
+            rec_per_job = rec_onehot.sum(axis=0).cpu().numpy()
+            rec = rec.cpu()
+            S = S.cpu()
+        # envy
+        envy = expected_envy_torch_vec(U, rec_onehot, k=1).numpy()
+
+        # competitors for each rec job
+        competitors = get_competitors(rec_per_job, rec)
+
+        # rank
+        better_competitors = get_num_better_competitors(rec, S)
+
+        # scores per job for later zoom in scores
+        scores = get_scores_per_job(rec, S)
+
+        return {'rec': rec, 'envy': envy, 'competitors': competitors, 'ranks': better_competitors, 'scores_job': scores}
+
+
+    def plot_user_envy(user=0, k=2):
+        plt.close('all')
+        user = int(user)
+        if k in recdata.lookup_dict:
+            ret_dict = recdata.lookup_dict[k]
+        else:
+            ret_dict = calculate_user_item_metrics(recdata.P_sub, recdata.S_sub, recdata.U_sub, k=k)
+            recdata.lookup_dict[k] = ret_dict
+        # user's recommended jobs
+        users_rec = ret_dict['rec'][user].numpy()
+        # Plot
+        fig, ax1 = plt.subplots(figsize=(10, 5))
+        # fig.tight_layout()
+        fig.subplots_adjust(bottom=0.2)
+
+        envy = ret_dict['envy'].sum(-1)
+        envy_user = envy[user]
+        # plot envy histogram
+        n, bins, patches = ax1.hist(envy, bins=50, color='grey', alpha=0.5)
+        ax1.set_yscale('symlog')
+        sns.kdeplot(envy, color='grey', bw_adjust=0.3, cut=0, ax=ax1)
+        # mark this user's envy
+        # index of the bin that contains this user's envy
+        idx = np.digitize(envy_user, bins)
+        # print(envy_user, idx)
+        patches[idx-1].set_fc('r')
+        ax1.legend(handles=[Patch(facecolor='r', edgecolor='r', alpha=0.5,
+                         label='Your envy group')])
+        ax1.set_xlabel('Envy')
+        ax1.set_ylabel('Number of users (log scale)')
+        
+        return fig
+
+    def plot_user_scores(user=0, k=2):
+        user = int(user)
+        if k in recdata.lookup_dict:
+            ret_dict = recdata.lookup_dict[k]
+        else:
+            ret_dict = calculate_user_item_metrics(recdata.P_sub, recdata.S_sub, recdata.U_sub, k=k)
+            recdata.lookup_dict[k] = ret_dict
+        users_rec = ret_dict['rec'][user].numpy()
+        scores = ret_dict['scores_job']
+
+        # scores = [softmax(np.array(scores[jb])*0.5) for jb in users_rec]
+        scores = [scores[jb] for jb in users_rec]
+
+        rank_xs = [list(range(1, len(s)+1)) for s in scores]
+        my_ranks = [1+int(i) for i in ret_dict['ranks'][user]]
+        # my scores are the scores of the recommended jobs with rank
+        # my_scores = [scores[i][j] for i, j in enumerate(my_ranks)]
+        my_scores = [recdata.S_sub[user, job_id].item() for job_id in users_rec]
+        # my_scores_log = np.log(np.array(my_scores).astype(float))
+        ys = np.arange(len(users_rec))
+        # user's recommended jobs
+        if (user, k) in recdata.user_temp_data:
+            df = recdata.user_temp_data[(user, k)]
+        else:
+            df = pd.DataFrame({'x': rank_xs, 's': scores, 'y': ys})
+            df = df.explode(list('xs'))
+            recdata.user_temp_data[(user, k)] = df
+
+        # df['log_scores'] = np.log(df['s'].values.astype(float))
+        fig, ax = plt.subplots(figsize=(10, 5))
+        # fig.tight_layout()
+        fig.subplots_adjust(bottom=0.3)
+
+        def sub_cmap(cmap, vmin, vmax):
+            return lambda v: cmap(vmin + (vmax - vmin) * v)
+
+        # palette=matplotlib.cm.get_cmap('Greens').reversed()
+        # palette = sub_cmap(palette,0.2, 0.8)
+
+        sns.scatterplot(data=df, x="y", y="s", ax=ax, alpha=0.6,
+                        legend=False, s=100, hue='y', palette="summer") #monotone color palette
+        sns.scatterplot(y=my_scores, x=range(k), ax=ax,
+                        alpha=0.8, s=200, ec='r', fc='none', label='Your rank')
+        # add ranking of this user's score for each job
+        # find score gaps
+        gaps = np.diff(np.sort(scores[0])).mean()
+        for i, (y, x) in enumerate(zip(my_scores, range(k))):
+            ax.text(x-0.3, y+gaps, my_ranks[i], color='r', fontsize=15)
+        # add notation for 'rank'
+        # ax.text(-0.8, 1.12, 'Your rank', color='r', fontsize=12)
+        ax.set_xticks(range(k))
+        # shorten the job title
+        titles = [recdata.job_metadata[jb] for jb in users_rec]
+        titles = [t[:20] + '...' if len(t) > 20 else t for t in titles]
+        ax.set_xticklabels(titles, rotation=30, ha='right')
+        ax.set_xlabel('')
+        ax.set_xlim(-1, k)
+        # ax.grid(False)
+        ax.set_ylabel('Score')
+        # ax.set_ylim(-0.09, 1.2)
+        ax.legend()
+        return fig
+
+
+    # demo = gr.Blocks(gr.themes.Base.from_hub('finlaymacklon/smooth_slate'))
+    demo = gr.Blocks(gr.themes.Soft())
+    with demo:
+        def submit0(user, k):
+            fig = plot_user_envy(user, k)
+            return {
+                hist_plot: gr.update(value=fig, visible=True),
+            }
+
+
+        def submit2(user, k):
+            bar = plot_user_scores(user, k)
+            return {
+               bar_plot2: gr.update(value=bar, visible=True)
+            }
+        
+        def submit(user):
+            new_job_num = random.randint(1,6)
+            # if new_job_num == 0, do nothing but clear the plots
+            if new_job_num > 0:
+                print(f'adding {new_job_num} new jobs')
+                recdata.update(new_user_num=0, new_job_num=new_job_num)
+                recdata.tweak_P(user)
+                
+            return {
+                hist_plot: gr.update(visible=False),
+                bar_plot2: gr.update(visible=False)
+            }   
+
+        # def submit_login(user):
+        #     return {
+        #         k: gr.update(visible=True),
+        #         btn: gr.update(visible=True),
+        #         btn0: gr.update(visible=True),
+        #         btn2: gr.update(visible=True),
+        #         pswd: gr.update(visible=False),
+        #         lgbtn: gr.update(visible=False),
+        #     }
+
+
+        # layout
+        gr.Markdown("## Job Recommendation Inferiority and Envy Monitor Demo")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                user = gr.Textbox(label='User ID',default='0', placeholder='Enter a random integer user ID')
+        #     with gr.Column(scale=1):                
+        #         pswd = gr.Textbox(label='Password',default='********')
+        #     with gr.Column(scale=1):
+        #         lgbtn = gr.Button("Login")
+        # with gr.Row():
+            with gr.Column(scale=1):
+                k = gr.Slider(minimum=1, maximum=20,
+                                 default=4, step=1, label='Number of Jobs', visible=True)
+            with gr.Column(scale=1):
+                btn = gr.Button("Refresh to see new jobs", visible=True)
+            
+        with gr.Tab('Envy'):
+            btn0 = gr.Button("User envy distribution", visible=True)
+            hist_plot = gr.Plot(visible=False)
+
+        with gr.Tab('Inferiority'):
+            with gr.Row():
+                # btn1 = gr.Button("User ranks for the recommended jobs")
+                btn2 = gr.Button("User scores/ranks for the recommended jobs", visible=True)
+
+            # bar_plot = gr.Plot()
+            bar_plot2 = gr.Plot(visible=False)
+
+        # lgbtn.click(submit_login, inputs=[user], outputs=[k, btn, btn0, btn2, pswd, lgbtn])
+        btn.click(submit, inputs=[user], outputs=[hist_plot, bar_plot2])
+        btn0.click(submit0, inputs=[user, k], outputs=[hist_plot])
+        # btn1.click(submit1, inputs=[user, k], outputs=[bar_plot])
+        btn2.click(submit2, inputs=[user, k], outputs=[bar_plot2])
+
+    return demo
+
+
+def developer_interface(Ufile, Pfile, Sfile=None, job_meta_file=None, user_meta_file=None, user_groups=None):
+
+    recdata = Data(Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups, sub_sample_size=500)
+
+    def calculate_all_metrics(k, S_sub, U_sub, P_sub):
+        print('calculating all metrics')
+        if k in recdata.lookup_dict:
+            print('Found in lookup dict')
+            return recdata.lookup_dict[k]
+        else:
+            if not torch.is_tensor(P_sub):
+                P_sub = torch.from_numpy(P_sub)
+            envy, inferiority, utility = eiu_cut_off2(
+                (S_sub, U_sub), P_sub, k=k, agg=False)
+            envy = envy.sum(-1)
+            inferiority = inferiority.sum(-1)
+
+            _, rec = torch.topk(P_sub, k=k, dim=1)
+            rec_onehot = slow_onehot(rec, P_sub)
+            try:
+                rec_per_job = rec_onehot.sum(axis=0).numpy()
+            except:
+                rec_per_job = rec_onehot.sum(axis=0).cpu().numpy()
+                rec = rec.cpu()
+            metrics_at_k = {'rec': rec, 'envy': envy, 'inferiority': inferiority, 'utility': utility,
+                            'rec_per_job': rec_per_job}
+            print('Finished calculating all metrics')
+            return metrics_at_k
+
+    def plot_user_box(metrics_dict):
+        print('plotting user box')
+        plt.close('all')
+        envy = metrics_dict['envy'].numpy()
+        inferiority = metrics_dict['inferiority'].numpy()
+        fig, (ax1, ax2) = plt.subplots(ncols=2)
+        fig.tight_layout()
+        ax1.boxplot(envy)
+        ax1.set_ylabel('envy')
+        ax1.set_title('Envy')
+        ax1.set_xticks([])
+        ax2.boxplot(inferiority)
+        ax2.set_ylabel('inferiority')
+        ax2.set_title('Inferiority')
+        ax2.set_xticks([])
+        return fig
+
+    def plot_scatter(k, group=None):
+        print('plotting scatter')
+        plt.close('all')
+        if group == 'None':
+            group = None
+        if k in recdata.lookup_dict:
+            metrics_dict = recdata.lookup_dict[k]
+        else:
+            metrics_dict = calculate_all_metrics(k, recdata.S_sub, recdata.U_sub, recdata.P_sub)
+            recdata.lookup_dict[k] = metrics_dict
+
+        data = {'log(envy+1)': np.log(metrics_dict['envy']+1),
+                'inferiority': metrics_dict['inferiority']}
+        data = pd.DataFrame(data)
+        data = data.join(recdata.user_metadata)
+        fig, ax = plt.subplots()
+        sns.scatterplot(data=data, x='log(envy+1)', y='inferiority', hue=group, ax=ax)
+        return fig
+    
+    def lorenz_curve(X, ax, label):
+        # ref: https://zhiyzuo.github.io/Plot-Lorenz/
+        X.sort()
+        X_lorenz = X.cumsum() / X.sum()
+        X_lorenz = np.insert(X_lorenz, 0, 0)
+        X_lorenz[0], X_lorenz[-1]
+
+        ax.plot(np.arange(X_lorenz.size) / (X_lorenz.size - 1), X_lorenz, label=label)
+        ## line plot of equality
+        ax.plot([0, 1], [0, 1], linestyle='dashed', color='k')
+        return ax
+
+    def plot_item(rec_per_job):
+        print('plotting item') 
+        plt.close('all')
+        fig, (ax1, ax2) = plt.subplots(nrows=2, figsize=(10, 10))
+        fig.tight_layout(pad=5.0)
+        labels, counts = np.unique(rec_per_job, return_counts=True)
+        ax1.bar(labels, counts, align='center')
+
+        ax1.set_xlabel('Number of times a job is recommended')
+        ax1.set_ylabel('Number of jobs')
+        ax1.set_title('Distribution of job exposure')
+        ax2 = lorenz_curve(rec_per_job, ax2,'')
+        ax2.set_title('Lorenz Curve')
+        return fig
+    
+
+    # build the interface
+    demo = gr.Blocks(gr.themes.Soft())
+    with demo:
+        # callbacks
+        def submit_u():
+            # generate two random integers including 0 representing user num and job num
+            user_num = np.random.randint(0, 5)
+            job_num = np.random.randint(0, 5)
+            
+            if user_num > 0 or job_num > 0:
+                recdata.update(user_num, job_num)
+
+            return{
+                info: gr.update(value='New {} users and {} jobs'.format(user_num, job_num),visible=True),
+            }      
+
+
+        def submit1(k):
+            metrics_dict = calculate_all_metrics(k, recdata.S_sub, recdata.U_sub, recdata.P_sub)
+            return {
+                user_box_plot: plot_user_box(metrics_dict),
+                scatter_plot: plot_scatter(k),
+                btn2: gr.update(visible=True)
+            }
+
+        def submit2():
+            return {
+                radio: gr.update(visible=True)
+            }
+
+        def submit3(k):
+            metrics_dict = calculate_all_metrics(k, recdata.S_sub, recdata.U_sub, recdata.P_sub)
+            return {
+                item_plots: plot_item(metrics_dict['rec_per_job'])
+            }
+
+        # layout
+        gr.Markdown("## Envy & Inferiority Monitor for Developers Demo")
+        # 1. accept k
+        with gr.Row():
+            with gr.Column(scale=1):
+                k = gr.inputs.Slider(minimum=1, maximum=min(30,len(
+                    recdata.P[0])), default=1, step=1, label='Number of Jobs')
+            with gr.Column(scale=1):
+                btn = gr.Button('Refresh')
+            with gr.Column(scale=1):
+                info = gr.Textbox('', label='Updated info', visible=False)
+            btn.click(submit_u, inputs=[], outputs=[info])
+
+        
+        with gr.Tab('User'):
+            plt.close('all')
+            btn1 = gr.Button('Visualize user-side fairness')
+            user_box_plot = gr.Plot()
+            scatter_plot = gr.Plot()
+
+            btn2 = gr.Button('Visualize intra-group fairness', visible=False)
+
+            radio = gr.Radio(choices=user_groups, value=user_groups[0] if len(user_groups) > 0 else "",
+                             interactive=True, label="User group", visible=False)
+
+            btn1.click(submit1, inputs=[k], outputs=[
+                       user_box_plot, scatter_plot, btn2])
+            btn2.click(submit2, inputs=[], outputs=[radio])
+            radio.change(fn=plot_scatter, inputs=[
+                         k, radio], outputs=[scatter_plot])
+
+        with gr.Tab('Item'):
+            plt.close('all')
+            btn3 = gr.Button('Visualize item-side fairness')
+            item_plots = gr.Plot()
+            btn3.click(submit3, inputs=[k], outputs=[item_plots])
+
+    return demo
+
+
+@hydra.main(version_base=None, config_path='./utils', config_name='monitor')
+def main(config: DictConfig):
+    print(config)
+    Ufile = config.Ufile
+    Sfile = config.Sfile
+    Pfile = config.Pfile
+    user_meta_file = config.user_meta_file
+    job_meta_file = config.job_meta_file
+    user_groups = ['None'] + \
+        list(config.user_groups) if config.user_groups else ['None']
+    server_name = config.server_name
+    role = config.role
+    if role == 'user':
+        demo = user_interface(Ufile, Pfile, Sfile,
+                              job_meta_file, user_meta_file, user_groups)
+    elif role == 'developer':
+        demo = developer_interface(
+            Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups)
+    demo.launch(server_name=server_name, server_port=config.server_port)
+    # demo.launch()
+
+
+if __name__ == "__main__":
+    main()