init

.gitignore

@@ -0,0 +1,136 @@
+# Datafiles
+**/outputs/
+**/data/
+note.md
+
+
+# Byte-compiled / optimized / DLL files
+.idea/
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+.python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/

app.py

@@ -1,7 +1,431 @@
+import sys
+
+from pathlib import Path
+import string
+import random
+import torch
+import numpy as np
+import pickle
+
 import gradio as gr
+import pandas as pd
+from scipy.special import softmax
+import numpy as np
+import seaborn as sns
+import matplotlib.pyplot as plt
+import hydra
+from omegaconf import open_dict, DictConfig
+import matplotlib.pyplot as plt
+import matplotlib
+from matplotlib.patches import Patch
+sns.set()
+sns.set_style("darkgrid")
+
+from utils.data import *
+from utils.metrics import *
+
+
+
+def user_interface(Ufile, Pfile, Sfile=None, job_meta_file=None, user_meta_file=None, user_groups=None):
+    recdata = Data(Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups)
+
+
+    def calculate_user_item_metrics(res, S, U, k=10):
+        # get rec
+        m, n = res.shape
+        if not torch.is_tensor(res):
+            res = torch.from_numpy(res)
+        if not torch.is_tensor(U):
+            U = torch.from_numpy(U)
+        _, rec = torch.topk(res, k, dim=1)
+        rec_onehot = slow_onehot(rec, res)
+        # rec_onehot = F.one_hot(rec, num_classes=n).sum(1).float()
+        try:
+            rec_per_job = rec_onehot.sum(axis=0).numpy()
+        except:
+            rec_per_job = rec_onehot.sum(axis=0).cpu().numpy()
+            rec = rec.cpu()
+            S = S.cpu()
+        # envy
+        envy = expected_envy_torch_vec(U, rec_onehot, k=1).numpy()
+
+        # competitors for each rec job
+        competitors = get_competitors(rec_per_job, rec)
+
+        # rank
+        better_competitors = get_num_better_competitors(rec, S)
+
+        # scores per job for later zoom in scores
+        scores = get_scores_per_job(rec, S)
+
+        return {'rec': rec, 'envy': envy, 'competitors': competitors, 'ranks': better_competitors, 'scores_job': scores}
+
+
+    def plot_user_envy(user=0, k=2):
+        plt.close('all')
+        user = int(user)
+        if k in recdata.lookup_dict:
+            ret_dict = recdata.lookup_dict[k]
+        else:
+            ret_dict = calculate_user_item_metrics(recdata.P_sub, recdata.S_sub, recdata.U_sub, k=k)
+            recdata.lookup_dict[k] = ret_dict
+        # user's recommended jobs
+        users_rec = ret_dict['rec'][user].numpy()
+        # Plot
+        fig, ax1 = plt.subplots(figsize=(10, 5))
+        # fig.tight_layout()
+        fig.subplots_adjust(bottom=0.2)
+
+        envy = ret_dict['envy'].sum(-1)
+        envy_user = envy[user]
+        # plot envy histogram
+        n, bins, patches = ax1.hist(envy, bins=50, color='grey', alpha=0.5)
+        ax1.set_yscale('symlog')
+        sns.kdeplot(envy, color='grey', bw_adjust=0.3, cut=0, ax=ax1)
+        # mark this user's envy
+        # index of the bin that contains this user's envy
+        idx = np.digitize(envy_user, bins)
+        # print(envy_user, idx)
+        patches[idx-1].set_fc('r')
+        ax1.legend(handles=[Patch(facecolor='r', edgecolor='r', alpha=0.5,
+                         label='Your envy group')])
+        ax1.set_xlabel('Envy')
+        ax1.set_ylabel('Number of users (log scale)')
+        
+        return fig
+
+    def plot_user_scores(user=0, k=2):
+        user = int(user)
+        if k in recdata.lookup_dict:
+            ret_dict = recdata.lookup_dict[k]
+        else:
+            ret_dict = calculate_user_item_metrics(recdata.P_sub, recdata.S_sub, recdata.U_sub, k=k)
+            recdata.lookup_dict[k] = ret_dict
+        users_rec = ret_dict['rec'][user].numpy()
+        scores = ret_dict['scores_job']
+
+        # scores = [softmax(np.array(scores[jb])*0.5) for jb in users_rec]
+        scores = [scores[jb] for jb in users_rec]
+
+        rank_xs = [list(range(1, len(s)+1)) for s in scores]
+        my_ranks = [1+int(i) for i in ret_dict['ranks'][user]]
+        # my scores are the scores of the recommended jobs with rank
+        # my_scores = [scores[i][j] for i, j in enumerate(my_ranks)]
+        my_scores = [recdata.S_sub[user, job_id].item() for job_id in users_rec]
+        # my_scores_log = np.log(np.array(my_scores).astype(float))
+        ys = np.arange(len(users_rec))
+        # user's recommended jobs
+        if (user, k) in recdata.user_temp_data:
+            df = recdata.user_temp_data[(user, k)]
+        else:
+            df = pd.DataFrame({'x': rank_xs, 's': scores, 'y': ys})
+            df = df.explode(list('xs'))
+            recdata.user_temp_data[(user, k)] = df
+
+        # df['log_scores'] = np.log(df['s'].values.astype(float))
+        fig, ax = plt.subplots(figsize=(10, 5))
+        # fig.tight_layout()
+        fig.subplots_adjust(bottom=0.3)
+
+        def sub_cmap(cmap, vmin, vmax):
+            return lambda v: cmap(vmin + (vmax - vmin) * v)
+
+        # palette=matplotlib.cm.get_cmap('Greens').reversed()
+        # palette = sub_cmap(palette,0.2, 0.8)
+
+        sns.scatterplot(data=df, x="y", y="s", ax=ax, alpha=0.6,
+                        legend=False, s=100, hue='y', palette="summer") #monotone color palette
+        sns.scatterplot(y=my_scores, x=range(k), ax=ax,
+                        alpha=0.8, s=200, ec='r', fc='none', label='Your rank')
+        # add ranking of this user's score for each job
+        # find score gaps
+        gaps = np.diff(np.sort(scores[0])).mean()
+        for i, (y, x) in enumerate(zip(my_scores, range(k))):
+            ax.text(x-0.3, y+gaps, my_ranks[i], color='r', fontsize=15)
+        # add notation for 'rank'
+        # ax.text(-0.8, 1.12, 'Your rank', color='r', fontsize=12)
+        ax.set_xticks(range(k))
+        # shorten the job title
+        titles = [recdata.job_metadata[jb] for jb in users_rec]
+        titles = [t[:20] + '...' if len(t) > 20 else t for t in titles]
+        ax.set_xticklabels(titles, rotation=30, ha='right')
+        ax.set_xlabel('')
+        ax.set_xlim(-1, k)
+        # ax.grid(False)
+        ax.set_ylabel('Score')
+        # ax.set_ylim(-0.09, 1.2)
+        ax.legend()
+        return fig
+
+
+    # demo = gr.Blocks(gr.themes.Base.from_hub('finlaymacklon/smooth_slate'))
+    demo = gr.Blocks(gr.themes.Soft())
+    with demo:
+        def submit0(user, k):
+            fig = plot_user_envy(user, k)
+            return {
+                hist_plot: gr.update(value=fig, visible=True),
+            }
+
+
+        def submit2(user, k):
+            bar = plot_user_scores(user, k)
+            return {
+               bar_plot2: gr.update(value=bar, visible=True)
+            }
+        
+        def submit(user):
+            new_job_num = random.randint(1,6)
+            # if new_job_num == 0, do nothing but clear the plots
+            if new_job_num > 0:
+                print(f'adding {new_job_num} new jobs')
+                recdata.update(new_user_num=0, new_job_num=new_job_num)
+                recdata.tweak_P(user)
+                
+            return {
+                hist_plot: gr.update(visible=False),
+                bar_plot2: gr.update(visible=False)
+            }   
+
+        # def submit_login(user):
+        #     return {
+        #         k: gr.update(visible=True),
+        #         btn: gr.update(visible=True),
+        #         btn0: gr.update(visible=True),
+        #         btn2: gr.update(visible=True),
+        #         pswd: gr.update(visible=False),
+        #         lgbtn: gr.update(visible=False),
+        #     }
+
+
+        # layout
+        gr.Markdown("## Job Recommendation Inferiority and Envy Monitor Demo")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                user = gr.Textbox(label='User ID',default='0', placeholder='Enter a random integer user ID')
+        #     with gr.Column(scale=1):                
+        #         pswd = gr.Textbox(label='Password',default='********')
+        #     with gr.Column(scale=1):
+        #         lgbtn = gr.Button("Login")
+        # with gr.Row():
+            with gr.Column(scale=1):
+                k = gr.Slider(minimum=1, maximum=20,
+                                 default=4, step=1, label='Number of Jobs', visible=True)
+            with gr.Column(scale=1):
+                btn = gr.Button("Refresh to see new jobs", visible=True)
+            
+        with gr.Tab('Envy'):
+            btn0 = gr.Button("User envy distribution", visible=True)
+            hist_plot = gr.Plot(visible=False)
+
+        with gr.Tab('Inferiority'):
+            with gr.Row():
+                # btn1 = gr.Button("User ranks for the recommended jobs")
+                btn2 = gr.Button("User scores/ranks for the recommended jobs", visible=True)
+
+            # bar_plot = gr.Plot()
+            bar_plot2 = gr.Plot(visible=False)
+
+        # lgbtn.click(submit_login, inputs=[user], outputs=[k, btn, btn0, btn2, pswd, lgbtn])
+        btn.click(submit, inputs=[user], outputs=[hist_plot, bar_plot2])
+        btn0.click(submit0, inputs=[user, k], outputs=[hist_plot])
+        # btn1.click(submit1, inputs=[user, k], outputs=[bar_plot])
+        btn2.click(submit2, inputs=[user, k], outputs=[bar_plot2])
+
+    return demo
+
+
+def developer_interface(Ufile, Pfile, Sfile=None, job_meta_file=None, user_meta_file=None, user_groups=None):
+
+    recdata = Data(Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups, sub_sample_size=500)
+
+    def calculate_all_metrics(k, S_sub, U_sub, P_sub):
+        print('calculating all metrics')
+        if k in recdata.lookup_dict:
+            print('Found in lookup dict')
+            return recdata.lookup_dict[k]
+        else:
+            if not torch.is_tensor(P_sub):
+                P_sub = torch.from_numpy(P_sub)
+            envy, inferiority, utility = eiu_cut_off2(
+                (S_sub, U_sub), P_sub, k=k, agg=False)
+            envy = envy.sum(-1)
+            inferiority = inferiority.sum(-1)
+
+            _, rec = torch.topk(P_sub, k=k, dim=1)
+            rec_onehot = slow_onehot(rec, P_sub)
+            try:
+                rec_per_job = rec_onehot.sum(axis=0).numpy()
+            except:
+                rec_per_job = rec_onehot.sum(axis=0).cpu().numpy()
+                rec = rec.cpu()
+            metrics_at_k = {'rec': rec, 'envy': envy, 'inferiority': inferiority, 'utility': utility,
+                            'rec_per_job': rec_per_job}
+            print('Finished calculating all metrics')
+            return metrics_at_k
+
+    def plot_user_box(metrics_dict):
+        print('plotting user box')
+        plt.close('all')
+        envy = metrics_dict['envy'].numpy()
+        inferiority = metrics_dict['inferiority'].numpy()
+        fig, (ax1, ax2) = plt.subplots(ncols=2)
+        fig.tight_layout()
+        ax1.boxplot(envy)
+        ax1.set_ylabel('envy')
+        ax1.set_title('Envy')
+        ax1.set_xticks([])
+        ax2.boxplot(inferiority)
+        ax2.set_ylabel('inferiority')
+        ax2.set_title('Inferiority')
+        ax2.set_xticks([])
+        return fig
+
+    def plot_scatter(k, group=None):
+        print('plotting scatter')
+        plt.close('all')
+        if group == 'None':
+            group = None
+        if k in recdata.lookup_dict:
+            metrics_dict = recdata.lookup_dict[k]
+        else:
+            metrics_dict = calculate_all_metrics(k, recdata.S_sub, recdata.U_sub, recdata.P_sub)
+            recdata.lookup_dict[k] = metrics_dict
+
+        data = {'log(envy+1)': np.log(metrics_dict['envy']+1),
+                'inferiority': metrics_dict['inferiority']}
+        data = pd.DataFrame(data)
+        data = data.join(recdata.user_metadata)
+        fig, ax = plt.subplots()
+        sns.scatterplot(data=data, x='log(envy+1)', y='inferiority', hue=group, ax=ax)
+        return fig
+    
+    def lorenz_curve(X, ax, label):
+        # ref: https://zhiyzuo.github.io/Plot-Lorenz/
+        X.sort()
+        X_lorenz = X.cumsum() / X.sum()
+        X_lorenz = np.insert(X_lorenz, 0, 0)
+        X_lorenz[0], X_lorenz[-1]
+
+        ax.plot(np.arange(X_lorenz.size) / (X_lorenz.size - 1), X_lorenz, label=label)
+        ## line plot of equality
+        ax.plot([0, 1], [0, 1], linestyle='dashed', color='k')
+        return ax
+
+    def plot_item(rec_per_job):
+        print('plotting item') 
+        plt.close('all')
+        fig, (ax1, ax2) = plt.subplots(nrows=2, figsize=(10, 10))
+        fig.tight_layout(pad=5.0)
+        labels, counts = np.unique(rec_per_job, return_counts=True)
+        ax1.bar(labels, counts, align='center')
+
+        ax1.set_xlabel('Number of times a job is recommended')
+        ax1.set_ylabel('Number of jobs')
+        ax1.set_title('Distribution of job exposure')
+        ax2 = lorenz_curve(rec_per_job, ax2,'')
+        ax2.set_title('Lorenz Curve')
+        return fig
+    
+
+    # build the interface
+    demo = gr.Blocks(gr.themes.Soft())
+    with demo:
+        # callbacks
+        def submit_u():
+            # generate two random integers including 0 representing user num and job num
+            user_num = np.random.randint(0, 5)
+            job_num = np.random.randint(0, 5)
+            
+            if user_num > 0 or job_num > 0:
+                recdata.update(user_num, job_num)
+
+            return{
+                info: gr.update(value='New {} users and {} jobs'.format(user_num, job_num),visible=True),
+            }      
+
+
+        def submit1(k):
+            metrics_dict = calculate_all_metrics(k, recdata.S_sub, recdata.U_sub, recdata.P_sub)
+            return {
+                user_box_plot: plot_user_box(metrics_dict),
+                scatter_plot: plot_scatter(k),
+                btn2: gr.update(visible=True)
+            }
+
+        def submit2():
+            return {
+                radio: gr.update(visible=True)
+            }
+
+        def submit3(k):
+            metrics_dict = calculate_all_metrics(k, recdata.S_sub, recdata.U_sub, recdata.P_sub)
+            return {
+                item_plots: plot_item(metrics_dict['rec_per_job'])
+            }
+
+        # layout
+        gr.Markdown("## Envy & Inferiority Monitor for Developers Demo")
+        # 1. accept k
+        with gr.Row():
+            with gr.Column(scale=1):
+                k = gr.inputs.Slider(minimum=1, maximum=min(30,len(
+                    recdata.P[0])), default=1, step=1, label='Number of Jobs')
+            with gr.Column(scale=1):
+                btn = gr.Button('Refresh')
+            with gr.Column(scale=1):
+                info = gr.Textbox('', label='Updated info', visible=False)
+            btn.click(submit_u, inputs=[], outputs=[info])
+
+        
+        with gr.Tab('User'):
+            plt.close('all')
+            btn1 = gr.Button('Visualize user-side fairness')
+            user_box_plot = gr.Plot()
+            scatter_plot = gr.Plot()
+
+            btn2 = gr.Button('Visualize intra-group fairness', visible=False)
+
+            radio = gr.Radio(choices=user_groups, value=user_groups[0] if len(user_groups) > 0 else "",
+                             interactive=True, label="User group", visible=False)
+
+            btn1.click(submit1, inputs=[k], outputs=[
+                       user_box_plot, scatter_plot, btn2])
+            btn2.click(submit2, inputs=[], outputs=[radio])
+            radio.change(fn=plot_scatter, inputs=[
+                         k, radio], outputs=[scatter_plot])
+
+        with gr.Tab('Item'):
+            plt.close('all')
+            btn3 = gr.Button('Visualize item-side fairness')
+            item_plots = gr.Plot()
+            btn3.click(submit3, inputs=[k], outputs=[item_plots])
+
+    return demo
+
+
+@hydra.main(version_base=None, config_path='./utils', config_name='monitor')
+def main(config: DictConfig):
+    print(config)
+    Ufile = config.Ufile
+    Sfile = config.Sfile
+    Pfile = config.Pfile
+    user_meta_file = config.user_meta_file
+    job_meta_file = config.job_meta_file
+    user_groups = ['None'] + \
+        list(config.user_groups) if config.user_groups else ['None']
+    server_name = config.server_name
+    role = config.role
+    if role == 'user':
+        demo = user_interface(Ufile, Pfile, Sfile,
+                              job_meta_file, user_meta_file, user_groups)
+    elif role == 'developer':
+        demo = developer_interface(
+            Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups)
+    demo.launch(server_name=server_name, server_port=config.server_port)
+    # demo.launch()
 
-def greet(name):
-    return "Hello " + name + "!!"
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()
+if __name__ == "__main__":
+    main()

fake_data/P.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf24f3cfccdc58272aad4bd3c81b55e6f9c527842f4f0f9cb9b8109b4957bbeb
+size 10158

fake_data/S.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d48ed2befd1b8a4039342b2545202488ccb4b0b8ddfff93cef9c12730a1db8e5
+size 10158

fake_data/U.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fbb03868752b74be97ad6f307cbfbf8b36c3de2d9eca5080a7d6f59890ab03d6
+size 10158

fake_data/user_metadata.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:737f77fccd84770a721abf1113222a370752c52cf72f7ef312cc5502128c659b
+size 1644

requirements.txt

@@ -0,0 +1,9 @@
+gradio==3.32.0
+hydra-core==1.3.2
+matplotlib==3.7.1
+numpy==1.23.5
+omegaconf==2.3.0
+pandas==1.5.3
+scipy==1.10.1
+seaborn==0.12.2
+torch==2.0.0

utils/data.py

@@ -0,0 +1,159 @@
+import sys
+
+from pathlib import Path
+import string
+import random
+import torch
+import numpy as np
+import pickle
+import pandas as pd
+import os
+import json
+import re
+
+
+
+# refactor as a class with the following methods
+class Data:
+    def __init__(self, Ufile, Pfile, Sfile, job_meta_file, user_meta_file, user_groups=None, sub_sample_size=1000):
+        self.U, self.P, self.S, self.job_metadata, self.job_metadata_reverse, self.user_metadata = self.load_data(Pfile, Sfile, Ufile, job_meta_file, user_meta_file)
+        # subsample the data
+        self.U_sub = self.sub_sample(self.U, sub_sample_size)
+        self.P_sub = self.sub_sample(self.P, sub_sample_size)
+        self.S_sub = self.sub_sample(self.S, sub_sample_size)
+        # self.U_sub = self.U
+        # self.P_sub = self.P
+        # self.S_sub = self.S
+        self.lookup_dict = {}
+        self.user_temp_data = {}
+        self.user_groups = user_groups
+
+
+    def load_data(self, Pfile, Sfile, Ufile, job_meta_file, user_meta_file):
+        U = torch.from_numpy(pickle.load(open(Ufile, 'rb')))
+        recommendations = torch.from_numpy(pickle.load(open(Pfile, 'rb')))
+        m, n = recommendations.shape
+        if Sfile:
+            S = torch.from_numpy(pickle.load(open(Sfile, 'rb')))
+        else:
+            S = U
+        if job_meta_file:
+            job_metadata = pickle.load(open(job_meta_file, 'rb'))
+        else:
+            job_metadata = {}
+            for i in range(n):
+                job_metadata[i] = 'Job {}'.format(i)
+        job_metadata_reverse = {v.capitalize(): k for k, v in job_metadata.items()}
+        if user_meta_file is not None:
+            user_metadata = pickle.load(open(user_meta_file, 'rb'))
+        else:
+            user_metadata = None
+
+        return U, recommendations, S, job_metadata, job_metadata_reverse, user_metadata
+    
+    def sub_sample(self, M, sample_size=500):
+        if len(M) > sample_size and len(M[0]) > sample_size:
+            # take the first sample_size columns and rows of M, copy without touching the original
+            M = M[:sample_size, :sample_size].clone()
+        return M
+    
+    def update(self, new_user_num, new_job_num):
+        # refactor this function
+        # recdata.lookup_dict = {}
+        # user_temp_data = {}
+        # U = add_jobs(U, new_job_num)
+        # recommendations = update_P(recommendations, new_job_num, 0)
+        # generate a random float between 0 and 1
+        # prob = random.random()
+        # if prob > 0.2:
+        #     recommendations[int(user),-1] = 1.
+        # S = add_jobs(S, new_job_num)
+        # U, recommendations, S = add_jobs(U, new_job_num), add_jobs(recommendations, new_job_num), add_jobs(S, new_job_num)
+        # job_metadata = update_job_metadata(job_metadata, new_job_num)
+        # job_metadata_reverse = {v: k for k, v in job_metadata.items()}
+        if new_job_num > 0 or new_user_num > 0:
+            self.U_sub = self.add_jobs_users(self.U_sub, self.U, new_job_num, new_user_num)
+            self.P_sub = self.add_jobs_users(self.P_sub, self.P, new_job_num, new_user_num)
+            self.S_sub = self.add_jobs_users(self.S_sub, self.S, new_job_num, new_user_num)
+            print('U_sub shape: ', self.U_sub.shape)
+            print('P_sub shape: ', self.P_sub.shape)
+            print('S_sub shape: ', self.S_sub.shape)
+            self.update_job_metadata(new_job_num)
+            self.update_user_metadata(new_user_num)
+            self.lookup_dict = {}
+            self.user_temp_data = {}
+
+
+
+    # def shuffle_rec(P):
+    #     rand_rec = P.copy()
+    #     rand_rec = rand_rec[:,np.random.permutation(rand_rec.shape[1])]
+    #     return rand_rec
+
+    def add_jobs(self, M_sub, M, new_job_num): # refactor this function, accept one matrix as input
+        if new_job_num == 0:
+            return M_sub
+        if len(M[0]) > len(M_sub[0]) + new_job_num:
+            M_updated = M[:len(M_sub), :len(M_sub[0]) + new_job_num].clone()
+        else:
+            # random number between 0 and 1 with size (S.shape[0],new_job_num)
+            new_jobM = np.random.rand(M.shape[0], new_job_num)
+            # concat new jobM to M as new columns
+            M_updated = np.concatenate((M_sub, new_jobM), axis=1)
+
+        return M_updated
+
+    def add_users(self, M_sub, M, new_user_num): # refactor this function, accept one matrix as input
+        if new_user_num == 0:
+            return M_sub
+        if len(M) > len(M_sub) + new_user_num:
+            M_updated = M[:len(M_sub) + new_user_num, :len(M_sub[0])].clone()
+        else:
+            # random number between 0 and 1 with size (new_user_num,S.shape[1])
+            new_userM = np.random.rand(new_user_num, M.shape[1])
+            # concat new userM to M as new rows
+            M_updated = np.concatenate((M_sub, new_userM), axis=0)
+
+        return M_updated
+
+
+    def add_jobs_users(self, M_sub, M, new_job_num, new_user_num):
+        # use add_jobs and add_users to add new jobs and users
+        M_updated = self.add_jobs(M_sub, M, new_job_num)
+        M_updated = self.add_users(M_updated, M, new_user_num)
+        print('M_updated shape: ', M_updated.shape)
+        return M_updated
+
+    def tweak_P(self, this_user):
+        # generate a random float between 0 and 1
+        prob = random.random()
+        if prob > 0.2:
+            self.P_sub[int(this_user),-1] = 1.
+            # 1 random indices of users within the range of P.shape[0]
+            user_indices = np.random.randint(0, self.P_sub.shape[0], 1)
+            self.P_sub[user_indices, -1] = 1.
+
+    def update_job_metadata(self, new_job_num):
+        if len(self.P_sub[0]) > len(self.P[0]):
+            for i in range(new_job_num):
+                self.job_metadata[len(self.job_metadata)] = 'Job {}'.format(len(self.job_metadata))
+                self.job_metadata_reverse['Job {}'.format(len(self.job_metadata_reverse))] = len(self.job_metadata_reverse)
+
+
+    def update_user_metadata(self, new_user_num): # TODO: generate fake user metadata for CB
+        if new_user_num > 0:
+            if len(self.P_sub) > len(self.P):
+                # make a new dataframe with new user metadata
+                new_user_metadata = {}
+                new_user_metadata['Id'] = [str(i) for i in range(len(self.user_metadata), len(self.user_metadata) + new_user_num)]
+                new_user_metadata['Sex'] = np.random.choice([0, 1], size=new_user_num, p=[.4, .6])
+                new_user_metadata['Edu'] = np.random.choice([0, 1, 2], size=new_user_num, p=[.2, .6, 0.2])
+                new_user_metadata = pd.DataFrame(new_user_metadata)
+                new_user_metadata['Sex'] = new_user_metadata['Sex'].map({0:'F', 1:'M'})
+                new_user_metadata['Edu'] = new_user_metadata['Edu'].map({0:'High school', 1:'College', 2:'Graduate+'})
+                # concat new user metadata to old user metadata
+                self.user_metadata = pd.concat([self.user_metadata, new_user_metadata], ignore_index=True)
+                # print(user_metadata)
+
+
+

utils/metrics.py

@@ -0,0 +1,606 @@
+import sys
+import torch
+import torch.nn.functional as F
+import numpy as np
+from collections import defaultdict
+
+np.set_printoptions(precision=4)
+from scipy.stats import rankdata
+
+
+"""Information Retrieval metrics
+Useful Resources:
+http://www.cs.utexas.edu/~mooney/ir-course/slides/Evaluation.ppt
+http://www.nii.ac.jp/TechReports/05-014E.pdf
+http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
+http://hal.archives-ouvertes.fr/docs/00/72/67/60/PDF/07-busa-fekete.pdf
+Learning to Rank for Information Retrieval (Tie-Yan Liu)
+"""
+
+
+def mean_reciprocal_rank(rs):
+    """Score is reciprocal of the rank of the first relevant item
+    First element is 'rank 1'.  Relevance is binary (nonzero is relevant).
+    Example from http://en.wikipedia.org/wiki/Mean_reciprocal_rank
+    >>> rs = [[0, 0, 1], [0, 1, 0], [1, 0, 0]]
+    >>> mean_reciprocal_rank(rs)
+    0.61111111111111105
+    >>> rs = np.array([[0, 0, 0], [0, 1, 0], [1, 0, 0]])
+    >>> mean_reciprocal_rank(rs)
+    0.5
+    >>> rs = [[0, 0, 0, 1], [1, 0, 0], [1, 0, 0]]
+    >>> mean_reciprocal_rank(rs)
+    0.75
+    Args:
+        rs: Iterator of relevance scores (list or numpy) in rank order
+            (first element is the first item)
+    Returns:
+        Mean reciprocal rank
+    """
+    rs = (np.asarray(r).nonzero()[0] for r in rs)
+    return np.mean([1. / (r[0] + 1) if r.size else 0. for r in rs])
+
+
+def r_precision(r):
+    """Score is precision after all relevant documents have been retrieved
+    Relevance is binary (nonzero is relevant).
+    >>> r = [0, 0, 1]
+    >>> r_precision(r)
+    0.33333333333333331
+    >>> r = [0, 1, 0]
+    >>> r_precision(r)
+    0.5
+    >>> r = [1, 0, 0]
+    >>> r_precision(r)
+    1.0
+    Args:
+        r: Relevance scores (list or numpy) in rank order
+            (first element is the first item)
+    Returns:
+        R Precision
+    """
+    r = np.asarray(r) != 0
+    z = r.nonzero()[0]
+    if not z.size:
+        return 0.
+    return np.mean(r[:z[-1] + 1])
+
+
+def precision_at_k(r, k):
+    """Score is precision @ k
+    Relevance is binary (nonzero is relevant).
+    >>> r = [0, 0, 1]
+    >>> precision_at_k(r, 1)
+    0.0
+    >>> precision_at_k(r, 2)
+    0.0
+    >>> precision_at_k(r, 3)
+    0.33333333333333331
+    >>> precision_at_k(r, 4)
+    Traceback (most recent call last):
+        File "<stdin>", line 1, in ?
+    ValueError: Relevance score length < k
+    Args:
+        r: Relevance scores (list or numpy) in rank order
+            (first element is the first item)
+    Returns:
+        Precision @ k
+    Raises:
+        ValueError: len(r) must be >= k
+    """
+    assert k >= 1
+    r = np.asarray(r)[:k] != 0
+    if r.size != k:
+        raise ValueError('Relevance score length < k')
+    return np.mean(r)
+
+
+def average_precision(r):
+    """Score is average precision (area under PR curve)
+    Relevance is binary (nonzero is relevant).
+    >>> r = [1, 1, 0, 1, 0, 1, 0, 0, 0, 1]
+    >>> delta_r = 1. / sum(r)
+    >>> sum([sum(r[:x + 1]) / (x + 1.) * delta_r for x, y in enumerate(r) if y])
+    0.7833333333333333
+    >>> average_precision(r)
+    0.78333333333333333
+    Args:
+        r: Relevance scores (list or numpy) in rank order
+            (first element is the first item)
+    Returns:
+        Average precision
+    """
+    r = np.asarray(r) != 0
+    out = [precision_at_k(r, k + 1) for k in range(r.size) if r[k]]
+    if not out:
+        return 0.
+    return np.mean(out)
+
+
+def mean_average_precision(rs):
+    """Score is mean average precision
+    Relevance is binary (nonzero is relevant).
+    >>> rs = [[1, 1, 0, 1, 0, 1, 0, 0, 0, 1]]
+    >>> mean_average_precision(rs)
+    0.78333333333333333
+    >>> rs = [[1, 1, 0, 1, 0, 1, 0, 0, 0, 1], [0]]
+    >>> mean_average_precision(rs)
+    0.39166666666666666
+    Args:
+        rs: Iterator of relevance scores (list or numpy) in rank order
+            (first element is the first item)
+    Returns:
+        Mean average precision
+    """
+    return np.mean([average_precision(r) for r in rs])
+
+
+def dcg_at_k(r, k, method=0):
+    """Score is discounted cumulative gain (dcg)
+    Relevance is positive real values.  Can use binary
+    as the previous methods.
+    Example from
+    http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
+    >>> r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]
+    >>> dcg_at_k(r, 1)
+    3.0
+    >>> dcg_at_k(r, 1, method=1)
+    3.0
+    >>> dcg_at_k(r, 2)
+    5.0
+    >>> dcg_at_k(r, 2, method=1)
+    4.2618595071429155
+    >>> dcg_at_k(r, 10)
+    9.6051177391888114
+    >>> dcg_at_k(r, 11)
+    9.6051177391888114
+    Args:
+        r: Relevance scores (list or numpy) in rank order
+            (first element is the first item)
+        k: Number of results to consider
+        method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]
+                If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]
+    Returns:
+        Discounted cumulative gain
+    """
+    r = np.asfarray(r)[:k]
+    if r.size:
+        if method == 0:
+            return r[0] + np.sum(r[1:] / np.log2(np.arange(2, r.size + 1)))
+        elif method == 1:
+            return np.sum(r / np.log2(np.arange(2, r.size + 2)))
+        else:
+            raise ValueError('method must be 0 or 1.')
+    return 0.
+
+
+def ndcg_at_k(r, k, method=0):
+    """Score is normalized discounted cumulative gain (ndcg)
+    Relevance is positive real values.  Can use binary
+    as the previous methods.
+    Example from
+    http://www.stanford.edu/class/cs276/handouts/EvaluationNew-handout-6-per.pdf
+    >>> r = [3, 2, 3, 0, 0, 1, 2, 2, 3, 0]
+    >>> ndcg_at_k(r, 1)
+    1.0
+    >>> r = [2, 1, 2, 0]
+    >>> ndcg_at_k(r, 4)
+    0.9203032077642922
+    >>> ndcg_at_k(r, 4, method=1)
+    0.96519546960144276
+    >>> ndcg_at_k([0], 1)
+    0.0
+    >>> ndcg_at_k([1], 2)
+    1.0
+    Args:
+        r: Relevance scores (list or numpy) in rank order
+            (first element is the first item)
+        k: Number of results to consider
+        method: If 0 then weights are [1.0, 1.0, 0.6309, 0.5, 0.4307, ...]
+                If 1 then weights are [1.0, 0.6309, 0.5, 0.4307, ...]
+    Returns:
+        Normalized discounted cumulative gain
+    """
+    dcg_max = dcg_at_k(sorted(r, reverse=True), k, method)
+    if not dcg_max:
+        return 0.
+    return dcg_at_k(r, k, method) / dcg_max
+
+
+"""
+Wealth inequality
+"""
+
+
+def gini(arr):
+    ## Gini = \frac{2\sum_i^n i\times y_i}{n\sum_i^n y_i} - \frac{n+1}{n}
+    sorted_arr = arr.copy()
+    sorted_arr.sort()
+    n = arr.size
+    coef_ = 2. / n
+    const_ = (n + 1.) / n
+    weighted_sum = sum([(i + 1) * yi for i, yi in enumerate(sorted_arr)])
+    return coef_ * weighted_sum / (sorted_arr.sum()) - const_
+
+
+"""
+Expected envy and inferiority under probabilistic recommendation as weighted sampling with replacement
+"""
+
+
+def expected_utility_u(Ru, ps, k):
+    return Ru @ ps * k
+
+
+def expected_utility(R, Pi, k):
+    U = (R * Pi * k).sum(axis=1)
+    # if not agg:
+    return U
+
+
+def expected_envy_u_v(Ru, pus, pvs, k):
+    return Ru @ (pvs - pus) * k
+
+
+def prob_in(ps, k):
+    return 1 - (1 - ps) ** k
+
+
+def prob_in_approx(ps, k):
+    return k * ps
+
+
+def expected_inferiority_u_v(Ru, Rv, pus, pvs, k, compensate=False, approx=False):
+    differ = Rv - Ru
+    if not compensate:
+        differ = np.clip(differ, a_min=0, a_max=None)
+    if not approx:
+        return differ @ (prob_in(pus, k) * prob_in(pvs, k))
+    else:
+        return differ @ (prob_in_approx(pus, k) * prob_in_approx(pvs, k))
+
+
+def expected_envy(R, Pi, k):
+    """
+    Measure expected envy for k-sized recommendation according to rec strategy Pi with respect to relevancy scores R
+    :param R: m x n real-valued matrix
+    :param Pi: m x n Markov matrix
+    :return: E: m x n envy matrix where Euv = envy from u to v if not agg, sum of E if agg
+    """
+    assert np.all(np.isclose(Pi.sum(axis=1), 1.)) or np.array_equal(Pi,
+                                                                    Pi.astype(bool))  # binary matrix for discrete rec
+    m, n = len(R), len(R[0])
+    E = np.zeros((m, m))
+    for u in range(m):
+        for v in range(m):
+            if v == u:
+                continue
+            E[u, v] = expected_envy_u_v(R[u], Pi[u], Pi[v], k=k)
+    E = np.clip(E, a_min=0., a_max=None)
+    # if not agg:
+    return E
+
+
+def expected_inferiority(R, Pi, k, compensate=True, approx=False):
+    """
+    Measure expected inferiority for k-sized recommendation according to rec strategy Pi with respect to relevancy scores R
+    :param R:
+    :param Pi:
+    :param k:
+    :param agg:
+    :return: I: m x n
+    """
+    assert np.all(np.isclose(Pi.sum(axis=1), 1.)) or np.array_equal(Pi,
+                                                                    Pi.astype(bool))  # binary matrix for discrete rec
+    m, n = len(R), len(R[0])
+    I = np.zeros((m, m))
+    for u in range(m):
+        for v in range(m):
+            if v == u:
+                continue
+            I[u, v] = expected_inferiority_u_v(R[u], R[v], Pi[u], Pi[v], k=k, approx=approx, compensate=compensate)
+
+    I = np.clip(I, a_min=0., a_max=None)
+    # if not agg:
+    return I
+
+
+def expected_envy_torch(R, Pi, k):
+    m, n = len(R), len(R[0])
+    E = torch.zeros(m, m)
+    for u in range(m):
+        for v in range(m):
+            if v == u:
+                continue
+            E[u, v] = expected_envy_u_v(R[u], Pi[u], Pi[v], k=k)
+    E = torch.clamp(E, min=0.)
+    return E
+
+
+def expected_envy_torch_vec(R, P, k):
+    res = R @ P.transpose(0, 1)
+    envy_mat = (res - torch.diagonal(res, 0).reshape(-1, 1))
+    return k * (torch.clamp(envy_mat, min=0.))
+
+
+def expected_inferiority_torch(R, Pi, k, compensate=False, approx=False):
+    m, n = R.shape
+    I = torch.zeros((m, m))
+    for u in range(m):
+        for v in range(m):
+            if v == u:
+                continue
+            if not approx:
+                joint_prob = prob_in(Pi[v], k) * prob_in(Pi[u], k)
+            else:
+                joint_prob = prob_in_approx(Pi[v], k) * prob_in_approx(Pi[u], k)
+
+            if not compensate:
+                I[u, v] = torch.clamp(R[v] - R[u], min=0., max=None) @ joint_prob
+            else:
+                I[u, v] = (R[v] - R[u]) @ joint_prob
+
+    return torch.clamp(I, min=0.)
+
+
+def expected_inferiority_torch_vec(R, P, k, compensate=False, approx=False):
+    m, n = R.shape
+    I = torch.zeros((m, m))
+    P_pow_k = 1 - (1 - P).pow(k) if not approx else P * k
+    for i in range(m):
+        first_term = torch.clamp(R - R[i], min=0.) if not compensate else R - R[i]
+        I[i] = (first_term * (P_pow_k[i] * P_pow_k)).sum(1)
+    return I
+
+
+def slow_onehot(idx, P):
+    m = P.shape[0]
+    res = torch.zeros_like(P)
+    for i in range(m):
+        res[i, idx[i]] = 1.
+    return res
+
+
+def eiu_cut_off(R, Pi, k, agg=True):
+    """
+    Evaluate envy, inferiority, utility based on top-k cut-off recommendation
+    :param R:
+    :param Pi:
+    :return: envy, inferiority, utility
+    """
+    # print('Start evaluation!')
+    m, n = R.shape
+    # _, rec = torch.topk(Pi, k, dim=1)
+    # rec_onehot = F.one_hot(rec, num_classes=n).sum(1).float()
+    rec_onehot = slow_onehot(torch.topk(Pi, k, dim=1)[1], Pi)
+    envy = expected_envy_torch_vec(R, rec_onehot, k=1)
+    inferiority = expected_inferiority_torch_vec(R, rec_onehot, k=1, compensate=False, approx=False)
+    utility = expected_utility(R, rec_onehot, k=1)
+    if agg:
+        envy = envy.sum(-1).mean()
+        inferiority = inferiority.sum(-1).mean()
+        utility = utility.mean()
+    return envy, inferiority, utility
+
+
+def eiu_cut_off2(R, Pi, k, agg=True):
+    """
+    Evaluate envy, inferiority, utility based on top-k cut-off recommendation
+    :param R:
+    :param Pi:
+    :return: envy, inferiority, utility
+    """
+    # print('Start evaluation!')
+    S, U = R
+    if not isinstance(S, torch.Tensor):
+        S = torch.tensor(S)
+    if not isinstance(U, torch.Tensor):
+        U = torch.tensor(U)
+    if not isinstance(Pi, torch.Tensor):
+        Pi = torch.tensor(Pi)
+    m, n = U.shape
+    # _, rec = torch.topk(Pi, k, dim=1)
+    # rec_onehot = F.one_hot(rec, num_classes=n).sum(1).float()
+    rec_onehot = slow_onehot(torch.topk(Pi, k, dim=1)[1], Pi)
+    envy = expected_envy_torch_vec(U, rec_onehot, k=1)
+    inferiority = expected_inferiority_torch_vec(S, rec_onehot, k=1, compensate=False, approx=False)
+    utility = expected_utility(U, rec_onehot, k=1)
+    if agg:
+        envy = envy.sum(-1).mean()
+        inferiority = inferiority.sum(-1).mean()
+        utility = utility.mean()
+    return envy, inferiority, utility
+
+
+"""
+Global congestion metrics
+"""
+
+
+def get_competitors(rec_per_job, rec):
+    m = rec.shape[0]
+    competitors = []
+    for i in range(m):
+        if len(rec[i]) == 1:
+            competitors.append([rec_per_job[rec[i]]])
+        else:
+            competitors.append(rec_per_job[rec[i]])
+    return np.array(competitors)
+
+
+def get_better_competitor_scores(rec, R):
+    m, n = R.shape
+    _, k = rec.shape
+    user_ids_per_job = defaultdict(list)
+    for i, r in enumerate(rec):
+        for j in r:
+            user_ids_per_job[j.item()].append(i)
+
+    mean_competitor_scores_per_job = np.zeros((m, k))
+    for i in range(m):
+        my_rec_jobs = rec[i].numpy()
+
+        my_mean_competitors = np.zeros(k)
+        for j_, j in enumerate(my_rec_jobs):
+            my_score = R[i, j]
+            all_ids = user_ids_per_job[j].copy()
+            all_ids.remove(i)
+            other_scores = R[all_ids, j]
+            if not all_ids:
+                other_scores = np.zeros(1)  # TODO if no competition, then it is the negative of my own score
+            my_mean_competitors[j_] = other_scores.mean() - my_score
+            # my_mean_competitors[my_mean_competitors < 0] = 0. # TODO only keep the better competitors
+        mean_competitor_scores_per_job[i] = my_mean_competitors
+    return mean_competitor_scores_per_job
+
+
+def get_num_better_competitors(rec, R):
+    m, n = R.shape
+    _, k = rec.shape
+    user_ids_per_job = defaultdict(list)
+    for i, r in enumerate(rec):
+        for j in r:
+            user_ids_per_job[j.item()].append(i)
+
+    num_better_competitors = np.zeros((m, k))
+    for i in range(m):
+        my_rec_jobs = rec[i].numpy()
+
+        better_competitors = np.zeros(k)
+        for j_, j in enumerate(my_rec_jobs):
+            my_score = R[i, j]
+            all_ids = user_ids_per_job[j].copy()
+            all_ids.remove(i)
+            other_scores = R[all_ids, j]
+            better_competitors[j_] = ((other_scores - my_score) > 0).sum()
+        num_better_competitors[i] = better_competitors
+    return num_better_competitors
+
+
+def get_scores_ids_per_job(rec, R):
+    scores_per_job = defaultdict(list)
+    ids_per_job = defaultdict(list)
+
+    for i in range(len(rec)):
+        u = rec[i]
+        for jb in u:
+            jb = jb.item()
+            ids_per_job[jb].append(i)
+            scores_per_job[jb].append(R[i, jb].item())
+    return scores_per_job, ids_per_job
+
+
+def get_rank(a, method='ordinal', axis=None, descending=False):
+    if descending:
+        a = np.array(a) * -1
+    return stats.rankdata(a, method=method, axis=axis)
+
+
+def get_ranks_per_job(scores_rec):
+    ranks_per_job = defaultdict(list)
+    for jb in scores_rec:
+        ranks_per_job[jb] = get_rank(scores_rec[jb], descending=True)
+    return ranks_per_job
+
+
+def get_ranks_per_user(ranks_per_job, ids_per_job):
+    for k, v in ranks_per_job.items():
+        ranks_per_job[k] = [i - 1 for i in v]
+    ranks_per_user = defaultdict(list)
+    for k, v in ids_per_job.items():
+        rks = ranks_per_job[k]
+        for i, u in enumerate(v):
+            ranks_per_user[u].append(rks[i])
+    return ranks_per_user
+
+
+def calculate_global_metrics(res, R, k=10):
+    # get rec
+    m, n = res.shape
+    if not torch.is_tensor(res):
+        res = torch.from_numpy(res)
+    _, rec = torch.topk(res, k, dim=1)
+    rec_onehot = slow_onehot(rec, res)
+    # rec_onehot = F.one_hot(rec, num_classes=n).sum(1).float()
+    try:
+        rec_per_job = rec_onehot.sum(axis=0).numpy()
+    except:
+        rec_per_job = rec_onehot.sum(axis=0).cpu().numpy()
+        rec = rec.cpu()
+        R = R.cpu()
+    opt_competitors = get_competitors(rec_per_job, rec)
+
+    # mean competitors per person
+    mean_competitors = opt_competitors.mean()
+
+    # mean better competitors per person
+    mean_better_competitors = get_num_better_competitors(rec, R).mean()
+
+    # mean competitor scores - my score
+    mean_diff_scores = get_better_competitor_scores(rec, R)
+    mean_diff_scores[mean_diff_scores < 0] = 0.
+    mean_diff_scores = mean_diff_scores.mean()
+
+    # mean rank
+    # scores_opt, ids_opt = get_scores_ids_per_job(rec, R)
+    # ranks_opt = get_ranks_per_job(scores_opt)
+    # ranks_per_user_opt = get_ranks_per_user(ranks_opt, ids_opt)
+    # mean_rank = np.array(list(ranks_per_user_opt.values())).mean()
+
+    # gini
+    gini_index = gini(rec_per_job)
+
+    return {'mean_competitors': mean_competitors, 'mean_better_competitors': mean_better_competitors, \
+            'mean_scores_diff': mean_diff_scores, 'mean_rank': mean_better_competitors, 'gini_index': gini_index}
+
+
+def calculate_global_metrics2(res, R, k=10):
+    # get rec
+    S, U = R
+    m, n = res.shape
+    if not torch.is_tensor(res):
+        res = torch.from_numpy(res)
+    _, rec = torch.topk(res, k, dim=1)
+    rec_onehot = slow_onehot(rec, res)
+    # rec_onehot = F.one_hot(rec, num_classes=n).sum(1).float()
+    try:
+        rec_per_job = rec_onehot.sum(axis=0).numpy()
+    except:
+        rec_per_job = rec_onehot.sum(axis=0).cpu().numpy()
+        rec = rec.cpu()
+        S = S.cpu()
+        U = U.cpu()
+    opt_competitors = get_competitors(rec_per_job, rec)
+
+    # mean competitors per person
+    mean_competitors = opt_competitors.mean()
+
+    # mean better competitors per person
+    mean_better_competitors = get_num_better_competitors(rec, S).mean()
+
+    # mean competitor scores - my score
+    mean_diff_scores = get_better_competitor_scores(rec, S)
+    mean_diff_scores[mean_diff_scores < 0] = 0.
+    mean_diff_scores = mean_diff_scores.mean()
+
+    # mean rank
+    scores_opt, ids_opt = get_scores_ids_per_job(rec, S)
+    ranks_opt = get_ranks_per_job(scores_opt)
+    ranks_per_user_opt = get_ranks_per_user(ranks_opt, ids_opt)
+    mean_rank = np.array(list(ranks_per_user_opt.values())).mean()
+
+    # gini
+    gini_index = gini(rec_per_job)
+
+    return {'mean_competitors': mean_competitors, 'mean_better_competitors': mean_better_competitors, \
+            'mean_scores_diff': mean_diff_scores, 'mean_rank': mean_rank, 'gini_index': gini_index}
+
+def get_scores_per_job(rec, S):
+    scores_per_job = defaultdict(list)
+    for i in range(len(rec)):
+        u = rec[i]
+        for jb in u:
+            jb = jb.item()
+            scores_per_job[jb].append(S[i, jb].item())
+    return scores_per_job
+

utils/monitor.yaml

@@ -0,0 +1,15 @@
+# Ufile: data/U.pkl
+# Sfile: data/S.pkl
+# Pfile: data/P.pkl
+# user_meta_file: data/user_meta_data.pkl
+# user_groups: ['degree']
+# job_meta_file: data/job_meta_data.pkl
+Ufile: fake_data/U.pkl
+Sfile: fake_data/S.pkl
+Pfile: fake_data/P.pkl
+user_meta_file: fake_data/user_metadata.pkl
+user_groups: ['Sex', 'Edu']
+job_meta_file: ''
+server_name: '10.10.133.46'
+server_port: 1122
+role: user