add results

results/AutoGluon (best)/all_results.csv

@@ -0,0 +1,6 @@
+dataset,model,eval_metrics/normalized-error,eval_metrics/fit-time-per-1K-rows,eval_metrics/inference-time-per-1K-rows,problem_type,num_features
+airline,AutoGluon (best),0.4579066991764239,64,47,Regression,12
+electricity,AutoGluon (best),0.5994102307296079,18,46,Classification,2
+solar-energy,AutoGluon (best),0.04653723902390405,98,12,Multi-classification,3
+traffic,AutoGluon (best),0.8329854602344595,56,57,Multi-classification,12
+volcano,AutoGluon (best),0.1941381662313174,15,11,Regression,12

results/AutoGluon (best)/config.json

@@ -0,0 +1 @@
+{"model": "AutoGluon (best)", "method_type": "AutoML"}

results/CatBoost/all_results.csv

@@ -0,0 +1,6 @@
+dataset,model,eval_metrics/normalized-error,eval_metrics/fit-time-per-1K-rows,eval_metrics/inference-time-per-1K-rows,problem_type,num_features
+airline,CatBoost,0.9390154997236811,9,60,Regression,12
+electricity,CatBoost,0.5553220864715752,83,99,Classification,2
+solar-energy,CatBoost,0.29752892257633534,75,75,Multi-classification,3
+traffic,CatBoost,0.40131222920888976,42,25,Multi-classification,12
+volcano,CatBoost,0.07149164567350186,93,30,Regression,12

results/CatBoost/config.json

@@ -0,0 +1 @@
+{"model": "CatBoost", "method_type": "Boosted-tree"}

results/KNN/all_results.csv

@@ -0,0 +1,6 @@
+dataset,model,eval_metrics/normalized-error,eval_metrics/fit-time-per-1K-rows,eval_metrics/inference-time-per-1K-rows,problem_type,num_features
+airline,KNN,0.16340155736699258,82,41,Regression,12
+electricity,KNN,0.9561332837281634,88,60,Classification,2
+solar-energy,KNN,0.8283974331685697,55,53,Multi-classification,3
+traffic,KNN,0.8769826951645302,11,65,Multi-classification,12
+volcano,KNN,0.5149493758906379,82,49,Regression,12

results/KNN/config.json

@@ -0,0 +1 @@
+{"model": "KNN", "method_type": "Other"}

results/TabPFN-v2/all_results.csv

@@ -0,0 +1,6 @@
+dataset,model,eval_metrics/normalized-error,eval_metrics/fit-time-per-1K-rows,eval_metrics/inference-time-per-1K-rows,problem_type,num_features
+airline,TabPFN-v2,0.442424494400528,35,25,Regression,12
+electricity,TabPFN-v2,0.6619106100291952,40,67,Classification,2
+solar-energy,TabPFN-v2,0.011669465671394597,72,70,Multi-classification,3
+traffic,TabPFN-v2,0.6169987849711932,30,57,Multi-classification,12
+volcano,TabPFN-v2,0.2498786665823265,2,57,Regression,12

results/TabPFN-v2/config.json

@@ -0,0 +1 @@
+{"model": "TabPFN-v2", "method_type": "Foundational"}

results/TabPFN/all_results.csv

@@ -0,0 +1,6 @@
+dataset,model,eval_metrics/normalized-error,eval_metrics/fit-time-per-1K-rows,eval_metrics/inference-time-per-1K-rows,problem_type,num_features
+airline,TabPFN,0.7101141981744494,76,94,Regression,12
+electricity,TabPFN,0.11984123950907233,70,7,Classification,2
+solar-energy,TabPFN,0.01033281899202676,69,69,Multi-classification,3
+traffic,TabPFN,0.8682598272397801,47,22,Multi-classification,12
+volcano,TabPFN,0.19462280337949245,3,71,Regression,12

results/TabPFN/config.json

@@ -0,0 +1 @@
+{"model": "TabPFN", "method_type": "Foundational"}

results/dataset_properties.csv

@@ -1,4 +1,4 @@
-dataset,problem_type,num_variates
+dataset,problem_type,num_features
 airline,Regression,12
 electricity,Classification,2
 solar-energy,Multi-classification,3

src/constants.py

@@ -22,4 +22,4 @@ class MethodTypes:
 class DatasetInfo:
     col_name: str = "dataset"
     num_rows: str = "num_rows"
-    num_features: str = "num_variates"
+    num_features: str = "num_features"

src/utils.py

@@ -122,9 +122,6 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie
             if file == 'all_results.csv':
                 file_path = os.path.join(subdir, file)
                 df = pd.read_csv(file_path)
-                # Rename the column if it exists
-                if 'eval_metrics/MASE[0.5]' in df.columns:
-                    df = df.rename(columns={'eval_metrics/MASE[0.5]': 'eval_metrics/normalized-error'})
                 df_list.append(df)
     # Concatenate all dataframes into one
     all_results_df = pd.concat(df_list, ignore_index=True)
@@ -167,27 +164,34 @@ def get_grouped_dfs(root_dir='results', ds_properties='results/dataset_propertie
     # df = unify_freq(df)
     # standardize by seasonal naive
     # df = standardize_df(df)
-    metric_columns = ['eval_metrics/MSE[mean]', 'eval_metrics/MSE[0.5]', 'eval_metrics/MAE[0.5]',
-                      'eval_metrics/normalized-error', 'eval_metrics/MAPE[0.5]', 'eval_metrics/sMAPE[0.5]',
-                      'eval_metrics/MSIS', 'eval_metrics/RMSE[mean]', 'eval_metrics/NRMSE[mean]',
-                      'eval_metrics/ND[0.5]', 'eval_metrics/mean_weighted_sum_quantile_loss']
+
+    # TODO change to ELO
     RANKING_METRIC = "eval_metrics/normalized-error"
-    df['rank'] = df.groupby(['dataset', ProblemTypes.col_name])[f'{RANKING_METRIC}'].rank(method='first',
-                                                                                               ascending=True)
-    # create a new column called rank
-    metric_columns.append('rank')
-    # create a new column called univariate. Set it to true if column num_variates is 1, otherwise set it to false
-    df['univariate'] = df['num_variates'] == 1
+
+    # compute metrics that requires all methods results such as Rank and Elo.
+    df['rank'] = df.groupby(['dataset', ProblemTypes.col_name])[f'{RANKING_METRIC}'].rank(method='first', ascending=True)
+
+    # TODO compute ELO from available data
+    df['ELO'] = df.groupby(['dataset', ProblemTypes.col_name])[f'{RANKING_METRIC}'].rank(method='first',
+                                                                                          ascending=True) * 100
+
+    # # create a new column called univariate. Set it to true if column num_variates is 1, otherwise set it to false
+    # df['univariate'] = df['num_variates'] == 1
 
     # group by domain
     grouped_results_overall = df.groupby(['model'])[METRIC_CHOICES].agg(stats.gmean)
     grouped_results_overall_rank = df.groupby(['model'])[['rank']].mean()
-    grouped_results_overall = pd.concat([grouped_results_overall, grouped_results_overall_rank], axis=1)
+    grouped_results_overall_elo = df.groupby(['model'])[['ELO']].mean()
+    grouped_results_overall = pd.concat([
+        grouped_results_overall,
+        grouped_results_overall_rank,
+        grouped_results_overall_elo],
+        axis=1
+    )
 
     # grouped_results_overall = grouped_results_overall.rename(columns={'model':'Model'})
     # grouped_results.to_csv(f'artefacts/grouped_results_by_model.csv')
     grouped_dfs = {}
-    # for col_name in ["domain", 'term_length', 'frequency', 'univariate']:
     for col_name in [ProblemTypes.col_name]:
         grouped_dfs[col_name] = group_by(df, col_name)
         # print(f"Grouping by {col_name}:\n {grouped_dfs.head(20)}")