{"vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.82, "ndcg_at_3": 0.86355, "ndcg_at_5": 0.87267, "ndcg_at_10": 0.88369, "ndcg_at_20": 0.88834, "ndcg_at_50": 0.89325, "ndcg_at_100": 0.8939, "map_at_1": 0.82, "map_at_3": 0.853, "map_at_5": 0.8581, "map_at_10": 0.86266, "map_at_20": 0.86398, "map_at_50": 0.86483, "map_at_100": 0.86489, "recall_at_1": 0.82, "recall_at_3": 0.894, "recall_at_5": 0.916, "recall_at_10": 0.95, "recall_at_20": 0.968, "recall_at_50": 0.992, "recall_at_100": 0.996, "precision_at_1": 0.82, "precision_at_3": 0.298, "precision_at_5": 0.1832, "precision_at_10": 0.095, "precision_at_20": 0.0484, "precision_at_50": 0.01984, "precision_at_100": 0.00996, "mrr_at_1": 0.82, "mrr_at_3": 0.8516666666666663, "mrr_at_5": 0.8576666666666662, "mrr_at_10": 0.8620460317460314, "mrr_at_20": 0.863583752358752, "mrr_at_50": 0.8644210675649312, "mrr_at_100": 0.8644797173539607, "naucs_at_1_max": 0.812446304659103, "naucs_at_1_std": -0.12099350148694808, "naucs_at_1_diff1": 0.9116973234937766, "naucs_at_3_max": 0.8052286885682325, "naucs_at_3_std": -0.10011891450579295, "naucs_at_3_diff1": 0.8805530411941155, "naucs_at_5_max": 0.8331665999733249, "naucs_at_5_std": -0.10378595882797696, "naucs_at_5_diff1": 0.8882330710061808, "naucs_at_10_max": 0.870214752567696, "naucs_at_10_std": -0.012511671335205815, "naucs_at_10_diff1": 0.8939309056956121, "naucs_at_20_max": 0.8751167133520102, "naucs_at_20_std": -0.03801937441644435, "naucs_at_20_diff1": 0.8853291316526594, "naucs_at_50_max": 0.8978758169934562, "naucs_at_50_std": -0.07457983193277996, "naucs_at_50_diff1": 0.9673202614378978, "naucs_at_100_max": 0.9346405228758466, "naucs_at_100_std": 0.41433239962652135, "naucs_at_100_diff1": 0.9346405228758466 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.49667, "ndcg_at_3": 0.58045, "ndcg_at_5": 0.5961, "ndcg_at_10": 0.61715, "ndcg_at_20": 0.63148, "ndcg_at_50": 0.64476, "ndcg_at_100": 0.65117, "map_at_1": 0.49667, "map_at_3": 0.56098, "map_at_5": 0.56973, "map_at_10": 0.57815, "map_at_20": 0.58195, "map_at_50": 0.58412, "map_at_100": 0.58466, "recall_at_1": 0.49667, "recall_at_3": 0.63636, "recall_at_5": 0.67406, "recall_at_10": 0.74058, "recall_at_20": 0.79823, "recall_at_50": 0.86475, "recall_at_100": 0.90466, "precision_at_1": 0.49667, "precision_at_3": 0.21212, "precision_at_5": 0.13481, "precision_at_10": 0.07406, "precision_at_20": 0.03991, "precision_at_50": 0.01729, "precision_at_100": 0.00905, "mrr_at_1": 0.49667405764966743, "mrr_at_3": 0.5598669623059869, "mrr_at_5": 0.5686252771618625, "mrr_at_10": 0.5774082286277407, "mrr_at_20": 0.5810969093386733, "mrr_at_50": 0.5833476204155297, "mrr_at_100": 0.5838896413403367, "naucs_at_1_max": 0.6447171350460632, "naucs_at_1_std": -0.09484358295898986, "naucs_at_1_diff1": 0.8458551941133076, "naucs_at_3_max": 0.7454063347349457, "naucs_at_3_std": 0.07229007919713754, "naucs_at_3_diff1": 0.7797700229519104, "naucs_at_5_max": 0.7393820262077632, "naucs_at_5_std": 0.07367377967418601, "naucs_at_5_diff1": 0.7716289000253728, "naucs_at_10_max": 0.8009576009267406, "naucs_at_10_std": 0.17268036524877706, "naucs_at_10_diff1": 0.771081787065486, "naucs_at_20_max": 0.8203029945264155, "naucs_at_20_std": 0.19845745554768213, "naucs_at_20_diff1": 0.7679325913145596, "naucs_at_50_max": 0.8514788472516402, "naucs_at_50_std": 0.4169328726103599, "naucs_at_50_diff1": 0.7559123754864738, "naucs_at_100_max": 0.9163273855022911, "naucs_at_100_std": 0.6626096316780477, "naucs_at_100_diff1": 0.7467504013388263 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.84413, "ndcg_at_3": 0.88746, "ndcg_at_5": 0.89835, "ndcg_at_10": 0.90248, "ndcg_at_20": 0.90764, "ndcg_at_50": 0.90851, "ndcg_at_100": 0.90882, "map_at_1": 0.84413, "map_at_3": 0.87719, "map_at_5": 0.88327, "map_at_10": 0.88509, "map_at_20": 0.88653, "map_at_50": 0.8867, "map_at_100": 0.88672, "recall_at_1": 0.84413, "recall_at_3": 0.917, "recall_at_5": 0.94332, "recall_at_10": 0.95547, "recall_at_20": 0.97571, "recall_at_50": 0.97976, "recall_at_100": 0.98178, "precision_at_1": 0.84413, "precision_at_3": 0.30567, "precision_at_5": 0.18866, "precision_at_10": 0.09555, "precision_at_20": 0.04879, "precision_at_50": 0.0196, "precision_at_100": 0.00982, "mrr_at_1": 0.8421052631578947, "mrr_at_3": 0.8761808367071524, "mrr_at_5": 0.8821524966261807, "mrr_at_10": 0.8839598997493734, "mrr_at_20": 0.8854126323282514, "mrr_at_50": 0.8855840009206725, "mrr_at_100": 0.8856042438356523, "naucs_at_1_max": 0.5820526964694293, "naucs_at_1_std": -0.12258720751834291, "naucs_at_1_diff1": 0.9179232290494516, "naucs_at_3_max": 0.5688472589103235, "naucs_at_3_std": -0.03525896756328502, "naucs_at_3_diff1": 0.8589169639952821, "naucs_at_5_max": 0.658104574327871, "naucs_at_5_std": 0.23877363760140033, "naucs_at_5_diff1": 0.870167572146584, "naucs_at_10_max": 0.7995309447871867, "naucs_at_10_std": 0.4714428670492628, "naucs_at_10_diff1": 0.8785680767019614, "naucs_at_20_max": 0.9891165454946104, "naucs_at_20_std": 0.8992936045133497, "naucs_at_20_diff1": 0.8757902959599302, "naucs_at_50_max": 0.9869398545935183, "naucs_at_50_std": 0.9199859445982981, "naucs_at_50_diff1": 0.8640085005583854, "naucs_at_100_max": 0.9854887273261316, "naucs_at_100_std": 0.9256067666719673, "naucs_at_100_diff1": 0.8488983339537634 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.875, "ndcg_at_3": 0.92542, "ndcg_at_5": 0.92972, "ndcg_at_10": 0.93672, "ndcg_at_20": 0.93858, "ndcg_at_50": 0.93921, "ndcg_at_100": 0.93921, "map_at_1": 0.875, "map_at_3": 0.91429, "map_at_5": 0.91661, "map_at_10": 0.91954, "map_at_20": 0.92007, "map_at_50": 0.92015, "map_at_100": 0.92015, "recall_at_1": 0.875, "recall_at_3": 0.95714, "recall_at_5": 0.96786, "recall_at_10": 0.98929, "recall_at_20": 0.99643, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.875, "precision_at_3": 0.31905, "precision_at_5": 0.19357, "precision_at_10": 0.09893, "precision_at_20": 0.04982, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.875, "mrr_at_3": 0.9142857142857143, "mrr_at_5": 0.9166071428571427, "mrr_at_10": 0.9195436507936509, "mrr_at_20": 0.9200931013431013, "mrr_at_50": 0.9201659876404774, "mrr_at_100": 0.9201659876404774, "naucs_at_1_max": 0.8578413915901933, "naucs_at_1_std": 0.3373099575400639, "naucs_at_1_diff1": 0.9291192987262019, "naucs_at_3_max": 0.94281045751634, "naucs_at_3_std": 0.7399626517273589, "naucs_at_3_diff1": 0.9192732648615032, "naucs_at_5_max": 0.9546114742193174, "naucs_at_5_std": 0.8474945533769075, "naucs_at_5_diff1": 0.9214130096483005, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.8638344226579515, "naucs_at_10_diff1": 0.807812013694365, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 0.8692810457516478, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.54678, "ndcg_at_3": 0.66776, "ndcg_at_5": 0.69393, "ndcg_at_10": 0.71886, "ndcg_at_20": 0.72921, "ndcg_at_50": 0.73649, "ndcg_at_100": 0.73867, "map_at_1": 0.54678, "map_at_3": 0.63761, "map_at_5": 0.65222, "map_at_10": 0.66285, "map_at_20": 0.66571, "map_at_50": 0.66691, "map_at_100": 0.6671, "recall_at_1": 0.54678, "recall_at_3": 0.75516, "recall_at_5": 0.81835, "recall_at_10": 0.89368, "recall_at_20": 0.93439, "recall_at_50": 0.97084, "recall_at_100": 0.9842, "precision_at_1": 0.54678, "precision_at_3": 0.25172, "precision_at_5": 0.16367, "precision_at_10": 0.08937, "precision_at_20": 0.04672, "precision_at_50": 0.01942, "precision_at_100": 0.00984, "mrr_at_1": 0.5449574726609964, "mrr_at_3": 0.6363912515188347, "mrr_at_5": 0.6513061968408275, "mrr_at_10": 0.6619414067773731, "mrr_at_20": 0.6648084703051904, "mrr_at_50": 0.666014531722658, "mrr_at_100": 0.6662005960985684, "naucs_at_1_max": 0.2731052125785171, "naucs_at_1_std": -0.14426853499658712, "naucs_at_1_diff1": 0.7112304282234161, "naucs_at_3_max": 0.32666962544180816, "naucs_at_3_std": -0.07666756779655057, "naucs_at_3_diff1": 0.5831996505006363, "naucs_at_5_max": 0.35015803161999937, "naucs_at_5_std": -0.056362663125386854, "naucs_at_5_diff1": 0.5605231352339403, "naucs_at_10_max": 0.45904296783912285, "naucs_at_10_std": 0.05818769804418837, "naucs_at_10_diff1": 0.5464314336687579, "naucs_at_20_max": 0.5489800312526507, "naucs_at_20_std": 0.19000550595425766, "naucs_at_20_diff1": 0.5437396218670546, "naucs_at_50_max": 0.6781089810313774, "naucs_at_50_std": 0.5656683520376952, "naucs_at_50_diff1": 0.5981871760182872, "naucs_at_100_max": 0.7245933213322641, "naucs_at_100_std": 0.6419966475290947, "naucs_at_100_diff1": 0.6649811059473633 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.77, "ndcg_at_3": 0.86309, "ndcg_at_5": 0.87171, "ndcg_at_10": 0.87819, "ndcg_at_20": 0.88532, "ndcg_at_50": 0.88532, "ndcg_at_100": 0.88532, "map_at_1": 0.77, "map_at_3": 0.84, "map_at_5": 0.845, "map_at_10": 0.84768, "map_at_20": 0.84939, "map_at_50": 0.84939, "map_at_100": 0.84939, "recall_at_1": 0.77, "recall_at_3": 0.93, "recall_at_5": 0.95, "recall_at_10": 0.97, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.77, "precision_at_3": 0.31, "precision_at_5": 0.19, "precision_at_10": 0.097, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.78, "mrr_at_3": 0.845, "mrr_at_5": 0.85, "mrr_at_10": 0.8528571428571429, "mrr_at_20": 0.8545376984126984, "mrr_at_50": 0.8545376984126984, "mrr_at_100": 0.8545376984126984, "naucs_at_1_max": 0.14487142185716537, "naucs_at_1_std": -0.49399073431660045, "naucs_at_1_diff1": 0.7040576531411565, "naucs_at_3_max": 0.36494597839135445, "naucs_at_3_std": -0.6566626650660264, "naucs_at_3_diff1": 0.7675736961451218, "naucs_at_5_max": 0.716713352007473, "naucs_at_5_std": -0.34098972922501514, "naucs_at_5_diff1": 0.7301587301587323, "naucs_at_10_max": 0.5714285714285722, "naucs_at_10_std": -0.8445378151260495, "naucs_at_10_diff1": 0.5938375350140046, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.94, "ndcg_at_3": 0.97524, "ndcg_at_5": 0.97524, "ndcg_at_10": 0.97524, "ndcg_at_20": 0.97524, "ndcg_at_50": 0.97524, "ndcg_at_100": 0.97524, "map_at_1": 0.94, "map_at_3": 0.96667, "map_at_5": 0.96667, "map_at_10": 0.96667, "map_at_20": 0.96667, "map_at_50": 0.96667, "map_at_100": 0.96667, "recall_at_1": 0.94, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.9716666666666667, "mrr_at_5": 0.9716666666666667, "mrr_at_10": 0.9716666666666667, "mrr_at_20": 0.9716666666666667, "mrr_at_50": 0.9716666666666667, "mrr_at_100": 0.9716666666666667, "naucs_at_1_max": 0.46444133208839294, "naucs_at_1_std": 0.1777155306567078, "naucs_at_1_diff1": 0.9319172113289763, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.89, "ndcg_at_3": 0.93286, "ndcg_at_5": 0.93286, "ndcg_at_10": 0.93957, "ndcg_at_20": 0.93957, "ndcg_at_50": 0.94334, "ndcg_at_100": 0.94334, "map_at_1": 0.89, "map_at_3": 0.92333, "map_at_5": 0.92333, "map_at_10": 0.92625, "map_at_20": 0.92625, "map_at_50": 0.92677, "map_at_100": 0.92677, "recall_at_1": 0.89, "recall_at_3": 0.96, "recall_at_5": 0.96, "recall_at_10": 0.98, "recall_at_20": 0.98, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.32, "precision_at_5": 0.192, "precision_at_10": 0.098, "precision_at_20": 0.049, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.89, "mrr_at_3": 0.9233333333333333, "mrr_at_5": 0.9233333333333333, "mrr_at_10": 0.9262499999999999, "mrr_at_20": 0.9262499999999999, "mrr_at_50": 0.9267777777777776, "mrr_at_100": 0.9267777777777776, "naucs_at_1_max": 0.26032986856799367, "naucs_at_1_std": -0.8372562494631047, "naucs_at_1_diff1": 0.9639206253758271, "naucs_at_3_max": 0.4293884220354785, "naucs_at_3_std": -0.9801587301587241, "naucs_at_3_diff1": 0.9673202614379101, "naucs_at_5_max": 0.42938842203548094, "naucs_at_5_std": -0.9801587301587192, "naucs_at_5_diff1": 0.967320261437907, "naucs_at_10_max": -0.07586367880484816, "naucs_at_10_std": -1.445845004668519, "naucs_at_10_diff1": 0.9346405228758136, "naucs_at_20_max": -0.07586367880484816, "naucs_at_20_std": -1.445845004668519, "naucs_at_20_diff1": 0.9346405228758136, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.94917, "ndcg_at_5": 0.9569, "ndcg_at_10": 0.9569, "ndcg_at_20": 0.9569, "ndcg_at_50": 0.9569, "ndcg_at_100": 0.9569, "map_at_1": 0.9, "map_at_3": 0.93833, "map_at_5": 0.94233, "map_at_10": 0.94233, "map_at_20": 0.94233, "map_at_50": 0.94233, "map_at_100": 0.94233, "recall_at_1": 0.9, "recall_at_3": 0.98, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.32667, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.9383333333333332, "mrr_at_5": 0.9423333333333334, "mrr_at_10": 0.9423333333333334, "mrr_at_20": 0.9423333333333334, "mrr_at_50": 0.9423333333333334, "mrr_at_100": 0.9423333333333334, "naucs_at_1_max": 0.5017740429505145, "naucs_at_1_std": -0.4561624649859955, "naucs_at_1_diff1": 0.8643790849673212, "naucs_at_3_max": 0.7770774976657261, "naucs_at_3_std": -1.151727357609717, "naucs_at_3_diff1": 0.7957516339869218, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.95, "ndcg_at_3": 0.97893, "ndcg_at_5": 0.97893, "ndcg_at_10": 0.97893, "ndcg_at_20": 0.97893, "ndcg_at_50": 0.97893, "ndcg_at_100": 0.97893, "map_at_1": 0.95, "map_at_3": 0.97167, "map_at_5": 0.97167, "map_at_10": 0.97167, "map_at_20": 0.97167, "map_at_50": 0.97167, "map_at_100": 0.97167, "recall_at_1": 0.95, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.95, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.95, "mrr_at_3": 0.9716666666666666, "mrr_at_5": 0.9716666666666666, "mrr_at_10": 0.9716666666666666, "mrr_at_20": 0.9716666666666666, "mrr_at_50": 0.9716666666666666, "mrr_at_100": 0.9716666666666666, "naucs_at_1_max": 0.33015873015873015, "naucs_at_1_std": -0.5752567693744151, "naucs_at_1_diff1": 0.8954248366013068, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.61875, "ndcg_at_3": 0.60533, "ndcg_at_5": 0.6249, "ndcg_at_10": 0.6533, "ndcg_at_20": 0.68742, "ndcg_at_50": 0.70992, "ndcg_at_100": 0.72024, "map_at_1": 0.38719, "map_at_3": 0.49654, "map_at_5": 0.53693, "map_at_10": 0.5662, "map_at_20": 0.58481, "map_at_50": 0.59335, "map_at_100": 0.59568, "recall_at_1": 0.38719, "recall_at_3": 0.55967, "recall_at_5": 0.64187, "recall_at_10": 0.73101, "recall_at_20": 0.82946, "recall_at_50": 0.8939, "recall_at_100": 0.93192, "precision_at_1": 0.61875, "precision_at_3": 0.36875, "precision_at_5": 0.27875, "precision_at_10": 0.17313, "precision_at_20": 0.105, "precision_at_50": 0.0505, "precision_at_100": 0.02744, "mrr_at_1": 0.60625, "mrr_at_3": 0.6947916666666666, "mrr_at_5": 0.7076041666666666, "mrr_at_10": 0.7141666666666665, "mrr_at_20": 0.7167027497302748, "mrr_at_50": 0.7175877436934345, "mrr_at_100": 0.7176993508362915, "naucs_at_1_max": 0.27072241887056714, "naucs_at_1_std": -0.30666589925849186, "naucs_at_1_diff1": 0.37145433441729697, "naucs_at_3_max": 0.1575493620566184, "naucs_at_3_std": -0.14675377187982588, "naucs_at_3_diff1": -0.012422723353258194, "naucs_at_5_max": 0.12243596821473231, "naucs_at_5_std": -0.05745497027699916, "naucs_at_5_diff1": -0.11057882383111498, "naucs_at_10_max": 0.09666741292736748, "naucs_at_10_std": 0.06431701154571953, "naucs_at_10_diff1": -0.16892677050351262, "naucs_at_20_max": -4.1139254210270476e-05, "naucs_at_20_std": 0.16775721777388994, "naucs_at_20_diff1": -0.22444494485174687, "naucs_at_50_max": -0.026208062431506616, "naucs_at_50_std": 0.27289212578690536, "naucs_at_50_diff1": -0.23337444980105218, "naucs_at_100_max": -0.07013692948482042, "naucs_at_100_std": 0.2614993895421859, "naucs_at_100_diff1": -0.2529383758901295 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.75862, "ndcg_at_3": 0.66558, "ndcg_at_5": 0.6364, "ndcg_at_10": 0.60412, "ndcg_at_20": 0.61161, "ndcg_at_50": 0.6674, "ndcg_at_100": 0.71286, "map_at_1": 0.13294, "map_at_3": 0.22432, "map_at_5": 0.28141, "map_at_10": 0.35042, "map_at_20": 0.39877, "map_at_50": 0.45406, "map_at_100": 0.48535, "recall_at_1": 0.13294, "recall_at_3": 0.24879, "recall_at_5": 0.33068, "recall_at_10": 0.45414, "recall_at_20": 0.57711, "recall_at_50": 0.7563, "recall_at_100": 0.89138, "precision_at_1": 0.75862, "precision_at_3": 0.59195, "precision_at_5": 0.53793, "precision_at_10": 0.42069, "precision_at_20": 0.30517, "precision_at_50": 0.19034, "precision_at_100": 0.1281, "mrr_at_1": 0.7586206896551724, "mrr_at_3": 0.8045977011494254, "mrr_at_5": 0.8166666666666668, "mrr_at_10": 0.8212643678160919, "mrr_at_20": 0.8248997410635341, "mrr_at_50": 0.8248997410635341, "mrr_at_100": 0.8248997410635341, "naucs_at_1_max": 0.4227698077082209, "naucs_at_1_std": 0.19946586940003241, "naucs_at_1_diff1": 0.16206457409265, "naucs_at_3_max": 0.5153814385463314, "naucs_at_3_std": 0.33515138197678757, "naucs_at_3_diff1": -0.20009189610763917, "naucs_at_5_max": 0.5900831255483465, "naucs_at_5_std": 0.42933205868067004, "naucs_at_5_diff1": -0.17086750774916223, "naucs_at_10_max": 0.5992696167986679, "naucs_at_10_std": 0.47588375698237223, "naucs_at_10_diff1": -0.13260066848766422, "naucs_at_20_max": 0.5582586435890009, "naucs_at_20_std": 0.5038374365864665, "naucs_at_20_diff1": -0.1176926445837237, "naucs_at_50_max": 0.5432600497682903, "naucs_at_50_std": 0.5479275799361487, "naucs_at_50_diff1": -0.10889833063192782, "naucs_at_100_max": 0.5229583863106058, "naucs_at_100_std": 0.5435637228955296, "naucs_at_100_diff1": -0.08468927917935376 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.40351, "ndcg_at_3": 0.45091, "ndcg_at_5": 0.49439, "ndcg_at_10": 0.55003, "ndcg_at_20": 0.58106, "ndcg_at_50": 0.5993, "ndcg_at_100": 0.61956, "map_at_1": 0.22383, "map_at_3": 0.33472, "map_at_5": 0.37842, "map_at_10": 0.41671, "map_at_20": 0.43823, "map_at_50": 0.44683, "map_at_100": 0.45288, "recall_at_1": 0.22383, "recall_at_3": 0.40691, "recall_at_5": 0.54033, "recall_at_10": 0.71479, "recall_at_20": 0.8034, "recall_at_50": 0.86038, "recall_at_100": 0.92373, "precision_at_1": 0.40351, "precision_at_3": 0.30994, "precision_at_5": 0.25614, "precision_at_10": 0.17544, "precision_at_20": 0.11316, "precision_at_50": 0.05474, "precision_at_100": 0.03333, "mrr_at_1": 0.42105263157894735, "mrr_at_3": 0.543859649122807, "mrr_at_5": 0.5763157894736841, "mrr_at_10": 0.5925160122528542, "mrr_at_20": 0.5925160122528542, "mrr_at_50": 0.59296585480796, "mrr_at_100": 0.59296585480796, "naucs_at_1_max": 0.3607011316641863, "naucs_at_1_std": 0.16281795519485523, "naucs_at_1_diff1": 0.4423264543117199, "naucs_at_3_max": 0.248943549054806, "naucs_at_3_std": 0.11598523586964242, "naucs_at_3_diff1": 0.3693732108201387, "naucs_at_5_max": 0.09070545701728222, "naucs_at_5_std": 0.035895685604651485, "naucs_at_5_diff1": 0.11310819654651481, "naucs_at_10_max": 0.025784503411264655, "naucs_at_10_std": 0.08680461640852888, "naucs_at_10_diff1": 0.17678674068585096, "naucs_at_20_max": -0.05564005116785854, "naucs_at_20_std": 0.07668382220435485, "naucs_at_20_diff1": 0.14421544846362475, "naucs_at_50_max": -0.07299334885090104, "naucs_at_50_std": 0.11838037986656147, "naucs_at_50_diff1": 0.08439872237038344, "naucs_at_100_max": -0.154028488184722, "naucs_at_100_std": 0.06646101830793238, "naucs_at_100_diff1": 0.07594418237521101 }, "vidore/synthetic_axa_filtered_v1.0": { "ndcg_at_1": 0.77778, "ndcg_at_3": 0.6939, "ndcg_at_5": 0.69627, "ndcg_at_10": 0.67735, "ndcg_at_20": 0.70844, "ndcg_at_50": 0.74866, "ndcg_at_100": 0.77447, "map_at_1": 0.37182, "map_at_3": 0.47004, "map_at_5": 0.51407, "map_at_10": 0.55255, "map_at_20": 0.58582, "map_at_50": 0.61023, "map_at_100": 0.61746, "recall_at_1": 0.37182, "recall_at_3": 0.54554, "recall_at_5": 0.61902, "recall_at_10": 0.68077, "recall_at_20": 0.78791, "recall_at_50": 0.88064, "recall_at_100": 0.98184, "precision_at_1": 0.77778, "precision_at_3": 0.44444, "precision_at_5": 0.38889, "precision_at_10": 0.25556, "precision_at_20": 0.16389, "precision_at_50": 0.08333, "precision_at_100": 0.04667, "mrr_at_1": 0.7777777777777778, "mrr_at_3": 0.8148148148148149, "mrr_at_5": 0.8287037037037037, "mrr_at_10": 0.8287037037037037, "mrr_at_20": 0.8287037037037037, "mrr_at_50": 0.8287037037037037, "mrr_at_100": 0.8296453232893912, "naucs_at_1_max": 0.6008361717023135, "naucs_at_1_std": 0.48745034493065986, "naucs_at_1_diff1": 0.6688901649531576, "naucs_at_3_max": 0.16331168559176326, "naucs_at_3_std": -0.10227627073049722, "naucs_at_3_diff1": 0.11304747268745052, "naucs_at_5_max": -0.16988833169699724, "naucs_at_5_std": -0.11738566845857755, "naucs_at_5_diff1": -0.1809970467861988, "naucs_at_10_max": -0.33338325513685335, "naucs_at_10_std": -0.025510902319384403, "naucs_at_10_diff1": -0.3463189316581034, "naucs_at_20_max": -0.4314950363511429, "naucs_at_20_std": -0.07814162322249632, "naucs_at_20_diff1": -0.3597465700560062, "naucs_at_50_max": -0.47238628583373166, "naucs_at_50_std": -0.11221281815013781, "naucs_at_50_diff1": -0.40715561083773455, "naucs_at_100_max": -0.5009545254403022, "naucs_at_100_std": -0.13543075269829394, "naucs_at_100_diff1": -0.42655217053366745 }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.40789, "ndcg_at_3": 0.45398, "ndcg_at_5": 0.49414, "ndcg_at_10": 0.55175, "ndcg_at_20": 0.58156, "ndcg_at_50": 0.60291, "ndcg_at_100": 0.61991, "map_at_1": 0.22474, "map_at_3": 0.33583, "map_at_5": 0.37953, "map_at_10": 0.42123, "map_at_20": 0.44047, "map_at_50": 0.45011, "map_at_100": 0.45529, "recall_at_1": 0.22474, "recall_at_3": 0.41185, "recall_at_5": 0.53813, "recall_at_10": 0.71024, "recall_at_20": 0.80291, "recall_at_50": 0.86707, "recall_at_100": 0.91936, "precision_at_1": 0.40789, "precision_at_3": 0.30994, "precision_at_5": 0.25175, "precision_at_10": 0.17895, "precision_at_20": 0.1125, "precision_at_50": 0.05605, "precision_at_100": 0.03316, "mrr_at_1": 0.41228070175438597, "mrr_at_3": 0.5460526315789472, "mrr_at_5": 0.5730263157894736, "mrr_at_10": 0.5878254664438874, "mrr_at_20": 0.5892122375675007, "mrr_at_50": 0.5898023340451821, "mrr_at_100": 0.5898023340451821, "naucs_at_1_max": 0.2357658534604348, "naucs_at_1_std": 0.20546148190736413, "naucs_at_1_diff1": 0.2942997782902726, "naucs_at_3_max": 0.19477834554983517, "naucs_at_3_std": 0.0901616305738152, "naucs_at_3_diff1": 0.31211579977785125, "naucs_at_5_max": 0.1270343582677278, "naucs_at_5_std": 0.10429681691824569, "naucs_at_5_diff1": 0.11650041665238493, "naucs_at_10_max": 0.14045225679722562, "naucs_at_10_std": 0.12904084502952987, "naucs_at_10_diff1": 0.07244553717017796, "naucs_at_20_max": 0.10806664996215463, "naucs_at_20_std": 0.1554193282914579, "naucs_at_20_diff1": 0.05276148208397267, "naucs_at_50_max": 0.10921282506726279, "naucs_at_50_std": 0.1603346667039166, "naucs_at_50_diff1": 0.03503742864425776, "naucs_at_100_max": 0.04641822985754839, "naucs_at_100_std": 0.10747576298030817, "naucs_at_100_diff1": 0.04796089754571697 }, "vidore/synthetic_axa_filtered_v1.0_multilingual": { "ndcg_at_1": 0.63889, "ndcg_at_3": 0.6012, "ndcg_at_5": 0.58827, "ndcg_at_10": 0.58895, "ndcg_at_20": 0.62443, "ndcg_at_50": 0.68019, "ndcg_at_100": 0.70219, "map_at_1": 0.30333, "map_at_3": 0.39672, "map_at_5": 0.43249, "map_at_10": 0.46899, "map_at_20": 0.4991, "map_at_50": 0.52563, "map_at_100": 0.5314, "recall_at_1": 0.30333, "recall_at_3": 0.46864, "recall_at_5": 0.53553, "recall_at_10": 0.6324, "recall_at_20": 0.73382, "recall_at_50": 0.88373, "recall_at_100": 0.96868, "precision_at_1": 0.63889, "precision_at_3": 0.39815, "precision_at_5": 0.31944, "precision_at_10": 0.21528, "precision_at_20": 0.14583, "precision_at_50": 0.08056, "precision_at_100": 0.045, "mrr_at_1": 0.6388888888888888, "mrr_at_3": 0.712962962962963, "mrr_at_5": 0.7226851851851852, "mrr_at_10": 0.7293375220458554, "mrr_at_20": 0.7300319664902999, "mrr_at_50": 0.7310105018438352, "mrr_at_100": 0.731609030458644, "naucs_at_1_max": 0.5446948235308462, "naucs_at_1_std": 0.5753689576495256, "naucs_at_1_diff1": 0.7063537335074642, "naucs_at_3_max": 0.19358302792161025, "naucs_at_3_std": 0.2382851719434907, "naucs_at_3_diff1": 0.0695515629856817, "naucs_at_5_max": 0.09842257096341021, "naucs_at_5_std": 0.19689282025634797, "naucs_at_5_diff1": -0.0671135182203377, "naucs_at_10_max": -0.029979142509033184, "naucs_at_10_std": 0.16524339827950454, "naucs_at_10_diff1": -0.17108930995116722, "naucs_at_20_max": -0.12064995451986389, "naucs_at_20_std": 0.11922834681158871, "naucs_at_20_diff1": -0.23678549597263934, "naucs_at_50_max": -0.24090917683281962, "naucs_at_50_std": 0.029789552466275192, "naucs_at_50_diff1": -0.3198962818502687, "naucs_at_100_max": -0.27973729295702227, "naucs_at_100_std": -0.02328206863515192, "naucs_at_100_diff1": -0.3372110499232704 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.54531, "ndcg_at_3": 0.56617, "ndcg_at_5": 0.58574, "ndcg_at_10": 0.61827, "ndcg_at_20": 0.65025, "ndcg_at_50": 0.67359, "ndcg_at_100": 0.68484, "map_at_1": 0.34753, "map_at_3": 0.46224, "map_at_5": 0.49892, "map_at_10": 0.53012, "map_at_20": 0.54677, "map_at_50": 0.55485, "map_at_100": 0.5575, "recall_at_1": 0.34753, "recall_at_3": 0.53074, "recall_at_5": 0.61208, "recall_at_10": 0.71018, "recall_at_20": 0.80497, "recall_at_50": 0.87594, "recall_at_100": 0.9172, "precision_at_1": 0.54531, "precision_at_3": 0.35104, "precision_at_5": 0.26437, "precision_at_10": 0.16797, "precision_at_20": 0.10125, "precision_at_50": 0.04875, "precision_at_100": 0.02677, "mrr_at_1": 0.55, "mrr_at_3": 0.646614583333333, "mrr_at_5": 0.6649739583333327, "mrr_at_10": 0.6720951140873008, "mrr_at_20": 0.6750117754582119, "mrr_at_50": 0.6760599947203341, "mrr_at_100": 0.6761451306608197, "naucs_at_1_max": 0.29928374487258724, "naucs_at_1_std": -0.12359275842271056, "naucs_at_1_diff1": 0.4201913652071598, "naucs_at_3_max": 0.18873193865163934, "naucs_at_3_std": -0.06245010840351605, "naucs_at_3_diff1": -0.00536703682885494, "naucs_at_5_max": 0.14100421323862303, "naucs_at_5_std": -0.027020439430341922, "naucs_at_5_diff1": -0.1077482172997515, "naucs_at_10_max": 0.09344561088695642, "naucs_at_10_std": 0.037090812384825686, "naucs_at_10_diff1": -0.1671981024117422, "naucs_at_20_max": 0.05221627904664171, "naucs_at_20_std": 0.1282632066851335, "naucs_at_20_diff1": -0.20221964901502987, "naucs_at_50_max": 0.013513375852412229, "naucs_at_50_std": 0.19309861534269265, "naucs_at_50_diff1": -0.2256755414561015, "naucs_at_100_max": -0.019349328678396466, "naucs_at_100_std": 0.1852766642822116, "naucs_at_100_diff1": -0.23520475571331936 }, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.6681, "ndcg_at_3": 0.60424, "ndcg_at_5": 0.5747, "ndcg_at_10": 0.54872, "ndcg_at_20": 0.56306, "ndcg_at_50": 0.63179, "ndcg_at_100": 0.67493, "map_at_1": 0.12387, "map_at_3": 0.19949, "map_at_5": 0.24693, "map_at_10": 0.30745, "map_at_20": 0.35555, "map_at_50": 0.41253, "map_at_100": 0.44204, "recall_at_1": 0.12387, "recall_at_3": 0.22225, "recall_at_5": 0.29097, "recall_at_10": 0.40934, "recall_at_20": 0.53643, "recall_at_50": 0.74545, "recall_at_100": 0.87651, "precision_at_1": 0.6681, "precision_at_3": 0.53736, "precision_at_5": 0.48448, "precision_at_10": 0.38405, "precision_at_20": 0.28578, "precision_at_50": 0.18707, "precision_at_100": 0.12509, "mrr_at_1": 0.6637931034482759, "mrr_at_3": 0.7399425287356322, "mrr_at_5": 0.7505028735632185, "mrr_at_10": 0.7577620415982483, "mrr_at_20": 0.7608588507644033, "mrr_at_50": 0.7619126745219256, "mrr_at_100": 0.7619126745219256, "naucs_at_1_max": 0.41911982711960527, "naucs_at_1_std": 0.2219319741427808, "naucs_at_1_diff1": 0.3323052156631965, "naucs_at_3_max": 0.395124099923616, "naucs_at_3_std": 0.2844189407584065, "naucs_at_3_diff1": 0.01968334915007435, "naucs_at_5_max": 0.4067543990404015, "naucs_at_5_std": 0.33349301021004324, "naucs_at_5_diff1": -0.014244035459054444, "naucs_at_10_max": 0.3990475683526399, "naucs_at_10_std": 0.38841554010300666, "naucs_at_10_diff1": -0.041117494510292424, "naucs_at_20_max": 0.3675017859948493, "naucs_at_20_std": 0.4157443195663025, "naucs_at_20_diff1": -0.056415429353860895, "naucs_at_50_max": 0.32391982590254215, "naucs_at_50_std": 0.42853708930677303, "naucs_at_50_diff1": -0.08088278760860802, "naucs_at_100_max": 0.31207698778205484, "naucs_at_100_std": 0.4190495880385969, "naucs_at_100_diff1": -0.07513720852823234 }, "vidore/restaurant_esg_reports_beir": { "ndcg_at_1": 0.58333, "ndcg_at_3": 0.59295, "ndcg_at_5": 0.59847, "ndcg_at_10": 0.62705, "ndcg_at_20": 0.65711, "ndcg_at_50": 0.68372, "ndcg_at_100": 0.70126, "map_at_1": 0.4223, "map_at_3": 0.51429, "map_at_5": 0.53744, "map_at_10": 0.55815, "map_at_20": 0.56952, "map_at_50": 0.57719, "map_at_100": 0.58103, "recall_at_1": 0.4223, "recall_at_3": 0.56429, "recall_at_5": 0.62743, "recall_at_10": 0.71155, "recall_at_20": 0.81252, "recall_at_50": 0.89204, "recall_at_100": 0.97231, "precision_at_1": 0.59615, "precision_at_3": 0.33333, "precision_at_5": 0.22308, "precision_at_10": 0.13462, "precision_at_20": 0.07981, "precision_at_50": 0.04, "precision_at_100": 0.02269, "mrr_at_1": 0.5576923076923077, "mrr_at_3": 0.6474358974358974, "mrr_at_5": 0.6560897435897435, "mrr_at_10": 0.6660485347985349, "mrr_at_20": 0.6703807410538178, "mrr_at_50": 0.6726175245406014, "mrr_at_100": 0.6729276982378719, "naucs_at_1_max": 0.2932533009402019, "naucs_at_1_std": 0.1517545343059939, "naucs_at_1_diff1": 0.5522786237911489, "naucs_at_3_max": 0.15711366105316388, "naucs_at_3_std": 0.2842435852496209, "naucs_at_3_diff1": 0.17146359446607332, "naucs_at_5_max": 0.23419658811407862, "naucs_at_5_std": 0.3633012317767692, "naucs_at_5_diff1": 0.12100489678963466, "naucs_at_10_max": 0.047216835392144454, "naucs_at_10_std": 0.2537248335660501, "naucs_at_10_diff1": -0.022798036346065944, "naucs_at_20_max": -0.009656649432423618, "naucs_at_20_std": 0.18171843093206275, "naucs_at_20_diff1": -0.15378487348591693, "naucs_at_50_max": -0.06160339362487552, "naucs_at_50_std": 0.18732123220299826, "naucs_at_50_diff1": -0.26637045721055785, "naucs_at_100_max": -0.11600370147170294, "naucs_at_100_std": 0.15935039822386396, "naucs_at_100_diff1": -0.292198998284397 } }