{"vidore/restaurant_esg_reports_beir": {"ndcg_at_1": 0.26923, "ndcg_at_3": 0.3045, "ndcg_at_5": 0.32114, "ndcg_at_10": 0.35676, "ndcg_at_20": 0.38618, "ndcg_at_50": 0.42052, "ndcg_at_100": 0.43636, "map_at_1": 0.18333, "map_at_3": 0.24391, "map_at_5": 0.26139, "map_at_10": 0.27761, "map_at_20": 0.28675, "map_at_50": 0.29345, "map_at_100": 0.29582, "recall_at_1": 0.18333, "recall_at_3": 0.31923, "recall_at_5": 0.37917, "recall_at_10": 0.49423, "recall_at_20": 0.5946, "recall_at_50": 0.73516, "recall_at_100": 0.80377, "precision_at_1": 0.26923, "precision_at_3": 0.16667, "precision_at_5": 0.12308, "precision_at_10": 0.08269, "precision_at_20": 0.05385, "precision_at_50": 0.02846, "precision_at_100": 0.01692, "mrr_at_1": 0.23076923076923078, "mrr_at_3": 0.3269230769230769, "mrr_at_5": 0.34038461538461545, "mrr_at_10": 0.3608669108669108, "mrr_at_20": 0.3674438809054194, "mrr_at_50": 0.37084971190489147, "mrr_at_100": 0.3713579847136337, "naucs_at_1_max": 0.021151055203054405, "naucs_at_1_std": -0.029813730522622425, "naucs_at_1_diff1": 0.25067341614916727, "naucs_at_3_max": -0.04143164067060292, "naucs_at_3_std": 0.00730042874767451, "naucs_at_3_diff1": 0.3380054031613406, "naucs_at_5_max": -0.1347500524905061, "naucs_at_5_std": -0.07424067226444503, "naucs_at_5_diff1": 0.2669507130764703, "naucs_at_10_max": -0.210749971018401, "naucs_at_10_std": -0.21961340947875993, "naucs_at_10_diff1": 0.16640286626959264, "naucs_at_20_max": -0.21891469085057838, "naucs_at_20_std": -0.23005504027592547, "naucs_at_20_diff1": 0.22055505233932227, "naucs_at_50_max": -0.21310892162468298, "naucs_at_50_std": -0.20734986109292497, "naucs_at_50_diff1": 0.09142935454845501, "naucs_at_100_max": -0.22587806170824862, "naucs_at_100_std": -0.21517085450107218, "naucs_at_100_diff1": 0.09673268004702926}, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.51293, "ndcg_at_3": 0.49649, "ndcg_at_5": 0.47893, "ndcg_at_10": 0.48004, "ndcg_at_20": 0.51068, "ndcg_at_50": 0.59426, "ndcg_at_100": 0.63376, "map_at_1": 0.10147, "map_at_3": 0.15877, "map_at_5": 0.19559, "map_at_10": 0.25349, "map_at_20": 0.30278, "map_at_50": 0.36487, "map_at_100": 0.39199, "recall_at_1": 0.10147, "recall_at_3": 0.17902, "recall_at_5": 0.23979, "recall_at_10": 0.373, "recall_at_20": 0.52076, "recall_at_50": 0.76023, "recall_at_100": 0.88108, "precision_at_1": 0.51293, "precision_at_3": 0.45259, "precision_at_5": 0.41293, "precision_at_10": 0.35259, "precision_at_20": 0.27909, "precision_at_50": 0.18828, "precision_at_100": 0.12366, "mrr_at_1": 0.4956896551724138, "mrr_at_3": 0.6149425287356322, "mrr_at_5": 0.6311063218390804, "mrr_at_10": 0.6426518883415435, "mrr_at_20": 0.6473663956998068, "mrr_at_50": 0.6479916726305025, "mrr_at_100": 0.6480600908023689, "naucs_at_1_max": 0.2909740112601449, "naucs_at_1_std": 0.34007231211982364, "naucs_at_1_diff1": 0.02010046418675761, "naucs_at_3_max": 0.1462981439133328, "naucs_at_3_std": 0.23727806279449618, "naucs_at_3_diff1": -0.045115704507227815, "naucs_at_5_max": 0.16402901891891258, "naucs_at_5_std": 0.2788211772941985, "naucs_at_5_diff1": -0.03422957119067939, "naucs_at_10_max": 0.1304595275607474, "naucs_at_10_std": 0.27024887845192075, "naucs_at_10_diff1": -0.06184582606627896, "naucs_at_20_max": 0.0827755061718375, "naucs_at_20_std": 0.32564544404546836, "naucs_at_20_diff1": -0.1398291345769381, "naucs_at_50_max": 0.024145276015777446, "naucs_at_50_std": 0.3093730698130927, "naucs_at_50_diff1": -0.16628805756811568, "naucs_at_100_max": 0.0009219308736878375, "naucs_at_100_std": 0.2960260201661105, "naucs_at_100_diff1": -0.16354176688825178 }, "vidore/synthetic_axa_filtered_v1.0_multilingual": { "ndcg_at_1": 0.44444, "ndcg_at_3": 0.48365, "ndcg_at_5": 0.4576, "ndcg_at_10": 0.47015, "ndcg_at_20": 0.48904, "ndcg_at_50": 0.54777, "ndcg_at_100": 0.58161, "map_at_1": 0.16613, "map_at_3": 0.28304, "map_at_5": 0.30204, "map_at_10": 0.33659, "map_at_20": 0.35316, "map_at_50": 0.37788, "map_at_100": 0.38576, "recall_at_1": 0.16613, "recall_at_3": 0.4128, "recall_at_5": 0.4461, "recall_at_10": 0.56445, "recall_at_20": 0.62203, "recall_at_50": 0.78078, "recall_at_100": 0.91945, "precision_at_1": 0.44444, "precision_at_3": 0.35648, "precision_at_5": 0.26944, "precision_at_10": 0.18333, "precision_at_20": 0.11875, "precision_at_50": 0.07028, "precision_at_100": 0.04181, "mrr_at_1": 0.4305555555555556, "mrr_at_3": 0.5486111111111112, "mrr_at_5": 0.5625, "mrr_at_10": 0.5719907407407407, "mrr_at_20": 0.5744107744107745, "mrr_at_50": 0.5755691882594056, "mrr_at_100": 0.5766185382360561, "naucs_at_1_max": -0.10844176576631265, "naucs_at_1_std": 0.11984795634852517, "naucs_at_1_diff1": 0.34926607045706065, "naucs_at_3_max": -0.16882440588261344, "naucs_at_3_std": 0.04935292481231464, "naucs_at_3_diff1": 0.07094770137246685, "naucs_at_5_max": -0.2677975915325592, "naucs_at_5_std": -0.013445545910716006, "naucs_at_5_diff1": 0.03560103469628687, "naucs_at_10_max": -0.2905586858742058, "naucs_at_10_std": 0.0024045562548918554, "naucs_at_10_diff1": -0.08824087823808019, "naucs_at_20_max": -0.40897273226767533, "naucs_at_20_std": -0.1138141721108663, "naucs_at_20_diff1": -0.10887906242990919, "naucs_at_50_max": -0.42683318222629896, "naucs_at_50_std": -0.13076918365032622, "naucs_at_50_diff1": -0.10724840220966861, "naucs_at_100_max": -0.44961995894718576, "naucs_at_100_std": -0.1844335335164004, "naucs_at_100_diff1": -0.08314001637229608 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.5125, "ndcg_at_3": 0.54949, "ndcg_at_5": 0.5847, "ndcg_at_10": 0.61161, "ndcg_at_20": 0.64415, "ndcg_at_50": 0.66856, "ndcg_at_100": 0.67862, "map_at_1": 0.31246, "map_at_3": 0.4406, "map_at_5": 0.48617, "map_at_10": 0.51574, "map_at_20": 0.53362, "map_at_50": 0.54211, "map_at_100": 0.54437, "recall_at_1": 0.31246, "recall_at_3": 0.52965, "recall_at_5": 0.6369, "recall_at_10": 0.72113, "recall_at_20": 0.80597, "recall_at_50": 0.88539, "recall_at_100": 0.92369, "precision_at_1": 0.5125, "precision_at_3": 0.34167, "precision_at_5": 0.27, "precision_at_10": 0.16937, "precision_at_20": 0.105, "precision_at_50": 0.04938, "precision_at_100": 0.02681, "mrr_at_1": 0.5375, "mrr_at_3": 0.6479166666666667, "mrr_at_5": 0.6663541666666666, "mrr_at_10": 0.674516369047619, "mrr_at_20": 0.6765751027975293, "mrr_at_50": 0.6772030349370826, "mrr_at_100": 0.677426534298513, "naucs_at_1_max": 0.34895314898857854, "naucs_at_1_std": -0.07257942064683737, "naucs_at_1_diff1": 0.5293978708686925, "naucs_at_3_max": 0.18142528545936487, "naucs_at_3_std": -0.16137866300838627, "naucs_at_3_diff1": -0.0014234288130872189, "naucs_at_5_max": 0.014672569636598775, "naucs_at_5_std": -0.21002767017155474, "naucs_at_5_diff1": -0.1473381294964024, "naucs_at_10_max": -0.06629123564607431, "naucs_at_10_std": -0.2046018497631402, "naucs_at_10_diff1": -0.252293405519212, "naucs_at_20_max": -0.16779142727147645, "naucs_at_20_std": -0.1998645631632962, "naucs_at_20_diff1": -0.3377181009216048, "naucs_at_50_max": -0.2319362541617097, "naucs_at_50_std": -0.20139508212155197, "naucs_at_50_diff1": -0.35750220388740067, "naucs_at_100_max": -0.26209531433075695, "naucs_at_100_std": -0.19593561386651334, "naucs_at_100_diff1": -0.36382106679824544 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.47368, "ndcg_at_3": 0.48077, "ndcg_at_5": 0.51913, "ndcg_at_10": 0.55339, "ndcg_at_20": 0.58423, "ndcg_at_50": 0.62437, "ndcg_at_100": 0.63473, "map_at_1": 0.22899, "map_at_3": 0.33833, "map_at_5": 0.38792, "map_at_10": 0.4251, "map_at_20": 0.44759, "map_at_50": 0.46783, "map_at_100": 0.47287, "recall_at_1": 0.22899, "recall_at_3": 0.42834, "recall_at_5": 0.54461, "recall_at_10": 0.66769, "recall_at_20": 0.77688, "recall_at_50": 0.90877, "recall_at_100": 0.93622, "precision_at_1": 0.47368, "precision_at_3": 0.32164, "precision_at_5": 0.27719, "precision_at_10": 0.18772, "precision_at_20": 0.12193, "precision_at_50": 0.06667, "precision_at_100": 0.03702, "mrr_at_1": 0.47368421052631576, "mrr_at_3": 0.5994152046783625, "mrr_at_5": 0.6152046783625732, "mrr_at_10": 0.622270955165692, "mrr_at_20": 0.6259990253411306, "mrr_at_50": 0.6267578992701391, "mrr_at_100": 0.6267578992701391, "naucs_at_1_max": -0.3156199695209114, "naucs_at_1_std": -0.26486119453698037, "naucs_at_1_diff1": 0.11795422382927509, "naucs_at_3_max": -0.2735188964882074, "naucs_at_3_std": -0.23310566021944892, "naucs_at_3_diff1": -0.19449853074061657, "naucs_at_5_max": -0.2421479017067623, "naucs_at_5_std": -0.13869753650963065, "naucs_at_5_diff1": -0.3054740364166881, "naucs_at_10_max": -0.25873997567219825, "naucs_at_10_std": -0.1458966410575204, "naucs_at_10_diff1": -0.3032000489911622, "naucs_at_20_max": -0.3256819936368209, "naucs_at_20_std": -0.21934883184399268, "naucs_at_20_diff1": -0.3948318811395443, "naucs_at_50_max": -0.3584500031494027, "naucs_at_50_std": -0.21644313778125723, "naucs_at_50_diff1": -0.4136042140657271, "naucs_at_100_max": -0.3530688199691307, "naucs_at_100_std": -0.19076877127775468, "naucs_at_100_diff1": -0.42236105076593466 }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.49561, "ndcg_at_3": 0.51051, "ndcg_at_5": 0.53966, "ndcg_at_10": 0.57033, "ndcg_at_20": 0.60598, "ndcg_at_50": 0.6415, "ndcg_at_100": 0.65306, "map_at_1": 0.24154, "map_at_3": 0.36508, "map_at_5": 0.40846, "map_at_10": 0.445, "map_at_20": 0.46996, "map_at_50": 0.48868, "map_at_100": 0.49392, "recall_at_1": 0.24154, "recall_at_3": 0.46057, "recall_at_5": 0.5649, "recall_at_10": 0.67685, "recall_at_20": 0.79962, "recall_at_50": 0.91497, "recall_at_100": 0.94749, "precision_at_1": 0.49561, "precision_at_3": 0.34649, "precision_at_5": 0.2807, "precision_at_10": 0.19079, "precision_at_20": 0.12544, "precision_at_50": 0.06693, "precision_at_100": 0.03737, "mrr_at_1": 0.4692982456140351, "mrr_at_3": 0.6052631578947368, "mrr_at_5": 0.618421052631579, "mrr_at_10": 0.6259433305485937, "mrr_at_20": 0.630267555040449, "mrr_at_50": 0.6310822394721419, "mrr_at_100": 0.6311605602741469, "naucs_at_1_max": -0.27139902693937923, "naucs_at_1_std": -0.28335975541834757, "naucs_at_1_diff1": 0.03844931932885215, "naucs_at_3_max": -0.2333237939707581, "naucs_at_3_std": -0.1653044765750373, "naucs_at_3_diff1": -0.10443046117117412, "naucs_at_5_max": -0.19561866039792095, "naucs_at_5_std": -0.09561859939322022, "naucs_at_5_diff1": -0.18423887457809085, "naucs_at_10_max": -0.2060244561628858, "naucs_at_10_std": -0.08726363088972981, "naucs_at_10_diff1": -0.20492069680185956, "naucs_at_20_max": -0.25096317818789565, "naucs_at_20_std": -0.12216017305154458, "naucs_at_20_diff1": -0.26997057451690837, "naucs_at_50_max": -0.30710698032302514, "naucs_at_50_std": -0.13529052277664438, "naucs_at_50_diff1": -0.29528418472148443, "naucs_at_100_max": -0.3124105461583974, "naucs_at_100_std": -0.11726920470788517, "naucs_at_100_diff1": -0.2735365969618431 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.50938, "ndcg_at_3": 0.53068, "ndcg_at_5": 0.55676, "ndcg_at_10": 0.58653, "ndcg_at_20": 0.61833, "ndcg_at_50": 0.64235, "ndcg_at_100": 0.6544, "map_at_1": 0.31519, "map_at_3": 0.42911, "map_at_5": 0.46586, "map_at_10": 0.49473, "map_at_20": 0.51197, "map_at_50": 0.52001, "map_at_100": 0.52243, "recall_at_1": 0.31519, "recall_at_3": 0.50707, "recall_at_5": 0.59431, "recall_at_10": 0.68853, "recall_at_20": 0.77673, "recall_at_50": 0.8526, "recall_at_100": 0.8999, "precision_at_1": 0.50938, "precision_at_3": 0.325, "precision_at_5": 0.25125, "precision_at_10": 0.16016, "precision_at_20": 0.09891, "precision_at_50": 0.04688, "precision_at_100": 0.02587, "mrr_at_1": 0.5234375, "mrr_at_3": 0.6184895833333333, "mrr_at_5": 0.6364583333333333, "mrr_at_10": 0.6463244047619048, "mrr_at_20": 0.6495780415378181, "mrr_at_50": 0.6503474764286958, "mrr_at_100": 0.6505948135497464, "naucs_at_1_max": 0.4131418193131225, "naucs_at_1_std": 0.06757995282919683, "naucs_at_1_diff1": 0.516900768942945, "naucs_at_3_max": 0.18254564757418557, "naucs_at_3_std": -0.04347712846310109, "naucs_at_3_diff1": 0.03059708341704999, "naucs_at_5_max": 0.06972269700281146, "naucs_at_5_std": -0.10787077250693489, "naucs_at_5_diff1": -0.08493373308302267, "naucs_at_10_max": -0.023642522063642934, "naucs_at_10_std": -0.11714079374203694, "naucs_at_10_diff1": -0.17673365636114774, "naucs_at_20_max": -0.09569239071775491, "naucs_at_20_std": -0.11183756071235801, "naucs_at_20_diff1": -0.241802482460874, "naucs_at_50_max": -0.1488266985733028, "naucs_at_50_std": -0.11009165758378252, "naucs_at_50_diff1": -0.2751290287492702, "naucs_at_100_max": -0.18389216502938294, "naucs_at_100_std": -0.1152902550355418, "naucs_at_100_diff1": -0.2899356806702993 }, "vidore/synthetic_axa_filtered_v1.0": { "ndcg_at_1": 0.5, "ndcg_at_3": 0.59177, "ndcg_at_5": 0.56037, "ndcg_at_10": 0.56184, "ndcg_at_20": 0.56464, "ndcg_at_50": 0.62655, "ndcg_at_100": 0.64607, "map_at_1": 0.18432, "map_at_3": 0.35086, "map_at_5": 0.37963, "map_at_10": 0.4161, "map_at_20": 0.42647, "map_at_50": 0.45181, "map_at_100": 0.45911, "recall_at_1": 0.18432, "recall_at_3": 0.54724, "recall_at_5": 0.58705, "recall_at_10": 0.68342, "recall_at_20": 0.7113, "recall_at_50": 0.88439, "recall_at_100": 0.9347, "precision_at_1": 0.5, "precision_at_3": 0.44444, "precision_at_5": 0.33333, "precision_at_10": 0.21667, "precision_at_20": 0.12222, "precision_at_50": 0.07333, "precision_at_100": 0.04278, "mrr_at_1": 0.5, "mrr_at_3": 0.6296296296296297, "mrr_at_5": 0.6574074074074074, "mrr_at_10": 0.6574074074074074, "mrr_at_20": 0.6574074074074074, "mrr_at_50": 0.658641975308642, "mrr_at_100": 0.658641975308642, "naucs_at_1_max": -0.20471943280726257, "naucs_at_1_std": 0.18546089088182968, "naucs_at_1_diff1": 0.7445083027491819, "naucs_at_3_max": -0.5294402069183541, "naucs_at_3_std": -0.04763265145855668, "naucs_at_3_diff1": 0.253185276095856, "naucs_at_5_max": -0.669223474626299, "naucs_at_5_std": -0.08636482482813103, "naucs_at_5_diff1": 0.2760137984114396, "naucs_at_10_max": -0.738950395976398, "naucs_at_10_std": -0.1685495451706665, "naucs_at_10_diff1": 0.07368539851746013, "naucs_at_20_max": -0.7021396616750007, "naucs_at_20_std": -0.11462769019487411, "naucs_at_20_diff1": 0.07121119110741668, "naucs_at_50_max": -0.7063938743299508, "naucs_at_50_std": -0.15679147163325738, "naucs_at_50_diff1": -0.013733388816798229, "naucs_at_100_max": -0.6781868824447755, "naucs_at_100_std": -0.16240281161384443, "naucs_at_100_diff1": 0.04104876407606115 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.56897, "ndcg_at_3": 0.5475, "ndcg_at_5": 0.531, "ndcg_at_10": 0.51659, "ndcg_at_20": 0.5526, "ndcg_at_50": 0.62369, "ndcg_at_100": 0.66764, "map_at_1": 0.10475, "map_at_3": 0.17297, "map_at_5": 0.22022, "map_at_10": 0.27707, "map_at_20": 0.33372, "map_at_50": 0.39288, "map_at_100": 0.4226, "recall_at_1": 0.10475, "recall_at_3": 0.20237, "recall_at_5": 0.26964, "recall_at_10": 0.39933, "recall_at_20": 0.56307, "recall_at_50": 0.77351, "recall_at_100": 0.90788, "precision_at_1": 0.56897, "precision_at_3": 0.5, "precision_at_5": 0.46207, "precision_at_10": 0.37069, "precision_at_20": 0.29569, "precision_at_50": 0.19034, "precision_at_100": 0.12552, "mrr_at_1": 0.5344827586206896, "mrr_at_3": 0.6609195402298851, "mrr_at_5": 0.6807471264367816, "mrr_at_10": 0.6899630541871922, "mrr_at_20": 0.6928555224267565, "mrr_at_50": 0.6928555224267565, "mrr_at_100": 0.6928555224267565, "naucs_at_1_max": 0.37666507028529217, "naucs_at_1_std": 0.2894204633073516, "naucs_at_1_diff1": 0.06680457995462946, "naucs_at_3_max": 0.20389557612636244, "naucs_at_3_std": 0.31592783637688704, "naucs_at_3_diff1": -0.1565134892191965, "naucs_at_5_max": 0.1649976619466542, "naucs_at_5_std": 0.3042431458723411, "naucs_at_5_diff1": -0.1168288609207652, "naucs_at_10_max": 0.1138447356410388, "naucs_at_10_std": 0.25980112410460104, "naucs_at_10_diff1": -0.2068349937437853, "naucs_at_20_max": 0.12497036442449745, "naucs_at_20_std": 0.35150590656604647, "naucs_at_20_diff1": -0.16276015911506578, "naucs_at_50_max": 0.04699279255355053, "naucs_at_50_std": 0.3178586829010547, "naucs_at_50_diff1": -0.14272426503448576, "naucs_at_100_max": 0.012556213981472325, "naucs_at_100_std": 0.2890479073896453, "naucs_at_100_diff1": -0.14921439241536574 }, "tatdqa_test": {"ndcg_at_1": 0.55043, "ndcg_at_3": 0.65383, "ndcg_at_5": 0.67972, "ndcg_at_10": 0.70182, "ndcg_at_20": 0.71531, "ndcg_at_100": 0.72798, "ndcg_at_1000": 0.73147, "map_at_1": 0.55043, "map_at_3": 0.6292, "map_at_5": 0.64363, "map_at_10": 0.65292, "map_at_20": 0.6567, "map_at_100": 0.65856, "map_at_1000": 0.65873, "recall_at_1": 0.55043, "recall_at_3": 0.72479, "recall_at_5": 0.78736, "recall_at_10": 0.8548, "recall_at_20": 0.90765, "recall_at_100": 0.97448, "recall_at_1000": 1.0, "precision_at_1": 0.55043, "precision_at_3": 0.2416, "precision_at_5": 0.15747, "precision_at_10": 0.08548, "precision_at_20": 0.04538, "precision_at_100": 0.00974, "precision_at_1000": 0.001, "mrr_at_1": 0.5498177399756987, "mrr_at_3": 0.6287970838396112, "mrr_at_5": 0.6434690157958688, "mrr_at_10": 0.6530902235337229, "mrr_at_20": 0.6567284540447504, "mrr_at_100": 0.658621212229932, "mrr_at_1000": 0.6587698669596878, "naucs_at_1_max": 0.27853365504109656, "naucs_at_1_std": -0.15447634347947511, "naucs_at_1_diff1": 0.7134897043728883, "naucs_at_3_max": 0.2862020499335342, "naucs_at_3_std": -0.1406768618430069, "naucs_at_3_diff1": 0.598287834466956, "naucs_at_5_max": 0.3082772653205975, "naucs_at_5_std": -0.1336535664737251, "naucs_at_5_diff1": 0.5847681853576895, "naucs_at_10_max": 0.4156924004771727, "naucs_at_10_std": -0.012527556044571743, "naucs_at_10_diff1": 0.5762936692610346, "naucs_at_20_max": 0.39150222479435925, "naucs_at_20_std": 0.004434403605241619, "naucs_at_20_diff1": 0.5602018066874601, "naucs_at_100_max": 0.39176268171942447, "naucs_at_100_std": 0.360160411337065, "naucs_at_100_diff1": 0.6304563665269014, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shiftproject_test": {"ndcg_at_1": 0.63, "ndcg_at_3": 0.76488, "ndcg_at_5": 0.79072, "ndcg_at_10": 0.80085, "ndcg_at_20": 0.80877, "ndcg_at_100": 0.81273, "ndcg_at_1000": 0.81415, "map_at_1": 0.63, "map_at_3": 0.735, "map_at_5": 0.75, "map_at_10": 0.75444, "map_at_20": 0.75679, "map_at_100": 0.75742, "map_at_1000": 0.75749, "recall_at_1": 0.63, "recall_at_3": 0.85, "recall_at_5": 0.91, "recall_at_10": 0.94, "recall_at_20": 0.97, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.63, "precision_at_3": 0.28333, "precision_at_5": 0.182, "precision_at_10": 0.094, "precision_at_20": 0.0485, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.65, "mrr_at_3": 0.7483333333333333, "mrr_at_5": 0.7608333333333333, "mrr_at_10": 0.7663888888888889, "mrr_at_20": 0.7679646464646465, "mrr_at_100": 0.768591427091427, "mrr_at_1000": 0.7686683501683501, "naucs_at_1_max": -0.061987809273100325, "naucs_at_1_std": -0.18575972672715027, "naucs_at_1_diff1": 0.7242513984863435, "naucs_at_3_max": 0.18226488773185812, "naucs_at_3_std": -0.19902375528799124, "naucs_at_3_diff1": 0.6877969410999027, "naucs_at_5_max": -0.24074074074073906, "naucs_at_5_std": -0.7552131963896631, "naucs_at_5_diff1": 0.6890756302521003, "naucs_at_10_max": -0.3181605975723611, "naucs_at_10_std": -1.07002801120448, "naucs_at_10_diff1": 0.7005135387488336, "naucs_at_20_max": -0.7791783380018631, "naucs_at_20_std": -0.5961718020541553, "naucs_at_20_diff1": 0.401027077497665, "naucs_at_100_max": 0.554154995331464, "naucs_at_100_std": -0.17133520074697067, "naucs_at_100_diff1": 0.35807656395892007, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.98131, "ndcg_at_5": 0.98131, "ndcg_at_10": 0.98131, "ndcg_at_20": 0.98131, "ndcg_at_100": 0.98321, "ndcg_at_1000": 0.98321, "map_at_1": 0.97, "map_at_3": 0.97833, "map_at_5": 0.97833, "map_at_10": 0.97833, "map_at_20": 0.97833, "map_at_100": 0.9786, "map_at_1000": 0.9786, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.97, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_100": 0.9786111111111111, "mrr_at_1000": 0.9786111111111111, "naucs_at_1_max": 0.41518829754123937, "naucs_at_1_std": -1.1517273576097102, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 1.0, "naucs_at_3_std": -1.1517273576097802, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": -1.1517273576097316, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": -1.1517273576097316, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.94786, "ndcg_at_5": 0.94786, "ndcg_at_10": 0.95142, "ndcg_at_20": 0.95142, "ndcg_at_100": 0.95142, "ndcg_at_1000": 0.95142, "map_at_1": 0.89, "map_at_3": 0.93333, "map_at_5": 0.93333, "map_at_10": 0.935, "map_at_20": 0.935, "map_at_100": 0.935, "map_at_1000": 0.935, "recall_at_1": 0.89, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.89, "mrr_at_3": 0.935, "mrr_at_5": 0.937, "mrr_at_10": 0.937, "mrr_at_20": 0.937, "mrr_at_100": 0.937, "mrr_at_1000": 0.937, "naucs_at_1_max": 0.5626234859548148, "naucs_at_1_std": 0.3716175586289838, "naucs_at_1_diff1": 0.834850957821493, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "arxivqa_test_subsampled": {"ndcg_at_1": 0.702, "ndcg_at_3": 0.76743, "ndcg_at_5": 0.77991, "ndcg_at_10": 0.79468, "ndcg_at_20": 0.80462, "ndcg_at_100": 0.8145, "ndcg_at_1000": 0.81657, "map_at_1": 0.702, "map_at_3": 0.75133, "map_at_5": 0.75833, "map_at_10": 0.76436, "map_at_20": 0.767, "map_at_100": 0.76831, "map_at_1000": 0.76839, "recall_at_1": 0.702, "recall_at_3": 0.814, "recall_at_5": 0.844, "recall_at_10": 0.89, "recall_at_20": 0.93, "recall_at_100": 0.984, "recall_at_1000": 1.0, "precision_at_1": 0.702, "precision_at_3": 0.27133, "precision_at_5": 0.1688, "precision_at_10": 0.089, "precision_at_20": 0.0465, "precision_at_100": 0.00984, "precision_at_1000": 0.001, "mrr_at_1": 0.7, "mrr_at_3": 0.7506666666666666, "mrr_at_5": 0.7575666666666667, "mrr_at_10": 0.76395, "mrr_at_20": 0.7664683712486344, "mrr_at_100": 0.7677856040474413, "mrr_at_1000": 0.767866298103699, "naucs_at_1_max": 0.6661847878465089, "naucs_at_1_std": -0.217519374789405, "naucs_at_1_diff1": 0.9071749220117173, "naucs_at_3_max": 0.697658762809486, "naucs_at_3_std": -0.18659371894467894, "naucs_at_3_diff1": 0.839073335969141, "naucs_at_5_max": 0.7447876108417706, "naucs_at_5_std": -0.17636087316938498, "naucs_at_5_diff1": 0.8526087065738912, "naucs_at_10_max": 0.7873636285542479, "naucs_at_10_std": -0.15812215445408626, "naucs_at_10_diff1": 0.8351945709131511, "naucs_at_20_max": 0.7746565292783784, "naucs_at_20_std": -0.23473389355742383, "naucs_at_20_diff1": 0.8061224489795931, "naucs_at_100_max": 0.8027544351073763, "naucs_at_100_std": 0.2001050420168152, "naucs_at_100_diff1": 0.7639472455648932, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_test_subsampled": {"ndcg_at_1": 0.49446, "ndcg_at_3": 0.54968, "ndcg_at_5": 0.57172, "ndcg_at_10": 0.59184, "ndcg_at_20": 0.60528, "ndcg_at_100": 0.62838, "ndcg_at_1000": 0.64353, "map_at_1": 0.49446, "map_at_3": 0.53585, "map_at_5": 0.54815, "map_at_10": 0.55648, "map_at_20": 0.56018, "map_at_100": 0.56326, "map_at_1000": 0.56389, "recall_at_1": 0.49446, "recall_at_3": 0.5898, "recall_at_5": 0.64302, "recall_at_10": 0.7051, "recall_at_20": 0.75831, "recall_at_100": 0.8847, "recall_at_1000": 1.0, "precision_at_1": 0.49446, "precision_at_3": 0.1966, "precision_at_5": 0.1286, "precision_at_10": 0.07051, "precision_at_20": 0.03792, "precision_at_100": 0.00885, "precision_at_1000": 0.001, "mrr_at_1": 0.49223946784922396, "mrr_at_3": 0.5339985218033999, "mrr_at_5": 0.5473022912047303, "mrr_at_10": 0.5553206278816035, "mrr_at_20": 0.5589821565391849, "mrr_at_100": 0.5619819074681309, "mrr_at_1000": 0.5626456935597429, "naucs_at_1_max": 0.32878731695222985, "naucs_at_1_std": 0.15591990126747984, "naucs_at_1_diff1": 0.8493613636054403, "naucs_at_3_max": 0.23098165064202703, "naucs_at_3_std": 0.1729343384390047, "naucs_at_3_diff1": 0.79653911998206, "naucs_at_5_max": 0.20443193032581358, "naucs_at_5_std": 0.22886452804275878, "naucs_at_5_diff1": 0.7709831107458899, "naucs_at_10_max": 0.11430383031823631, "naucs_at_10_std": 0.3505331687518112, "naucs_at_10_diff1": 0.7655459781075918, "naucs_at_20_max": 0.06375294772557255, "naucs_at_20_std": 0.47875225635393476, "naucs_at_20_diff1": 0.7402047132383933, "naucs_at_100_max": -0.17203513994521832, "naucs_at_100_std": 0.719961371895444, "naucs_at_100_diff1": 0.7516431118242167, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.94, "ndcg_at_3": 0.96262, "ndcg_at_5": 0.96693, "ndcg_at_10": 0.96693, "ndcg_at_20": 0.96948, "ndcg_at_100": 0.96948, "ndcg_at_1000": 0.96948, "map_at_1": 0.94, "map_at_3": 0.95667, "map_at_5": 0.95917, "map_at_10": 0.95917, "map_at_20": 0.95988, "map_at_100": 0.95988, "map_at_1000": 0.95988, "recall_at_1": 0.94, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.96, "mrr_at_3": 0.9683333333333333, "mrr_at_5": 0.9708333333333333, "mrr_at_10": 0.9708333333333333, "mrr_at_20": 0.971547619047619, "mrr_at_100": 0.971547619047619, "mrr_at_1000": 0.971547619047619, "naucs_at_1_max": 0.4701991907874248, "naucs_at_1_std": -0.2544351073762836, "naucs_at_1_diff1": 0.9319172113289763, "naucs_at_3_max": -0.004668534080301393, "naucs_at_3_std": -0.5088702147525775, "naucs_at_3_diff1": 0.7957516339869218, "naucs_at_5_max": -0.5634920634920767, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 0.7222222222222276, "naucs_at_10_max": -0.5634920634920767, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 0.7222222222222276, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_test_subsampled": {"ndcg_at_1": 0.76518, "ndcg_at_3": 0.81846, "ndcg_at_5": 0.82779, "ndcg_at_10": 0.84084, "ndcg_at_20": 0.8454, "ndcg_at_100": 0.85306, "ndcg_at_1000": 0.85616, "map_at_1": 0.76518, "map_at_3": 0.80601, "map_at_5": 0.81127, "map_at_10": 0.81663, "map_at_20": 0.81786, "map_at_100": 0.81898, "map_at_1000": 0.81909, "recall_at_1": 0.76518, "recall_at_3": 0.85425, "recall_at_5": 0.87652, "recall_at_10": 0.917, "recall_at_20": 0.93522, "recall_at_100": 0.97571, "recall_at_1000": 1.0, "precision_at_1": 0.76518, "precision_at_3": 0.28475, "precision_at_5": 0.1753, "precision_at_10": 0.0917, "precision_at_20": 0.04676, "precision_at_100": 0.00976, "precision_at_1000": 0.001, "mrr_at_1": 0.7672064777327935, "mrr_at_3": 0.8076923076923077, "mrr_at_5": 0.8130566801619433, "mrr_at_10": 0.8183712486344065, "mrr_at_20": 0.8196021674007504, "mrr_at_100": 0.8207219702130965, "mrr_at_1000": 0.8208340179158132, "naucs_at_1_max": 0.526736932815251, "naucs_at_1_std": -0.08005662601905021, "naucs_at_1_diff1": 0.8906383040101385, "naucs_at_3_max": 0.5628217589502187, "naucs_at_3_std": 0.07419124093710198, "naucs_at_3_diff1": 0.819273316147957, "naucs_at_5_max": 0.6142211700114351, "naucs_at_5_std": 0.13484006376865668, "naucs_at_5_diff1": 0.7974348993177082, "naucs_at_10_max": 0.5107732925204674, "naucs_at_10_std": 0.08282924765220158, "naucs_at_10_diff1": 0.7501472816409511, "naucs_at_20_max": 0.6069714717598735, "naucs_at_20_std": 0.24818260935350467, "naucs_at_20_diff1": 0.7972653744711068, "naucs_at_100_max": 0.7715054503371943, "naucs_at_100_std": 0.6498035527508992, "naucs_at_100_diff1": 0.7627810617053321, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95155, "ndcg_at_5": 0.95155, "ndcg_at_10": 0.95155, "ndcg_at_20": 0.95425, "ndcg_at_100": 0.95618, "ndcg_at_1000": 0.95745, "map_at_1": 0.92, "map_at_3": 0.945, "map_at_5": 0.945, "map_at_10": 0.945, "map_at_20": 0.94583, "map_at_100": 0.94612, "map_at_1000": 0.94616, "recall_at_1": 0.92, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.92, "mrr_at_3": 0.945, "mrr_at_5": 0.945, "mrr_at_10": 0.945, "mrr_at_20": 0.9458333333333333, "mrr_at_100": 0.9461190476190476, "mrr_at_1000": 0.9461614205004035, "naucs_at_1_max": 0.5274859943977597, "naucs_at_1_std": -0.6526027077497683, "naucs_at_1_diff1": 0.9673202614379083, "naucs_at_3_max": 0.807812013694371, "naucs_at_3_std": -1.1517273576097098, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.807812013694364, "naucs_at_5_std": -1.1517273576097071, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.807812013694364, "naucs_at_10_std": -1.1517273576097071, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": -0.8576097105508901, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": -0.5634920634920583, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.85, "ndcg_at_3": 0.88465, "ndcg_at_5": 0.89664, "ndcg_at_10": 0.90491, "ndcg_at_20": 0.91464, "ndcg_at_100": 0.91464, "ndcg_at_1000": 0.91464, "map_at_1": 0.85, "map_at_3": 0.87679, "map_at_5": 0.88357, "map_at_10": 0.8871, "map_at_20": 0.88966, "map_at_100": 0.88966, "map_at_1000": 0.88966, "recall_at_1": 0.85, "recall_at_3": 0.90714, "recall_at_5": 0.93571, "recall_at_10": 0.96071, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.30238, "precision_at_5": 0.18714, "precision_at_10": 0.09607, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.85, "mrr_at_3": 0.8779761904761905, "mrr_at_5": 0.8838690476190476, "mrr_at_10": 0.8874900793650794, "mrr_at_20": 0.8898663632119514, "mrr_at_100": 0.8900364312391623, "mrr_at_1000": 0.8900364312391623, "naucs_at_1_max": 0.6249012133327131, "naucs_at_1_std": 0.27568453349449135, "naucs_at_1_diff1": 0.851971084561387, "naucs_at_3_max": 0.7154528478057898, "naucs_at_3_std": 0.3263664440135032, "naucs_at_3_diff1": 0.8540364863894296, "naucs_at_5_max": 0.7756769374416456, "naucs_at_5_std": 0.5388525780682669, "naucs_at_5_diff1": 0.8674914410208527, "naucs_at_10_max": 0.7903403785756714, "naucs_at_10_std": 0.6866564807741266, "naucs_at_10_diff1": 0.832187420422716, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}}