| {"vidore/restaurant_esg_reports_beir": {"ndcg_at_1": 0.26923, "ndcg_at_3": 0.3045, "ndcg_at_5": 0.32114, "ndcg_at_10": 0.35676, "ndcg_at_20": 0.38618, "ndcg_at_50": 0.42052, "ndcg_at_100": 0.43636, "map_at_1": 0.18333, "map_at_3": 0.24391, "map_at_5": 0.26139, "map_at_10": 0.27761, "map_at_20": 0.28675, "map_at_50": 0.29345, "map_at_100": 0.29582, "recall_at_1": 0.18333, "recall_at_3": 0.31923, "recall_at_5": 0.37917, "recall_at_10": 0.49423, "recall_at_20": 0.5946, "recall_at_50": 0.73516, "recall_at_100": 0.80377, "precision_at_1": 0.26923, "precision_at_3": 0.16667, "precision_at_5": 0.12308, "precision_at_10": 0.08269, "precision_at_20": 0.05385, "precision_at_50": 0.02846, "precision_at_100": 0.01692, "mrr_at_1": 0.23076923076923078, "mrr_at_3": 0.3269230769230769, "mrr_at_5": 0.34038461538461545, "mrr_at_10": 0.3608669108669108, "mrr_at_20": 0.3674438809054194, "mrr_at_50": 0.37084971190489147, "mrr_at_100": 0.3713579847136337, "naucs_at_1_max": 0.021151055203054405, "naucs_at_1_std": -0.029813730522622425, "naucs_at_1_diff1": 0.25067341614916727, "naucs_at_3_max": -0.04143164067060292, "naucs_at_3_std": 0.00730042874767451, "naucs_at_3_diff1": 0.3380054031613406, "naucs_at_5_max": -0.1347500524905061, "naucs_at_5_std": -0.07424067226444503, "naucs_at_5_diff1": 0.2669507130764703, "naucs_at_10_max": -0.210749971018401, "naucs_at_10_std": -0.21961340947875993, "naucs_at_10_diff1": 0.16640286626959264, "naucs_at_20_max": -0.21891469085057838, "naucs_at_20_std": -0.23005504027592547, "naucs_at_20_diff1": 0.22055505233932227, "naucs_at_50_max": -0.21310892162468298, "naucs_at_50_std": -0.20734986109292497, "naucs_at_50_diff1": 0.09142935454845501, "naucs_at_100_max": -0.22587806170824862, "naucs_at_100_std": -0.21517085450107218, "naucs_at_100_diff1": 0.09673268004702926}, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.51293, | |
| "ndcg_at_3": 0.49649, | |
| "ndcg_at_5": 0.47893, | |
| "ndcg_at_10": 0.48004, | |
| "ndcg_at_20": 0.51068, | |
| "ndcg_at_50": 0.59426, | |
| "ndcg_at_100": 0.63376, | |
| "map_at_1": 0.10147, | |
| "map_at_3": 0.15877, | |
| "map_at_5": 0.19559, | |
| "map_at_10": 0.25349, | |
| "map_at_20": 0.30278, | |
| "map_at_50": 0.36487, | |
| "map_at_100": 0.39199, | |
| "recall_at_1": 0.10147, | |
| "recall_at_3": 0.17902, | |
| "recall_at_5": 0.23979, | |
| "recall_at_10": 0.373, | |
| "recall_at_20": 0.52076, | |
| "recall_at_50": 0.76023, | |
| "recall_at_100": 0.88108, | |
| "precision_at_1": 0.51293, | |
| "precision_at_3": 0.45259, | |
| "precision_at_5": 0.41293, | |
| "precision_at_10": 0.35259, | |
| "precision_at_20": 0.27909, | |
| "precision_at_50": 0.18828, | |
| "precision_at_100": 0.12366, | |
| "mrr_at_1": 0.4956896551724138, | |
| "mrr_at_3": 0.6149425287356322, | |
| "mrr_at_5": 0.6311063218390804, | |
| "mrr_at_10": 0.6426518883415435, | |
| "mrr_at_20": 0.6473663956998068, | |
| "mrr_at_50": 0.6479916726305025, | |
| "mrr_at_100": 0.6480600908023689, | |
| "naucs_at_1_max": 0.2909740112601449, | |
| "naucs_at_1_std": 0.34007231211982364, | |
| "naucs_at_1_diff1": 0.02010046418675761, | |
| "naucs_at_3_max": 0.1462981439133328, | |
| "naucs_at_3_std": 0.23727806279449618, | |
| "naucs_at_3_diff1": -0.045115704507227815, | |
| "naucs_at_5_max": 0.16402901891891258, | |
| "naucs_at_5_std": 0.2788211772941985, | |
| "naucs_at_5_diff1": -0.03422957119067939, | |
| "naucs_at_10_max": 0.1304595275607474, | |
| "naucs_at_10_std": 0.27024887845192075, | |
| "naucs_at_10_diff1": -0.06184582606627896, | |
| "naucs_at_20_max": 0.0827755061718375, | |
| "naucs_at_20_std": 0.32564544404546836, | |
| "naucs_at_20_diff1": -0.1398291345769381, | |
| "naucs_at_50_max": 0.024145276015777446, | |
| "naucs_at_50_std": 0.3093730698130927, | |
| "naucs_at_50_diff1": -0.16628805756811568, | |
| "naucs_at_100_max": 0.0009219308736878375, | |
| "naucs_at_100_std": 0.2960260201661105, | |
| "naucs_at_100_diff1": -0.16354176688825178 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.44444, | |
| "ndcg_at_3": 0.48365, | |
| "ndcg_at_5": 0.4576, | |
| "ndcg_at_10": 0.47015, | |
| "ndcg_at_20": 0.48904, | |
| "ndcg_at_50": 0.54777, | |
| "ndcg_at_100": 0.58161, | |
| "map_at_1": 0.16613, | |
| "map_at_3": 0.28304, | |
| "map_at_5": 0.30204, | |
| "map_at_10": 0.33659, | |
| "map_at_20": 0.35316, | |
| "map_at_50": 0.37788, | |
| "map_at_100": 0.38576, | |
| "recall_at_1": 0.16613, | |
| "recall_at_3": 0.4128, | |
| "recall_at_5": 0.4461, | |
| "recall_at_10": 0.56445, | |
| "recall_at_20": 0.62203, | |
| "recall_at_50": 0.78078, | |
| "recall_at_100": 0.91945, | |
| "precision_at_1": 0.44444, | |
| "precision_at_3": 0.35648, | |
| "precision_at_5": 0.26944, | |
| "precision_at_10": 0.18333, | |
| "precision_at_20": 0.11875, | |
| "precision_at_50": 0.07028, | |
| "precision_at_100": 0.04181, | |
| "mrr_at_1": 0.4305555555555556, | |
| "mrr_at_3": 0.5486111111111112, | |
| "mrr_at_5": 0.5625, | |
| "mrr_at_10": 0.5719907407407407, | |
| "mrr_at_20": 0.5744107744107745, | |
| "mrr_at_50": 0.5755691882594056, | |
| "mrr_at_100": 0.5766185382360561, | |
| "naucs_at_1_max": -0.10844176576631265, | |
| "naucs_at_1_std": 0.11984795634852517, | |
| "naucs_at_1_diff1": 0.34926607045706065, | |
| "naucs_at_3_max": -0.16882440588261344, | |
| "naucs_at_3_std": 0.04935292481231464, | |
| "naucs_at_3_diff1": 0.07094770137246685, | |
| "naucs_at_5_max": -0.2677975915325592, | |
| "naucs_at_5_std": -0.013445545910716006, | |
| "naucs_at_5_diff1": 0.03560103469628687, | |
| "naucs_at_10_max": -0.2905586858742058, | |
| "naucs_at_10_std": 0.0024045562548918554, | |
| "naucs_at_10_diff1": -0.08824087823808019, | |
| "naucs_at_20_max": -0.40897273226767533, | |
| "naucs_at_20_std": -0.1138141721108663, | |
| "naucs_at_20_diff1": -0.10887906242990919, | |
| "naucs_at_50_max": -0.42683318222629896, | |
| "naucs_at_50_std": -0.13076918365032622, | |
| "naucs_at_50_diff1": -0.10724840220966861, | |
| "naucs_at_100_max": -0.44961995894718576, | |
| "naucs_at_100_std": -0.1844335335164004, | |
| "naucs_at_100_diff1": -0.08314001637229608 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.5125, | |
| "ndcg_at_3": 0.54949, | |
| "ndcg_at_5": 0.5847, | |
| "ndcg_at_10": 0.61161, | |
| "ndcg_at_20": 0.64415, | |
| "ndcg_at_50": 0.66856, | |
| "ndcg_at_100": 0.67862, | |
| "map_at_1": 0.31246, | |
| "map_at_3": 0.4406, | |
| "map_at_5": 0.48617, | |
| "map_at_10": 0.51574, | |
| "map_at_20": 0.53362, | |
| "map_at_50": 0.54211, | |
| "map_at_100": 0.54437, | |
| "recall_at_1": 0.31246, | |
| "recall_at_3": 0.52965, | |
| "recall_at_5": 0.6369, | |
| "recall_at_10": 0.72113, | |
| "recall_at_20": 0.80597, | |
| "recall_at_50": 0.88539, | |
| "recall_at_100": 0.92369, | |
| "precision_at_1": 0.5125, | |
| "precision_at_3": 0.34167, | |
| "precision_at_5": 0.27, | |
| "precision_at_10": 0.16937, | |
| "precision_at_20": 0.105, | |
| "precision_at_50": 0.04938, | |
| "precision_at_100": 0.02681, | |
| "mrr_at_1": 0.5375, | |
| "mrr_at_3": 0.6479166666666667, | |
| "mrr_at_5": 0.6663541666666666, | |
| "mrr_at_10": 0.674516369047619, | |
| "mrr_at_20": 0.6765751027975293, | |
| "mrr_at_50": 0.6772030349370826, | |
| "mrr_at_100": 0.677426534298513, | |
| "naucs_at_1_max": 0.34895314898857854, | |
| "naucs_at_1_std": -0.07257942064683737, | |
| "naucs_at_1_diff1": 0.5293978708686925, | |
| "naucs_at_3_max": 0.18142528545936487, | |
| "naucs_at_3_std": -0.16137866300838627, | |
| "naucs_at_3_diff1": -0.0014234288130872189, | |
| "naucs_at_5_max": 0.014672569636598775, | |
| "naucs_at_5_std": -0.21002767017155474, | |
| "naucs_at_5_diff1": -0.1473381294964024, | |
| "naucs_at_10_max": -0.06629123564607431, | |
| "naucs_at_10_std": -0.2046018497631402, | |
| "naucs_at_10_diff1": -0.252293405519212, | |
| "naucs_at_20_max": -0.16779142727147645, | |
| "naucs_at_20_std": -0.1998645631632962, | |
| "naucs_at_20_diff1": -0.3377181009216048, | |
| "naucs_at_50_max": -0.2319362541617097, | |
| "naucs_at_50_std": -0.20139508212155197, | |
| "naucs_at_50_diff1": -0.35750220388740067, | |
| "naucs_at_100_max": -0.26209531433075695, | |
| "naucs_at_100_std": -0.19593561386651334, | |
| "naucs_at_100_diff1": -0.36382106679824544 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.47368, | |
| "ndcg_at_3": 0.48077, | |
| "ndcg_at_5": 0.51913, | |
| "ndcg_at_10": 0.55339, | |
| "ndcg_at_20": 0.58423, | |
| "ndcg_at_50": 0.62437, | |
| "ndcg_at_100": 0.63473, | |
| "map_at_1": 0.22899, | |
| "map_at_3": 0.33833, | |
| "map_at_5": 0.38792, | |
| "map_at_10": 0.4251, | |
| "map_at_20": 0.44759, | |
| "map_at_50": 0.46783, | |
| "map_at_100": 0.47287, | |
| "recall_at_1": 0.22899, | |
| "recall_at_3": 0.42834, | |
| "recall_at_5": 0.54461, | |
| "recall_at_10": 0.66769, | |
| "recall_at_20": 0.77688, | |
| "recall_at_50": 0.90877, | |
| "recall_at_100": 0.93622, | |
| "precision_at_1": 0.47368, | |
| "precision_at_3": 0.32164, | |
| "precision_at_5": 0.27719, | |
| "precision_at_10": 0.18772, | |
| "precision_at_20": 0.12193, | |
| "precision_at_50": 0.06667, | |
| "precision_at_100": 0.03702, | |
| "mrr_at_1": 0.47368421052631576, | |
| "mrr_at_3": 0.5994152046783625, | |
| "mrr_at_5": 0.6152046783625732, | |
| "mrr_at_10": 0.622270955165692, | |
| "mrr_at_20": 0.6259990253411306, | |
| "mrr_at_50": 0.6267578992701391, | |
| "mrr_at_100": 0.6267578992701391, | |
| "naucs_at_1_max": -0.3156199695209114, | |
| "naucs_at_1_std": -0.26486119453698037, | |
| "naucs_at_1_diff1": 0.11795422382927509, | |
| "naucs_at_3_max": -0.2735188964882074, | |
| "naucs_at_3_std": -0.23310566021944892, | |
| "naucs_at_3_diff1": -0.19449853074061657, | |
| "naucs_at_5_max": -0.2421479017067623, | |
| "naucs_at_5_std": -0.13869753650963065, | |
| "naucs_at_5_diff1": -0.3054740364166881, | |
| "naucs_at_10_max": -0.25873997567219825, | |
| "naucs_at_10_std": -0.1458966410575204, | |
| "naucs_at_10_diff1": -0.3032000489911622, | |
| "naucs_at_20_max": -0.3256819936368209, | |
| "naucs_at_20_std": -0.21934883184399268, | |
| "naucs_at_20_diff1": -0.3948318811395443, | |
| "naucs_at_50_max": -0.3584500031494027, | |
| "naucs_at_50_std": -0.21644313778125723, | |
| "naucs_at_50_diff1": -0.4136042140657271, | |
| "naucs_at_100_max": -0.3530688199691307, | |
| "naucs_at_100_std": -0.19076877127775468, | |
| "naucs_at_100_diff1": -0.42236105076593466 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.49561, | |
| "ndcg_at_3": 0.51051, | |
| "ndcg_at_5": 0.53966, | |
| "ndcg_at_10": 0.57033, | |
| "ndcg_at_20": 0.60598, | |
| "ndcg_at_50": 0.6415, | |
| "ndcg_at_100": 0.65306, | |
| "map_at_1": 0.24154, | |
| "map_at_3": 0.36508, | |
| "map_at_5": 0.40846, | |
| "map_at_10": 0.445, | |
| "map_at_20": 0.46996, | |
| "map_at_50": 0.48868, | |
| "map_at_100": 0.49392, | |
| "recall_at_1": 0.24154, | |
| "recall_at_3": 0.46057, | |
| "recall_at_5": 0.5649, | |
| "recall_at_10": 0.67685, | |
| "recall_at_20": 0.79962, | |
| "recall_at_50": 0.91497, | |
| "recall_at_100": 0.94749, | |
| "precision_at_1": 0.49561, | |
| "precision_at_3": 0.34649, | |
| "precision_at_5": 0.2807, | |
| "precision_at_10": 0.19079, | |
| "precision_at_20": 0.12544, | |
| "precision_at_50": 0.06693, | |
| "precision_at_100": 0.03737, | |
| "mrr_at_1": 0.4692982456140351, | |
| "mrr_at_3": 0.6052631578947368, | |
| "mrr_at_5": 0.618421052631579, | |
| "mrr_at_10": 0.6259433305485937, | |
| "mrr_at_20": 0.630267555040449, | |
| "mrr_at_50": 0.6310822394721419, | |
| "mrr_at_100": 0.6311605602741469, | |
| "naucs_at_1_max": -0.27139902693937923, | |
| "naucs_at_1_std": -0.28335975541834757, | |
| "naucs_at_1_diff1": 0.03844931932885215, | |
| "naucs_at_3_max": -0.2333237939707581, | |
| "naucs_at_3_std": -0.1653044765750373, | |
| "naucs_at_3_diff1": -0.10443046117117412, | |
| "naucs_at_5_max": -0.19561866039792095, | |
| "naucs_at_5_std": -0.09561859939322022, | |
| "naucs_at_5_diff1": -0.18423887457809085, | |
| "naucs_at_10_max": -0.2060244561628858, | |
| "naucs_at_10_std": -0.08726363088972981, | |
| "naucs_at_10_diff1": -0.20492069680185956, | |
| "naucs_at_20_max": -0.25096317818789565, | |
| "naucs_at_20_std": -0.12216017305154458, | |
| "naucs_at_20_diff1": -0.26997057451690837, | |
| "naucs_at_50_max": -0.30710698032302514, | |
| "naucs_at_50_std": -0.13529052277664438, | |
| "naucs_at_50_diff1": -0.29528418472148443, | |
| "naucs_at_100_max": -0.3124105461583974, | |
| "naucs_at_100_std": -0.11726920470788517, | |
| "naucs_at_100_diff1": -0.2735365969618431 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.50938, | |
| "ndcg_at_3": 0.53068, | |
| "ndcg_at_5": 0.55676, | |
| "ndcg_at_10": 0.58653, | |
| "ndcg_at_20": 0.61833, | |
| "ndcg_at_50": 0.64235, | |
| "ndcg_at_100": 0.6544, | |
| "map_at_1": 0.31519, | |
| "map_at_3": 0.42911, | |
| "map_at_5": 0.46586, | |
| "map_at_10": 0.49473, | |
| "map_at_20": 0.51197, | |
| "map_at_50": 0.52001, | |
| "map_at_100": 0.52243, | |
| "recall_at_1": 0.31519, | |
| "recall_at_3": 0.50707, | |
| "recall_at_5": 0.59431, | |
| "recall_at_10": 0.68853, | |
| "recall_at_20": 0.77673, | |
| "recall_at_50": 0.8526, | |
| "recall_at_100": 0.8999, | |
| "precision_at_1": 0.50938, | |
| "precision_at_3": 0.325, | |
| "precision_at_5": 0.25125, | |
| "precision_at_10": 0.16016, | |
| "precision_at_20": 0.09891, | |
| "precision_at_50": 0.04688, | |
| "precision_at_100": 0.02587, | |
| "mrr_at_1": 0.5234375, | |
| "mrr_at_3": 0.6184895833333333, | |
| "mrr_at_5": 0.6364583333333333, | |
| "mrr_at_10": 0.6463244047619048, | |
| "mrr_at_20": 0.6495780415378181, | |
| "mrr_at_50": 0.6503474764286958, | |
| "mrr_at_100": 0.6505948135497464, | |
| "naucs_at_1_max": 0.4131418193131225, | |
| "naucs_at_1_std": 0.06757995282919683, | |
| "naucs_at_1_diff1": 0.516900768942945, | |
| "naucs_at_3_max": 0.18254564757418557, | |
| "naucs_at_3_std": -0.04347712846310109, | |
| "naucs_at_3_diff1": 0.03059708341704999, | |
| "naucs_at_5_max": 0.06972269700281146, | |
| "naucs_at_5_std": -0.10787077250693489, | |
| "naucs_at_5_diff1": -0.08493373308302267, | |
| "naucs_at_10_max": -0.023642522063642934, | |
| "naucs_at_10_std": -0.11714079374203694, | |
| "naucs_at_10_diff1": -0.17673365636114774, | |
| "naucs_at_20_max": -0.09569239071775491, | |
| "naucs_at_20_std": -0.11183756071235801, | |
| "naucs_at_20_diff1": -0.241802482460874, | |
| "naucs_at_50_max": -0.1488266985733028, | |
| "naucs_at_50_std": -0.11009165758378252, | |
| "naucs_at_50_diff1": -0.2751290287492702, | |
| "naucs_at_100_max": -0.18389216502938294, | |
| "naucs_at_100_std": -0.1152902550355418, | |
| "naucs_at_100_diff1": -0.2899356806702993 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0": { | |
| "ndcg_at_1": 0.5, | |
| "ndcg_at_3": 0.59177, | |
| "ndcg_at_5": 0.56037, | |
| "ndcg_at_10": 0.56184, | |
| "ndcg_at_20": 0.56464, | |
| "ndcg_at_50": 0.62655, | |
| "ndcg_at_100": 0.64607, | |
| "map_at_1": 0.18432, | |
| "map_at_3": 0.35086, | |
| "map_at_5": 0.37963, | |
| "map_at_10": 0.4161, | |
| "map_at_20": 0.42647, | |
| "map_at_50": 0.45181, | |
| "map_at_100": 0.45911, | |
| "recall_at_1": 0.18432, | |
| "recall_at_3": 0.54724, | |
| "recall_at_5": 0.58705, | |
| "recall_at_10": 0.68342, | |
| "recall_at_20": 0.7113, | |
| "recall_at_50": 0.88439, | |
| "recall_at_100": 0.9347, | |
| "precision_at_1": 0.5, | |
| "precision_at_3": 0.44444, | |
| "precision_at_5": 0.33333, | |
| "precision_at_10": 0.21667, | |
| "precision_at_20": 0.12222, | |
| "precision_at_50": 0.07333, | |
| "precision_at_100": 0.04278, | |
| "mrr_at_1": 0.5, | |
| "mrr_at_3": 0.6296296296296297, | |
| "mrr_at_5": 0.6574074074074074, | |
| "mrr_at_10": 0.6574074074074074, | |
| "mrr_at_20": 0.6574074074074074, | |
| "mrr_at_50": 0.658641975308642, | |
| "mrr_at_100": 0.658641975308642, | |
| "naucs_at_1_max": -0.20471943280726257, | |
| "naucs_at_1_std": 0.18546089088182968, | |
| "naucs_at_1_diff1": 0.7445083027491819, | |
| "naucs_at_3_max": -0.5294402069183541, | |
| "naucs_at_3_std": -0.04763265145855668, | |
| "naucs_at_3_diff1": 0.253185276095856, | |
| "naucs_at_5_max": -0.669223474626299, | |
| "naucs_at_5_std": -0.08636482482813103, | |
| "naucs_at_5_diff1": 0.2760137984114396, | |
| "naucs_at_10_max": -0.738950395976398, | |
| "naucs_at_10_std": -0.1685495451706665, | |
| "naucs_at_10_diff1": 0.07368539851746013, | |
| "naucs_at_20_max": -0.7021396616750007, | |
| "naucs_at_20_std": -0.11462769019487411, | |
| "naucs_at_20_diff1": 0.07121119110741668, | |
| "naucs_at_50_max": -0.7063938743299508, | |
| "naucs_at_50_std": -0.15679147163325738, | |
| "naucs_at_50_diff1": -0.013733388816798229, | |
| "naucs_at_100_max": -0.6781868824447755, | |
| "naucs_at_100_std": -0.16240281161384443, | |
| "naucs_at_100_diff1": 0.04104876407606115 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.56897, | |
| "ndcg_at_3": 0.5475, | |
| "ndcg_at_5": 0.531, | |
| "ndcg_at_10": 0.51659, | |
| "ndcg_at_20": 0.5526, | |
| "ndcg_at_50": 0.62369, | |
| "ndcg_at_100": 0.66764, | |
| "map_at_1": 0.10475, | |
| "map_at_3": 0.17297, | |
| "map_at_5": 0.22022, | |
| "map_at_10": 0.27707, | |
| "map_at_20": 0.33372, | |
| "map_at_50": 0.39288, | |
| "map_at_100": 0.4226, | |
| "recall_at_1": 0.10475, | |
| "recall_at_3": 0.20237, | |
| "recall_at_5": 0.26964, | |
| "recall_at_10": 0.39933, | |
| "recall_at_20": 0.56307, | |
| "recall_at_50": 0.77351, | |
| "recall_at_100": 0.90788, | |
| "precision_at_1": 0.56897, | |
| "precision_at_3": 0.5, | |
| "precision_at_5": 0.46207, | |
| "precision_at_10": 0.37069, | |
| "precision_at_20": 0.29569, | |
| "precision_at_50": 0.19034, | |
| "precision_at_100": 0.12552, | |
| "mrr_at_1": 0.5344827586206896, | |
| "mrr_at_3": 0.6609195402298851, | |
| "mrr_at_5": 0.6807471264367816, | |
| "mrr_at_10": 0.6899630541871922, | |
| "mrr_at_20": 0.6928555224267565, | |
| "mrr_at_50": 0.6928555224267565, | |
| "mrr_at_100": 0.6928555224267565, | |
| "naucs_at_1_max": 0.37666507028529217, | |
| "naucs_at_1_std": 0.2894204633073516, | |
| "naucs_at_1_diff1": 0.06680457995462946, | |
| "naucs_at_3_max": 0.20389557612636244, | |
| "naucs_at_3_std": 0.31592783637688704, | |
| "naucs_at_3_diff1": -0.1565134892191965, | |
| "naucs_at_5_max": 0.1649976619466542, | |
| "naucs_at_5_std": 0.3042431458723411, | |
| "naucs_at_5_diff1": -0.1168288609207652, | |
| "naucs_at_10_max": 0.1138447356410388, | |
| "naucs_at_10_std": 0.25980112410460104, | |
| "naucs_at_10_diff1": -0.2068349937437853, | |
| "naucs_at_20_max": 0.12497036442449745, | |
| "naucs_at_20_std": 0.35150590656604647, | |
| "naucs_at_20_diff1": -0.16276015911506578, | |
| "naucs_at_50_max": 0.04699279255355053, | |
| "naucs_at_50_std": 0.3178586829010547, | |
| "naucs_at_50_diff1": -0.14272426503448576, | |
| "naucs_at_100_max": 0.012556213981472325, | |
| "naucs_at_100_std": 0.2890479073896453, | |
| "naucs_at_100_diff1": -0.14921439241536574 | |
| }, "tatdqa_test": {"ndcg_at_1": 0.55043, "ndcg_at_3": 0.65383, "ndcg_at_5": 0.67972, "ndcg_at_10": 0.70182, "ndcg_at_20": 0.71531, "ndcg_at_100": 0.72798, "ndcg_at_1000": 0.73147, "map_at_1": 0.55043, "map_at_3": 0.6292, "map_at_5": 0.64363, "map_at_10": 0.65292, "map_at_20": 0.6567, "map_at_100": 0.65856, "map_at_1000": 0.65873, "recall_at_1": 0.55043, "recall_at_3": 0.72479, "recall_at_5": 0.78736, "recall_at_10": 0.8548, "recall_at_20": 0.90765, "recall_at_100": 0.97448, "recall_at_1000": 1.0, "precision_at_1": 0.55043, "precision_at_3": 0.2416, "precision_at_5": 0.15747, "precision_at_10": 0.08548, "precision_at_20": 0.04538, "precision_at_100": 0.00974, "precision_at_1000": 0.001, "mrr_at_1": 0.5498177399756987, "mrr_at_3": 0.6287970838396112, "mrr_at_5": 0.6434690157958688, "mrr_at_10": 0.6530902235337229, "mrr_at_20": 0.6567284540447504, "mrr_at_100": 0.658621212229932, "mrr_at_1000": 0.6587698669596878, "naucs_at_1_max": 0.27853365504109656, "naucs_at_1_std": -0.15447634347947511, "naucs_at_1_diff1": 0.7134897043728883, "naucs_at_3_max": 0.2862020499335342, "naucs_at_3_std": -0.1406768618430069, "naucs_at_3_diff1": 0.598287834466956, "naucs_at_5_max": 0.3082772653205975, "naucs_at_5_std": -0.1336535664737251, "naucs_at_5_diff1": 0.5847681853576895, "naucs_at_10_max": 0.4156924004771727, "naucs_at_10_std": -0.012527556044571743, "naucs_at_10_diff1": 0.5762936692610346, "naucs_at_20_max": 0.39150222479435925, "naucs_at_20_std": 0.004434403605241619, "naucs_at_20_diff1": 0.5602018066874601, "naucs_at_100_max": 0.39176268171942447, "naucs_at_100_std": 0.360160411337065, "naucs_at_100_diff1": 0.6304563665269014, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "shiftproject_test": {"ndcg_at_1": 0.63, "ndcg_at_3": 0.76488, "ndcg_at_5": 0.79072, "ndcg_at_10": 0.80085, "ndcg_at_20": 0.80877, "ndcg_at_100": 0.81273, "ndcg_at_1000": 0.81415, "map_at_1": 0.63, "map_at_3": 0.735, "map_at_5": 0.75, "map_at_10": 0.75444, "map_at_20": 0.75679, "map_at_100": 0.75742, "map_at_1000": 0.75749, "recall_at_1": 0.63, "recall_at_3": 0.85, "recall_at_5": 0.91, "recall_at_10": 0.94, "recall_at_20": 0.97, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.63, "precision_at_3": 0.28333, "precision_at_5": 0.182, "precision_at_10": 0.094, "precision_at_20": 0.0485, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.65, "mrr_at_3": 0.7483333333333333, "mrr_at_5": 0.7608333333333333, "mrr_at_10": 0.7663888888888889, "mrr_at_20": 0.7679646464646465, "mrr_at_100": 0.768591427091427, "mrr_at_1000": 0.7686683501683501, "naucs_at_1_max": -0.061987809273100325, "naucs_at_1_std": -0.18575972672715027, "naucs_at_1_diff1": 0.7242513984863435, "naucs_at_3_max": 0.18226488773185812, "naucs_at_3_std": -0.19902375528799124, "naucs_at_3_diff1": 0.6877969410999027, "naucs_at_5_max": -0.24074074074073906, "naucs_at_5_std": -0.7552131963896631, "naucs_at_5_diff1": 0.6890756302521003, "naucs_at_10_max": -0.3181605975723611, "naucs_at_10_std": -1.07002801120448, "naucs_at_10_diff1": 0.7005135387488336, "naucs_at_20_max": -0.7791783380018631, "naucs_at_20_std": -0.5961718020541553, "naucs_at_20_diff1": 0.401027077497665, "naucs_at_100_max": 0.554154995331464, "naucs_at_100_std": -0.17133520074697067, "naucs_at_100_diff1": 0.35807656395892007, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_artificial_intelligence_test": {"ndcg_at_1": 0.97, "ndcg_at_3": 0.98131, "ndcg_at_5": 0.98131, "ndcg_at_10": 0.98131, "ndcg_at_20": 0.98131, "ndcg_at_100": 0.98321, "ndcg_at_1000": 0.98321, "map_at_1": 0.97, "map_at_3": 0.97833, "map_at_5": 0.97833, "map_at_10": 0.97833, "map_at_20": 0.97833, "map_at_100": 0.9786, "map_at_1000": 0.9786, "recall_at_1": 0.97, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 0.99, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.97, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.0495, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.97, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_100": 0.9786111111111111, "mrr_at_1000": 0.9786111111111111, "naucs_at_1_max": 0.41518829754123937, "naucs_at_1_std": -1.1517273576097102, "naucs_at_1_diff1": 0.9564270152505465, "naucs_at_3_max": 1.0, "naucs_at_3_std": -1.1517273576097802, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": -1.1517273576097316, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": -1.1517273576097316, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.1517273576097316, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "syntheticDocQA_government_reports_test": {"ndcg_at_1": 0.89, "ndcg_at_3": 0.94786, "ndcg_at_5": 0.94786, "ndcg_at_10": 0.95142, "ndcg_at_20": 0.95142, "ndcg_at_100": 0.95142, "ndcg_at_1000": 0.95142, "map_at_1": 0.89, "map_at_3": 0.93333, "map_at_5": 0.93333, "map_at_10": 0.935, "map_at_20": 0.935, "map_at_100": 0.935, "map_at_1000": 0.935, "recall_at_1": 0.89, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.89, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.89, "mrr_at_3": 0.935, "mrr_at_5": 0.937, "mrr_at_10": 0.937, "mrr_at_20": 0.937, "mrr_at_100": 0.937, "mrr_at_1000": 0.937, "naucs_at_1_max": 0.5626234859548148, "naucs_at_1_std": 0.3716175586289838, "naucs_at_1_diff1": 0.834850957821493, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "arxivqa_test_subsampled": {"ndcg_at_1": 0.702, "ndcg_at_3": 0.76743, "ndcg_at_5": 0.77991, "ndcg_at_10": 0.79468, "ndcg_at_20": 0.80462, "ndcg_at_100": 0.8145, "ndcg_at_1000": 0.81657, "map_at_1": 0.702, "map_at_3": 0.75133, "map_at_5": 0.75833, "map_at_10": 0.76436, "map_at_20": 0.767, "map_at_100": 0.76831, "map_at_1000": 0.76839, "recall_at_1": 0.702, "recall_at_3": 0.814, "recall_at_5": 0.844, "recall_at_10": 0.89, "recall_at_20": 0.93, "recall_at_100": 0.984, "recall_at_1000": 1.0, "precision_at_1": 0.702, "precision_at_3": 0.27133, "precision_at_5": 0.1688, "precision_at_10": 0.089, "precision_at_20": 0.0465, "precision_at_100": 0.00984, "precision_at_1000": 0.001, "mrr_at_1": 0.7, "mrr_at_3": 0.7506666666666666, "mrr_at_5": 0.7575666666666667, "mrr_at_10": 0.76395, "mrr_at_20": 0.7664683712486344, "mrr_at_100": 0.7677856040474413, "mrr_at_1000": 0.767866298103699, "naucs_at_1_max": 0.6661847878465089, "naucs_at_1_std": -0.217519374789405, "naucs_at_1_diff1": 0.9071749220117173, "naucs_at_3_max": 0.697658762809486, "naucs_at_3_std": -0.18659371894467894, "naucs_at_3_diff1": 0.839073335969141, "naucs_at_5_max": 0.7447876108417706, "naucs_at_5_std": -0.17636087316938498, "naucs_at_5_diff1": 0.8526087065738912, "naucs_at_10_max": 0.7873636285542479, "naucs_at_10_std": -0.15812215445408626, "naucs_at_10_diff1": 0.8351945709131511, "naucs_at_20_max": 0.7746565292783784, "naucs_at_20_std": -0.23473389355742383, "naucs_at_20_diff1": 0.8061224489795931, "naucs_at_100_max": 0.8027544351073763, "naucs_at_100_std": 0.2001050420168152, "naucs_at_100_diff1": 0.7639472455648932, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "docvqa_test_subsampled": {"ndcg_at_1": 0.49446, "ndcg_at_3": 0.54968, "ndcg_at_5": 0.57172, "ndcg_at_10": 0.59184, "ndcg_at_20": 0.60528, "ndcg_at_100": 0.62838, "ndcg_at_1000": 0.64353, "map_at_1": 0.49446, "map_at_3": 0.53585, "map_at_5": 0.54815, "map_at_10": 0.55648, "map_at_20": 0.56018, "map_at_100": 0.56326, "map_at_1000": 0.56389, "recall_at_1": 0.49446, "recall_at_3": 0.5898, "recall_at_5": 0.64302, "recall_at_10": 0.7051, "recall_at_20": 0.75831, "recall_at_100": 0.8847, "recall_at_1000": 1.0, "precision_at_1": 0.49446, "precision_at_3": 0.1966, "precision_at_5": 0.1286, "precision_at_10": 0.07051, "precision_at_20": 0.03792, "precision_at_100": 0.00885, "precision_at_1000": 0.001, "mrr_at_1": 0.49223946784922396, "mrr_at_3": 0.5339985218033999, "mrr_at_5": 0.5473022912047303, "mrr_at_10": 0.5553206278816035, "mrr_at_20": 0.5589821565391849, "mrr_at_100": 0.5619819074681309, "mrr_at_1000": 0.5626456935597429, "naucs_at_1_max": 0.32878731695222985, "naucs_at_1_std": 0.15591990126747984, "naucs_at_1_diff1": 0.8493613636054403, "naucs_at_3_max": 0.23098165064202703, "naucs_at_3_std": 0.1729343384390047, "naucs_at_3_diff1": 0.79653911998206, "naucs_at_5_max": 0.20443193032581358, "naucs_at_5_std": 0.22886452804275878, "naucs_at_5_diff1": 0.7709831107458899, "naucs_at_10_max": 0.11430383031823631, "naucs_at_10_std": 0.3505331687518112, "naucs_at_10_diff1": 0.7655459781075918, "naucs_at_20_max": 0.06375294772557255, "naucs_at_20_std": 0.47875225635393476, "naucs_at_20_diff1": 0.7402047132383933, "naucs_at_100_max": -0.17203513994521832, "naucs_at_100_std": 0.719961371895444, "naucs_at_100_diff1": 0.7516431118242167, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_healthcare_industry_test": {"ndcg_at_1": 0.94, "ndcg_at_3": 0.96262, "ndcg_at_5": 0.96693, "ndcg_at_10": 0.96693, "ndcg_at_20": 0.96948, "ndcg_at_100": 0.96948, "ndcg_at_1000": 0.96948, "map_at_1": 0.94, "map_at_3": 0.95667, "map_at_5": 0.95917, "map_at_10": 0.95917, "map_at_20": 0.95988, "map_at_100": 0.95988, "map_at_1000": 0.95988, "recall_at_1": 0.94, "recall_at_3": 0.98, "recall_at_5": 0.99, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.32667, "precision_at_5": 0.198, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.96, "mrr_at_3": 0.9683333333333333, "mrr_at_5": 0.9708333333333333, "mrr_at_10": 0.9708333333333333, "mrr_at_20": 0.971547619047619, "mrr_at_100": 0.971547619047619, "mrr_at_1000": 0.971547619047619, "naucs_at_1_max": 0.4701991907874248, "naucs_at_1_std": -0.2544351073762836, "naucs_at_1_diff1": 0.9319172113289763, "naucs_at_3_max": -0.004668534080301393, "naucs_at_3_std": -0.5088702147525775, "naucs_at_3_diff1": 0.7957516339869218, "naucs_at_5_max": -0.5634920634920767, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 0.7222222222222276, "naucs_at_10_max": -0.5634920634920767, "naucs_at_10_std": -1.7399626517273863, "naucs_at_10_diff1": 0.7222222222222276, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": NaN, "naucs_at_100_std": NaN, "naucs_at_100_diff1": NaN, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "infovqa_test_subsampled": {"ndcg_at_1": 0.76518, "ndcg_at_3": 0.81846, "ndcg_at_5": 0.82779, "ndcg_at_10": 0.84084, "ndcg_at_20": 0.8454, "ndcg_at_100": 0.85306, "ndcg_at_1000": 0.85616, "map_at_1": 0.76518, "map_at_3": 0.80601, "map_at_5": 0.81127, "map_at_10": 0.81663, "map_at_20": 0.81786, "map_at_100": 0.81898, "map_at_1000": 0.81909, "recall_at_1": 0.76518, "recall_at_3": 0.85425, "recall_at_5": 0.87652, "recall_at_10": 0.917, "recall_at_20": 0.93522, "recall_at_100": 0.97571, "recall_at_1000": 1.0, "precision_at_1": 0.76518, "precision_at_3": 0.28475, "precision_at_5": 0.1753, "precision_at_10": 0.0917, "precision_at_20": 0.04676, "precision_at_100": 0.00976, "precision_at_1000": 0.001, "mrr_at_1": 0.7672064777327935, "mrr_at_3": 0.8076923076923077, "mrr_at_5": 0.8130566801619433, "mrr_at_10": 0.8183712486344065, "mrr_at_20": 0.8196021674007504, "mrr_at_100": 0.8207219702130965, "mrr_at_1000": 0.8208340179158132, "naucs_at_1_max": 0.526736932815251, "naucs_at_1_std": -0.08005662601905021, "naucs_at_1_diff1": 0.8906383040101385, "naucs_at_3_max": 0.5628217589502187, "naucs_at_3_std": 0.07419124093710198, "naucs_at_3_diff1": 0.819273316147957, "naucs_at_5_max": 0.6142211700114351, "naucs_at_5_std": 0.13484006376865668, "naucs_at_5_diff1": 0.7974348993177082, "naucs_at_10_max": 0.5107732925204674, "naucs_at_10_std": 0.08282924765220158, "naucs_at_10_diff1": 0.7501472816409511, "naucs_at_20_max": 0.6069714717598735, "naucs_at_20_std": 0.24818260935350467, "naucs_at_20_diff1": 0.7972653744711068, "naucs_at_100_max": 0.7715054503371943, "naucs_at_100_std": 0.6498035527508992, "naucs_at_100_diff1": 0.7627810617053321, "naucs_at_1000_max": 1.0, "naucs_at_1000_std": 1.0, "naucs_at_1000_diff1": 1.0}, "syntheticDocQA_energy_test": {"ndcg_at_1": 0.92, "ndcg_at_3": 0.95155, "ndcg_at_5": 0.95155, "ndcg_at_10": 0.95155, "ndcg_at_20": 0.95425, "ndcg_at_100": 0.95618, "ndcg_at_1000": 0.95745, "map_at_1": 0.92, "map_at_3": 0.945, "map_at_5": 0.945, "map_at_10": 0.945, "map_at_20": 0.94583, "map_at_100": 0.94612, "map_at_1000": 0.94616, "recall_at_1": 0.92, "recall_at_3": 0.97, "recall_at_5": 0.97, "recall_at_10": 0.97, "recall_at_20": 0.98, "recall_at_100": 0.99, "recall_at_1000": 1.0, "precision_at_1": 0.92, "precision_at_3": 0.32333, "precision_at_5": 0.194, "precision_at_10": 0.097, "precision_at_20": 0.049, "precision_at_100": 0.0099, "precision_at_1000": 0.001, "mrr_at_1": 0.92, "mrr_at_3": 0.945, "mrr_at_5": 0.945, "mrr_at_10": 0.945, "mrr_at_20": 0.9458333333333333, "mrr_at_100": 0.9461190476190476, "mrr_at_1000": 0.9461614205004035, "naucs_at_1_max": 0.5274859943977597, "naucs_at_1_std": -0.6526027077497683, "naucs_at_1_diff1": 0.9673202614379083, "naucs_at_3_max": 0.807812013694371, "naucs_at_3_std": -1.1517273576097098, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.807812013694364, "naucs_at_5_std": -1.1517273576097071, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 0.807812013694364, "naucs_at_10_std": -1.1517273576097071, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 0.9346405228758136, "naucs_at_20_std": -0.8576097105508901, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": -0.5634920634920583, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}, "tabfquad_test_subsampled": {"ndcg_at_1": 0.85, "ndcg_at_3": 0.88465, "ndcg_at_5": 0.89664, "ndcg_at_10": 0.90491, "ndcg_at_20": 0.91464, "ndcg_at_100": 0.91464, "ndcg_at_1000": 0.91464, "map_at_1": 0.85, "map_at_3": 0.87679, "map_at_5": 0.88357, "map_at_10": 0.8871, "map_at_20": 0.88966, "map_at_100": 0.88966, "map_at_1000": 0.88966, "recall_at_1": 0.85, "recall_at_3": 0.90714, "recall_at_5": 0.93571, "recall_at_10": 0.96071, "recall_at_20": 1.0, "recall_at_100": 1.0, "recall_at_1000": 1.0, "precision_at_1": 0.85, "precision_at_3": 0.30238, "precision_at_5": 0.18714, "precision_at_10": 0.09607, "precision_at_20": 0.05, "precision_at_100": 0.01, "precision_at_1000": 0.001, "mrr_at_1": 0.85, "mrr_at_3": 0.8779761904761905, "mrr_at_5": 0.8838690476190476, "mrr_at_10": 0.8874900793650794, "mrr_at_20": 0.8898663632119514, "mrr_at_100": 0.8900364312391623, "mrr_at_1000": 0.8900364312391623, "naucs_at_1_max": 0.6249012133327131, "naucs_at_1_std": 0.27568453349449135, "naucs_at_1_diff1": 0.851971084561387, "naucs_at_3_max": 0.7154528478057898, "naucs_at_3_std": 0.3263664440135032, "naucs_at_3_diff1": 0.8540364863894296, "naucs_at_5_max": 0.7756769374416456, "naucs_at_5_std": 0.5388525780682669, "naucs_at_5_diff1": 0.8674914410208527, "naucs_at_10_max": 0.7903403785756714, "naucs_at_10_std": 0.6866564807741266, "naucs_at_10_diff1": 0.832187420422716, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0, "naucs_at_1000_max": NaN, "naucs_at_1000_std": NaN, "naucs_at_1000_diff1": NaN}} |