| { | |
| "metadata": { | |
| "timestamp": "2025-01-31T14:00:16.349261", | |
| "vidore_benchmark_version": "4.0.3.dev20+g2d72668" | |
| }, | |
| "metrics": { | |
| "vidore/restaurant_esg_reports_beir": {"ndcg_at_1": 0.64103, "ndcg_at_3": 0.67538, "ndcg_at_5": 0.68397, "ndcg_at_10": 0.71199, "ndcg_at_20": 0.73099, "ndcg_at_50": 0.75581, "ndcg_at_100": 0.76163, "map_at_1": 0.43942, "map_at_3": 0.58034, "map_at_5": 0.6105, "map_at_10": 0.63715, "map_at_20": 0.64674, "map_at_50": 0.65463, "map_at_100": 0.65653, "recall_at_1": 0.43942, "recall_at_3": 0.6766, "recall_at_5": 0.73558, "recall_at_10": 0.81426, "recall_at_20": 0.87567, "recall_at_50": 0.95485, "recall_at_100": 0.981, "precision_at_1": 0.65385, "precision_at_3": 0.39744, "precision_at_5": 0.27692, "precision_at_10": 0.16538, "precision_at_20": 0.09423, "precision_at_50": 0.04346, "precision_at_100": 0.02308, "mrr_at_1": 0.6730769230769231, "mrr_at_3": 0.7467948717948718, "mrr_at_5": 0.7592948717948718, "mrr_at_10": 0.7647893772893772, "mrr_at_20": 0.7647893772893772, "mrr_at_50": 0.76724122651542, "mrr_at_100": 0.76724122651542, "naucs_at_1_max": 0.10629249714250551, "naucs_at_1_std": 0.03656703481974433, "naucs_at_1_diff1": 0.6471348946136236, "naucs_at_3_max": -0.04531273341377843, "naucs_at_3_std": 0.00041838369923711004, "naucs_at_3_diff1": -0.10904002079315368, "naucs_at_5_max": -0.11437346877355879, "naucs_at_5_std": -0.03247506128676773, "naucs_at_5_diff1": -0.23163739837531055, "naucs_at_10_max": -0.17966299098993738, "naucs_at_10_std": -0.08845607083585832, "naucs_at_10_diff1": -0.3668906677620788, "naucs_at_20_max": -0.1623997659302278, "naucs_at_20_std": -0.05328921373458377, "naucs_at_20_diff1": -0.4139154609733948, "naucs_at_50_max": -0.15835976853630981, "naucs_at_50_std": -0.042923187520787366, "naucs_at_50_diff1": -0.45626822378009313, "naucs_at_100_max": -0.14433851613678755, "naucs_at_100_std": -0.03148299510892002, "naucs_at_100_diff1": -0.44576138476398025}, | |
| "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.60345, | |
| "ndcg_at_3": 0.59658, | |
| "ndcg_at_5": 0.56521, | |
| "ndcg_at_10": 0.55055, | |
| "ndcg_at_20": 0.57899, | |
| "ndcg_at_50": 0.65246, | |
| "ndcg_at_100": 0.68858, | |
| "map_at_1": 0.08192, | |
| "map_at_3": 0.18869, | |
| "map_at_5": 0.23861, | |
| "map_at_10": 0.30685, | |
| "map_at_20": 0.36485, | |
| "map_at_50": 0.42784, | |
| "map_at_100": 0.45362, | |
| "recall_at_1": 0.08192, | |
| "recall_at_3": 0.23525, | |
| "recall_at_5": 0.31079, | |
| "recall_at_10": 0.43833, | |
| "recall_at_20": 0.58306, | |
| "recall_at_50": 0.78859, | |
| "recall_at_100": 0.90965, | |
| "precision_at_1": 0.60345, | |
| "precision_at_3": 0.5546, | |
| "precision_at_5": 0.49138, | |
| "precision_at_10": 0.39914, | |
| "precision_at_20": 0.30539, | |
| "precision_at_50": 0.1981, | |
| "precision_at_100": 0.12647, | |
| "mrr_at_1": 0.5948275862068966, | |
| "mrr_at_3": 0.7140804597701149, | |
| "mrr_at_5": 0.7311063218390805, | |
| "mrr_at_10": 0.7372383004926109, | |
| "mrr_at_20": 0.7380691285134744, | |
| "mrr_at_50": 0.738889335917265, | |
| "mrr_at_100": 0.738889335917265, | |
| "naucs_at_1_max": 0.009301400086149124, | |
| "naucs_at_1_std": -0.018176205759631064, | |
| "naucs_at_1_diff1": 0.06589141538556753, | |
| "naucs_at_3_max": 0.03608007053002765, | |
| "naucs_at_3_std": 0.045332981794521165, | |
| "naucs_at_3_diff1": 0.0027274204838523214, | |
| "naucs_at_5_max": 0.06351874275538175, | |
| "naucs_at_5_std": 0.09161670694703962, | |
| "naucs_at_5_diff1": 0.030104357247264922, | |
| "naucs_at_10_max": 0.09045732284339944, | |
| "naucs_at_10_std": 0.11937912992266166, | |
| "naucs_at_10_diff1": -0.007723827231040398, | |
| "naucs_at_20_max": 0.04097490582062997, | |
| "naucs_at_20_std": 0.11121740670187244, | |
| "naucs_at_20_diff1": -0.04080919911505186, | |
| "naucs_at_50_max": -0.04431654376754045, | |
| "naucs_at_50_std": 0.1075378398002067, | |
| "naucs_at_50_diff1": -0.0408768243788732, | |
| "naucs_at_100_max": -0.08194156783211, | |
| "naucs_at_100_std": 0.08152221906964778, | |
| "naucs_at_100_diff1": -0.002462405639259035 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.44444, | |
| "ndcg_at_3": 0.52927, | |
| "ndcg_at_5": 0.53193, | |
| "ndcg_at_10": 0.55273, | |
| "ndcg_at_20": 0.57791, | |
| "ndcg_at_50": 0.62959, | |
| "ndcg_at_100": 0.64947, | |
| "map_at_1": 0.19923, | |
| "map_at_3": 0.31897, | |
| "map_at_5": 0.3681, | |
| "map_at_10": 0.4284, | |
| "map_at_20": 0.45565, | |
| "map_at_50": 0.47849, | |
| "map_at_100": 0.48445, | |
| "recall_at_1": 0.19923, | |
| "recall_at_3": 0.43016, | |
| "recall_at_5": 0.51283, | |
| "recall_at_10": 0.63101, | |
| "recall_at_20": 0.7193, | |
| "recall_at_50": 0.86749, | |
| "recall_at_100": 0.93786, | |
| "precision_at_1": 0.44444, | |
| "precision_at_3": 0.39815, | |
| "precision_at_5": 0.33611, | |
| "precision_at_10": 0.25, | |
| "precision_at_20": 0.15347, | |
| "precision_at_50": 0.08167, | |
| "precision_at_100": 0.04556, | |
| "mrr_at_1": 0.4444444444444444, | |
| "mrr_at_3": 0.6041666666666666, | |
| "mrr_at_5": 0.6111111111111112, | |
| "mrr_at_10": 0.6134259259259259, | |
| "mrr_at_20": 0.6134259259259259, | |
| "mrr_at_50": 0.6156871267846877, | |
| "mrr_at_100": 0.6160388005738615, | |
| "naucs_at_1_max": -0.007676494689249098, | |
| "naucs_at_1_std": 0.15632886231764911, | |
| "naucs_at_1_diff1": 0.3454147669831923, | |
| "naucs_at_3_max": -0.21520431732796824, | |
| "naucs_at_3_std": -0.039307896873910286, | |
| "naucs_at_3_diff1": 0.07075823324379214, | |
| "naucs_at_5_max": -0.23427920743418434, | |
| "naucs_at_5_std": -0.04955213973003769, | |
| "naucs_at_5_diff1": 0.03587555672187707, | |
| "naucs_at_10_max": -0.32076687527735803, | |
| "naucs_at_10_std": -0.08584531062590048, | |
| "naucs_at_10_diff1": -0.13920437910700148, | |
| "naucs_at_20_max": -0.3500838034005256, | |
| "naucs_at_20_std": -0.11679539412969686, | |
| "naucs_at_20_diff1": -0.2051894549829469, | |
| "naucs_at_50_max": -0.4102497000902005, | |
| "naucs_at_50_std": -0.1409315271751862, | |
| "naucs_at_50_diff1": -0.16443764913412845, | |
| "naucs_at_100_max": -0.43730180953377223, | |
| "naucs_at_100_std": -0.17380745870110292, | |
| "naucs_at_100_diff1": -0.19442949153976513 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { | |
| "ndcg_at_1": 0.61875, | |
| "ndcg_at_3": 0.61406, | |
| "ndcg_at_5": 0.63644, | |
| "ndcg_at_10": 0.67591, | |
| "ndcg_at_20": 0.70178, | |
| "ndcg_at_50": 0.72212, | |
| "ndcg_at_100": 0.7297, | |
| "map_at_1": 0.38142, | |
| "map_at_3": 0.50057, | |
| "map_at_5": 0.54282, | |
| "map_at_10": 0.58278, | |
| "map_at_20": 0.59965, | |
| "map_at_50": 0.60789, | |
| "map_at_100": 0.60942, | |
| "recall_at_1": 0.38142, | |
| "recall_at_3": 0.55686, | |
| "recall_at_5": 0.65567, | |
| "recall_at_10": 0.77349, | |
| "recall_at_20": 0.84542, | |
| "recall_at_50": 0.902, | |
| "recall_at_100": 0.93314, | |
| "precision_at_1": 0.61875, | |
| "precision_at_3": 0.38333, | |
| "precision_at_5": 0.28875, | |
| "precision_at_10": 0.18875, | |
| "precision_at_20": 0.11063, | |
| "precision_at_50": 0.052, | |
| "precision_at_100": 0.02756, | |
| "mrr_at_1": 0.6, | |
| "mrr_at_3": 0.6947916666666667, | |
| "mrr_at_5": 0.7104166666666667, | |
| "mrr_at_10": 0.7197544642857143, | |
| "mrr_at_20": 0.7207690746753247, | |
| "mrr_at_50": 0.7216319385110503, | |
| "mrr_at_100": 0.7218020773999393, | |
| "naucs_at_1_max": 0.23570942089460648, | |
| "naucs_at_1_std": -0.0331808109585888, | |
| "naucs_at_1_diff1": 0.38829894385449926, | |
| "naucs_at_3_max": -0.02996939892755872, | |
| "naucs_at_3_std": -0.06455703309969216, | |
| "naucs_at_3_diff1": -0.10409043693090879, | |
| "naucs_at_5_max": -0.10452188626653831, | |
| "naucs_at_5_std": -0.052524231286355125, | |
| "naucs_at_5_diff1": -0.26201383842532566, | |
| "naucs_at_10_max": -0.18753439904110775, | |
| "naucs_at_10_std": -0.07594574430351896, | |
| "naucs_at_10_diff1": -0.3404263646481817, | |
| "naucs_at_20_max": -0.20945097879456231, | |
| "naucs_at_20_std": -0.04291331785264603, | |
| "naucs_at_20_diff1": -0.35991045256760157, | |
| "naucs_at_50_max": -0.25078519595626036, | |
| "naucs_at_50_std": -0.021452523948669356, | |
| "naucs_at_50_diff1": -0.3863780246795678, | |
| "naucs_at_100_max": -0.2723431611743009, | |
| "naucs_at_100_std": -0.031237471793638266, | |
| "naucs_at_100_diff1": -0.38639136112161093 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0": { | |
| "ndcg_at_1": 0.57895, | |
| "ndcg_at_3": 0.57099, | |
| "ndcg_at_5": 0.57396, | |
| "ndcg_at_10": 0.6394, | |
| "ndcg_at_20": 0.67256, | |
| "ndcg_at_50": 0.70473, | |
| "ndcg_at_100": 0.71518, | |
| "map_at_1": 0.26466, | |
| "map_at_3": 0.41771, | |
| "map_at_5": 0.45328, | |
| "map_at_10": 0.515, | |
| "map_at_20": 0.53681, | |
| "map_at_50": 0.55475, | |
| "map_at_100": 0.55975, | |
| "recall_at_1": 0.26466, | |
| "recall_at_3": 0.50881, | |
| "recall_at_5": 0.57932, | |
| "recall_at_10": 0.77663, | |
| "recall_at_20": 0.8768, | |
| "recall_at_50": 0.96898, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.57895, | |
| "precision_at_3": 0.39766, | |
| "precision_at_5": 0.29123, | |
| "precision_at_10": 0.21053, | |
| "precision_at_20": 0.13246, | |
| "precision_at_50": 0.07123, | |
| "precision_at_100": 0.03895, | |
| "mrr_at_1": 0.5614035087719298, | |
| "mrr_at_3": 0.6637426900584795, | |
| "mrr_at_5": 0.6751461988304094, | |
| "mrr_at_10": 0.6886173767752715, | |
| "mrr_at_20": 0.6913087643350802, | |
| "mrr_at_50": 0.6925429103060682, | |
| "mrr_at_100": 0.6925429103060682, | |
| "naucs_at_1_max": -0.32399679124866293, | |
| "naucs_at_1_std": -0.3157496852536844, | |
| "naucs_at_1_diff1": 0.059217413521438306, | |
| "naucs_at_3_max": -0.16317940152105928, | |
| "naucs_at_3_std": -0.036766880845353206, | |
| "naucs_at_3_diff1": 0.13451653330920363, | |
| "naucs_at_5_max": -0.18872080560435078, | |
| "naucs_at_5_std": -0.08965899629213353, | |
| "naucs_at_5_diff1": 0.09733357617902798, | |
| "naucs_at_10_max": -0.21096127425243222, | |
| "naucs_at_10_std": -0.0492682492736064, | |
| "naucs_at_10_diff1": 0.06579043413332678, | |
| "naucs_at_20_max": -0.3151765999140926, | |
| "naucs_at_20_std": -0.11272929289257284, | |
| "naucs_at_20_diff1": -0.030784322032566636, | |
| "naucs_at_50_max": -0.29489025135650376, | |
| "naucs_at_50_std": -0.008523958191869246, | |
| "naucs_at_50_diff1": -0.11796822928434904, | |
| "naucs_at_100_max": -0.2661970083875134, | |
| "naucs_at_100_std": 0.05555820249689567, | |
| "naucs_at_100_diff1": -0.1271611921091859 | |
| }, | |
| "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { | |
| "ndcg_at_1": 0.55702, | |
| "ndcg_at_3": 0.5542, | |
| "ndcg_at_5": 0.5738, | |
| "ndcg_at_10": 0.62953, | |
| "ndcg_at_20": 0.67065, | |
| "ndcg_at_50": 0.6964, | |
| "ndcg_at_100": 0.70872, | |
| "map_at_1": 0.25589, | |
| "map_at_3": 0.40701, | |
| "map_at_5": 0.45304, | |
| "map_at_10": 0.5087, | |
| "map_at_20": 0.53389, | |
| "map_at_50": 0.5501, | |
| "map_at_100": 0.55542, | |
| "recall_at_1": 0.25589, | |
| "recall_at_3": 0.49373, | |
| "recall_at_5": 0.59174, | |
| "recall_at_10": 0.76938, | |
| "recall_at_20": 0.89178, | |
| "recall_at_50": 0.9594, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.55702, | |
| "precision_at_3": 0.3845, | |
| "precision_at_5": 0.29649, | |
| "precision_at_10": 0.20746, | |
| "precision_at_20": 0.13487, | |
| "precision_at_50": 0.0707, | |
| "precision_at_100": 0.03895, | |
| "mrr_at_1": 0.5570175438596491, | |
| "mrr_at_3": 0.6571637426900585, | |
| "mrr_at_5": 0.6698830409356724, | |
| "mrr_at_10": 0.683766708437761, | |
| "mrr_at_20": 0.687599964534949, | |
| "mrr_at_50": 0.6879085010276961, | |
| "mrr_at_100": 0.6880572205275953, | |
| "naucs_at_1_max": -0.17166276074798967, | |
| "naucs_at_1_std": -0.20325235499471186, | |
| "naucs_at_1_diff1": 0.33001833715444423, | |
| "naucs_at_3_max": -0.04474887443402487, | |
| "naucs_at_3_std": -0.008560331928141176, | |
| "naucs_at_3_diff1": 0.1746255699902219, | |
| "naucs_at_5_max": -0.08458531884834858, | |
| "naucs_at_5_std": -0.06118868392476525, | |
| "naucs_at_5_diff1": 0.13235525060593817, | |
| "naucs_at_10_max": -0.12880215182381638, | |
| "naucs_at_10_std": -0.08311726109553826, | |
| "naucs_at_10_diff1": 0.11900519462159719, | |
| "naucs_at_20_max": -0.18439650375622735, | |
| "naucs_at_20_std": -0.051821500134533655, | |
| "naucs_at_20_diff1": -0.015123572562768748, | |
| "naucs_at_50_max": -0.20702547660326365, | |
| "naucs_at_50_std": 0.02211997502899773, | |
| "naucs_at_50_diff1": -0.1539679823951051, | |
| "naucs_at_100_max": -0.1731060459048004, | |
| "naucs_at_100_std": 0.1019715434458989, | |
| "naucs_at_100_diff1": -0.1841288831466728 | |
| }, | |
| "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { | |
| "ndcg_at_1": 0.57812, | |
| "ndcg_at_3": 0.58744, | |
| "ndcg_at_5": 0.61127, | |
| "ndcg_at_10": 0.64614, | |
| "ndcg_at_20": 0.67298, | |
| "ndcg_at_50": 0.6947, | |
| "ndcg_at_100": 0.70357, | |
| "map_at_1": 0.35834, | |
| "map_at_3": 0.47597, | |
| "map_at_5": 0.51819, | |
| "map_at_10": 0.55455, | |
| "map_at_20": 0.57016, | |
| "map_at_50": 0.5782, | |
| "map_at_100": 0.58034, | |
| "recall_at_1": 0.35834, | |
| "recall_at_3": 0.54214, | |
| "recall_at_5": 0.63727, | |
| "recall_at_10": 0.74383, | |
| "recall_at_20": 0.82133, | |
| "recall_at_50": 0.88659, | |
| "recall_at_100": 0.91866, | |
| "precision_at_1": 0.57812, | |
| "precision_at_3": 0.3651, | |
| "precision_at_5": 0.28, | |
| "precision_at_10": 0.18, | |
| "precision_at_20": 0.10617, | |
| "precision_at_50": 0.05, | |
| "precision_at_100": 0.02695, | |
| "mrr_at_1": 0.5671875, | |
| "mrr_at_3": 0.6708333333333333, | |
| "mrr_at_5": 0.6866145833333334, | |
| "mrr_at_10": 0.6939360119047618, | |
| "mrr_at_20": 0.6963204666729361, | |
| "mrr_at_50": 0.6971792252566347, | |
| "mrr_at_100": 0.6973040137004454, | |
| "naucs_at_1_max": 0.22350385772789982, | |
| "naucs_at_1_std": -0.014236260379664127, | |
| "naucs_at_1_diff1": 0.415396972645202, | |
| "naucs_at_3_max": 0.05168321374193644, | |
| "naucs_at_3_std": -0.05262932689279111, | |
| "naucs_at_3_diff1": -0.046426105116023936, | |
| "naucs_at_5_max": -0.019935793098795593, | |
| "naucs_at_5_std": -0.07180552504536146, | |
| "naucs_at_5_diff1": -0.15736139387093928, | |
| "naucs_at_10_max": -0.1020058048777736, | |
| "naucs_at_10_std": -0.09370704449884888, | |
| "naucs_at_10_diff1": -0.23614803376814614, | |
| "naucs_at_20_max": -0.13810591303119216, | |
| "naucs_at_20_std": -0.0692107572861515, | |
| "naucs_at_20_diff1": -0.28493482904408224, | |
| "naucs_at_50_max": -0.17286447176352046, | |
| "naucs_at_50_std": -0.04984221651971845, | |
| "naucs_at_50_diff1": -0.317010129235068, | |
| "naucs_at_100_max": -0.18980306870058375, | |
| "naucs_at_100_std": -0.048954124365436114, | |
| "naucs_at_100_diff1": -0.3281886367637723 | |
| }, | |
| "vidore/synthetic_axa_filtered_v1.0": { | |
| "ndcg_at_1": 0.44444, | |
| "ndcg_at_3": 0.55779, | |
| "ndcg_at_5": 0.60297, | |
| "ndcg_at_10": 0.63112, | |
| "ndcg_at_20": 0.65055, | |
| "ndcg_at_50": 0.69129, | |
| "ndcg_at_100": 0.70318, | |
| "map_at_1": 0.22361, | |
| "map_at_3": 0.34019, | |
| "map_at_5": 0.42558, | |
| "map_at_10": 0.50246, | |
| "map_at_20": 0.53023, | |
| "map_at_50": 0.54858, | |
| "map_at_100": 0.55304, | |
| "recall_at_1": 0.22361, | |
| "recall_at_3": 0.43907, | |
| "recall_at_5": 0.59758, | |
| "recall_at_10": 0.75267, | |
| "recall_at_20": 0.82791, | |
| "recall_at_50": 0.94587, | |
| "recall_at_100": 0.9784, | |
| "precision_at_1": 0.44444, | |
| "precision_at_3": 0.42593, | |
| "precision_at_5": 0.4, | |
| "precision_at_10": 0.29444, | |
| "precision_at_20": 0.175, | |
| "precision_at_50": 0.08556, | |
| "precision_at_100": 0.04611, | |
| "mrr_at_1": 0.4444444444444444, | |
| "mrr_at_3": 0.6111111111111112, | |
| "mrr_at_5": 0.625, | |
| "mrr_at_10": 0.6342592592592592, | |
| "mrr_at_20": 0.6342592592592592, | |
| "mrr_at_50": 0.6367845117845118, | |
| "mrr_at_100": 0.6367845117845118, | |
| "naucs_at_1_max": 0.27180734937990775, | |
| "naucs_at_1_std": 0.1460001284215161, | |
| "naucs_at_1_diff1": 0.52680845013576, | |
| "naucs_at_3_max": -0.4472601618035251, | |
| "naucs_at_3_std": -0.35334842300017605, | |
| "naucs_at_3_diff1": -0.022038941836526545, | |
| "naucs_at_5_max": -0.5746685319786979, | |
| "naucs_at_5_std": -0.45581287349947885, | |
| "naucs_at_5_diff1": -0.02909625842935311, | |
| "naucs_at_10_max": -0.6670105211539744, | |
| "naucs_at_10_std": -0.4579454710599414, | |
| "naucs_at_10_diff1": -0.291350928264914, | |
| "naucs_at_20_max": -0.7088454526103415, | |
| "naucs_at_20_std": -0.45884776633256535, | |
| "naucs_at_20_diff1": -0.39562206735708155, | |
| "naucs_at_50_max": -0.7078100142972502, | |
| "naucs_at_50_std": -0.4315828703198929, | |
| "naucs_at_50_diff1": -0.37094602463897214, | |
| "naucs_at_100_max": -0.7059796119743438, | |
| "naucs_at_100_std": -0.400631653419931, | |
| "naucs_at_100_diff1": -0.33114628268848284 | |
| }, | |
| "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { | |
| "ndcg_at_1": 0.67241, | |
| "ndcg_at_3": 0.64169, | |
| "ndcg_at_5": 0.5978, | |
| "ndcg_at_10": 0.58577, | |
| "ndcg_at_20": 0.61496, | |
| "ndcg_at_50": 0.6772, | |
| "ndcg_at_100": 0.71432, | |
| "map_at_1": 0.08152, | |
| "map_at_3": 0.20404, | |
| "map_at_5": 0.2537, | |
| "map_at_10": 0.32889, | |
| "map_at_20": 0.39447, | |
| "map_at_50": 0.4536, | |
| "map_at_100": 0.48141, | |
| "recall_at_1": 0.08152, | |
| "recall_at_3": 0.2494, | |
| "recall_at_5": 0.32267, | |
| "recall_at_10": 0.45814, | |
| "recall_at_20": 0.61677, | |
| "recall_at_50": 0.80189, | |
| "recall_at_100": 0.92313, | |
| "precision_at_1": 0.67241, | |
| "precision_at_3": 0.59195, | |
| "precision_at_5": 0.51379, | |
| "precision_at_10": 0.42586, | |
| "precision_at_20": 0.325, | |
| "precision_at_50": 0.20138, | |
| "precision_at_100": 0.12983, | |
| "mrr_at_1": 0.6724137931034483, | |
| "mrr_at_3": 0.7729885057471265, | |
| "mrr_at_5": 0.7807471264367816, | |
| "mrr_at_10": 0.7857758620689655, | |
| "mrr_at_20": 0.7868534482758621, | |
| "mrr_at_50": 0.7876030734632684, | |
| "mrr_at_100": 0.7876030734632684, | |
| "naucs_at_1_max": 0.34454681161877004, | |
| "naucs_at_1_std": 0.5235431895381841, | |
| "naucs_at_1_diff1": -0.08387631081097016, | |
| "naucs_at_3_max": 0.08884531420395679, | |
| "naucs_at_3_std": 0.2382734233882671, | |
| "naucs_at_3_diff1": -0.1871866677817215, | |
| "naucs_at_5_max": 0.01311909348957855, | |
| "naucs_at_5_std": 0.2167381673457986, | |
| "naucs_at_5_diff1": -0.14497179289018872, | |
| "naucs_at_10_max": 0.14293369671235354, | |
| "naucs_at_10_std": 0.22265260242487017, | |
| "naucs_at_10_diff1": -0.09048933555139305, | |
| "naucs_at_20_max": 0.08115085333286959, | |
| "naucs_at_20_std": 0.20688202040577058, | |
| "naucs_at_20_diff1": -0.15069532949644948, | |
| "naucs_at_50_max": -0.010126566692404412, | |
| "naucs_at_50_std": 0.24164336672801914, | |
| "naucs_at_50_diff1": -0.10468292902065172, | |
| "naucs_at_100_max": -0.04114659355878923, | |
| "naucs_at_100_std": 0.2129831852026439, | |
| "naucs_at_100_diff1": -0.03795755522744344 | |
| }, | |
| "data_dir/eval_vidore/arxivqa_test_subsampled": { | |
| "ndcg_at_1": 0.856, | |
| "ndcg_at_3": 0.88745, | |
| "ndcg_at_5": 0.89227, | |
| "ndcg_at_10": 0.90332, | |
| "ndcg_at_20": 0.90941, | |
| "ndcg_at_50": 0.91153, | |
| "ndcg_at_100": 0.91252, | |
| "map_at_1": 0.856, | |
| "map_at_3": 0.87967, | |
| "map_at_5": 0.88227, | |
| "map_at_10": 0.88687, | |
| "map_at_20": 0.88854, | |
| "map_at_50": 0.88895, | |
| "map_at_100": 0.88904, | |
| "recall_at_1": 0.856, | |
| "recall_at_3": 0.91, | |
| "recall_at_5": 0.922, | |
| "recall_at_10": 0.956, | |
| "recall_at_20": 0.98, | |
| "recall_at_50": 0.99, | |
| "recall_at_100": 0.996, | |
| "precision_at_1": 0.856, | |
| "precision_at_3": 0.30333, | |
| "precision_at_5": 0.1844, | |
| "precision_at_10": 0.0956, | |
| "precision_at_20": 0.049, | |
| "precision_at_50": 0.0198, | |
| "precision_at_100": 0.00996, | |
| "mrr_at_1": 0.854, | |
| "mrr_at_3": 0.8773333333333334, | |
| "mrr_at_5": 0.8809333333333332, | |
| "mrr_at_10": 0.8855674603174604, | |
| "mrr_at_20": 0.8873665921453847, | |
| "mrr_at_50": 0.8876643128304966, | |
| "mrr_at_100": 0.887751797810069, | |
| "naucs_at_1_max": 0.8146679814134421, | |
| "naucs_at_1_std": 0.07396801383185674, | |
| "naucs_at_1_diff1": 0.9423560082126644, | |
| "naucs_at_3_max": 0.8066396929142029, | |
| "naucs_at_3_std": 0.07559912854030806, | |
| "naucs_at_3_diff1": 0.9181969083929873, | |
| "naucs_at_5_max": 0.8034068328185975, | |
| "naucs_at_5_std": 0.02020637314754663, | |
| "naucs_at_5_diff1": 0.9160860925566815, | |
| "naucs_at_10_max": 0.825566590272474, | |
| "naucs_at_10_std": -0.15187590187589842, | |
| "naucs_at_10_diff1": 0.9390968508615584, | |
| "naucs_at_20_max": 0.8921568627450932, | |
| "naucs_at_20_std": 0.0641923436041116, | |
| "naucs_at_20_diff1": 0.9738562091503187, | |
| "naucs_at_50_max": 0.947712418300658, | |
| "naucs_at_50_std": 0.37030812324930756, | |
| "naucs_at_50_diff1": 0.9738562091503188, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "data_dir/eval_vidore/docvqa_test_subsampled": { | |
| "ndcg_at_1": 0.55876, | |
| "ndcg_at_3": 0.61485, | |
| "ndcg_at_5": 0.63222, | |
| "ndcg_at_10": 0.65493, | |
| "ndcg_at_20": 0.66746, | |
| "ndcg_at_50": 0.67657, | |
| "ndcg_at_100": 0.68329, | |
| "map_at_1": 0.55876, | |
| "map_at_3": 0.60126, | |
| "map_at_5": 0.6109, | |
| "map_at_10": 0.62013, | |
| "map_at_20": 0.62368, | |
| "map_at_50": 0.62509, | |
| "map_at_100": 0.62564, | |
| "recall_at_1": 0.55876, | |
| "recall_at_3": 0.6541, | |
| "recall_at_5": 0.69623, | |
| "recall_at_10": 0.76718, | |
| "recall_at_20": 0.81596, | |
| "recall_at_50": 0.86253, | |
| "recall_at_100": 0.90466, | |
| "precision_at_1": 0.55876, | |
| "precision_at_3": 0.21803, | |
| "precision_at_5": 0.13925, | |
| "precision_at_10": 0.07672, | |
| "precision_at_20": 0.0408, | |
| "precision_at_50": 0.01725, | |
| "precision_at_100": 0.00905, | |
| "mrr_at_1": 0.5543237250554324, | |
| "mrr_at_3": 0.599039172209904, | |
| "mrr_at_5": 0.609349593495935, | |
| "mrr_at_10": 0.6183401963889769, | |
| "mrr_at_20": 0.6209344044461912, | |
| "mrr_at_50": 0.6227106859869997, | |
| "mrr_at_100": 0.6232889837626537, | |
| "naucs_at_1_max": 0.29569387287927856, | |
| "naucs_at_1_std": 0.6842470452244832, | |
| "naucs_at_1_diff1": 0.9092758189268736, | |
| "naucs_at_3_max": 0.21005633503267365, | |
| "naucs_at_3_std": 0.805068083051039, | |
| "naucs_at_3_diff1": 0.8754563618133548, | |
| "naucs_at_5_max": 0.19259019824979384, | |
| "naucs_at_5_std": 0.8409490067520664, | |
| "naucs_at_5_diff1": 0.8726101181684977, | |
| "naucs_at_10_max": 0.048646855770537796, | |
| "naucs_at_10_std": 0.8833213018935924, | |
| "naucs_at_10_diff1": 0.8643899050698751, | |
| "naucs_at_20_max": -0.06225799483664109, | |
| "naucs_at_20_std": 0.9127300541361831, | |
| "naucs_at_20_diff1": 0.8597071895030789, | |
| "naucs_at_50_max": -0.24442668101014947, | |
| "naucs_at_50_std": 0.930294669379938, | |
| "naucs_at_50_diff1": 0.8714057920484658, | |
| "naucs_at_100_max": -0.3617257884810223, | |
| "naucs_at_100_std": 0.9386292283530702, | |
| "naucs_at_100_diff1": 0.8597379463433718 | |
| }, | |
| "data_dir/eval_vidore/syntheticDocQA_energy_test": { | |
| "ndcg_at_1": 0.93, | |
| "ndcg_at_3": 0.95893, | |
| "ndcg_at_5": 0.95893, | |
| "ndcg_at_10": 0.95893, | |
| "ndcg_at_20": 0.96163, | |
| "ndcg_at_50": 0.96365, | |
| "ndcg_at_100": 0.96365, | |
| "map_at_1": 0.93, | |
| "map_at_3": 0.95167, | |
| "map_at_5": 0.95167, | |
| "map_at_10": 0.95167, | |
| "map_at_20": 0.9525, | |
| "map_at_50": 0.95283, | |
| "map_at_100": 0.95283, | |
| "recall_at_1": 0.93, | |
| "recall_at_3": 0.98, | |
| "recall_at_5": 0.98, | |
| "recall_at_10": 0.98, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.93, | |
| "precision_at_3": 0.32667, | |
| "precision_at_5": 0.196, | |
| "precision_at_10": 0.098, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.93, | |
| "mrr_at_3": 0.9516666666666667, | |
| "mrr_at_5": 0.9516666666666667, | |
| "mrr_at_10": 0.9516666666666667, | |
| "mrr_at_20": 0.9525757575757575, | |
| "mrr_at_50": 0.9529205851619644, | |
| "mrr_at_100": 0.9529205851619644, | |
| "naucs_at_1_max": 0.39482459650526885, | |
| "naucs_at_1_std": -0.31419234360410914, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 0.6790382819794457, | |
| "naucs_at_3_std": -0.9556489262371661, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 0.6790382819794609, | |
| "naucs_at_5_std": -0.9556489262371534, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 0.6790382819794609, | |
| "naucs_at_10_std": -0.9556489262371534, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": -0.1713352007469681, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "data_dir/eval_vidore/tatdqa_test": { | |
| "ndcg_at_1": 0.69927, | |
| "ndcg_at_3": 0.79372, | |
| "ndcg_at_5": 0.81105, | |
| "ndcg_at_10": 0.82459, | |
| "ndcg_at_20": 0.82878, | |
| "ndcg_at_50": 0.83303, | |
| "ndcg_at_100": 0.83442, | |
| "map_at_1": 0.69927, | |
| "map_at_3": 0.77106, | |
| "map_at_5": 0.78072, | |
| "map_at_10": 0.78653, | |
| "map_at_20": 0.7877, | |
| "map_at_50": 0.78839, | |
| "map_at_100": 0.78852, | |
| "recall_at_1": 0.69927, | |
| "recall_at_3": 0.85905, | |
| "recall_at_5": 0.90097, | |
| "recall_at_10": 0.94168, | |
| "recall_at_20": 0.95808, | |
| "recall_at_50": 0.97934, | |
| "recall_at_100": 0.98785, | |
| "precision_at_1": 0.69927, | |
| "precision_at_3": 0.28635, | |
| "precision_at_5": 0.18019, | |
| "precision_at_10": 0.09417, | |
| "precision_at_20": 0.0479, | |
| "precision_at_50": 0.01959, | |
| "precision_at_100": 0.00988, | |
| "mrr_at_1": 0.7004860267314702, | |
| "mrr_at_3": 0.7722762251923856, | |
| "mrr_at_5": 0.7813284730660186, | |
| "mrr_at_10": 0.7874199598835079, | |
| "mrr_at_20": 0.7885261582425689, | |
| "mrr_at_50": 0.7892193634018624, | |
| "mrr_at_100": 0.7893291443742743, | |
| "naucs_at_1_max": 0.2682416909968819, | |
| "naucs_at_1_std": -0.1346098281401034, | |
| "naucs_at_1_diff1": 0.8296403667835969, | |
| "naucs_at_3_max": 0.36050493227494845, | |
| "naucs_at_3_std": -0.01834048113954263, | |
| "naucs_at_3_diff1": 0.7445505884597561, | |
| "naucs_at_5_max": 0.36523716935916267, | |
| "naucs_at_5_std": 0.06489405058952001, | |
| "naucs_at_5_diff1": 0.7142046612217674, | |
| "naucs_at_10_max": 0.4293315802144752, | |
| "naucs_at_10_std": 0.2564808038730297, | |
| "naucs_at_10_diff1": 0.6873136665710184, | |
| "naucs_at_20_max": 0.4688188620078513, | |
| "naucs_at_20_std": 0.26917226500908054, | |
| "naucs_at_20_diff1": 0.6720615167289586, | |
| "naucs_at_50_max": 0.5671944664000176, | |
| "naucs_at_50_std": 0.4964253483275758, | |
| "naucs_at_50_diff1": 0.722603370462453, | |
| "naucs_at_100_max": 0.6585255212623138, | |
| "naucs_at_100_std": 0.5978336814194427, | |
| "naucs_at_100_diff1": 0.8138609714332596 | |
| }, | |
| "data_dir/eval_vidore/infovqa_test_subsampled": { | |
| "ndcg_at_1": 0.88462, | |
| "ndcg_at_3": 0.91698, | |
| "ndcg_at_5": 0.92378, | |
| "ndcg_at_10": 0.92838, | |
| "ndcg_at_20": 0.93146, | |
| "ndcg_at_50": 0.93263, | |
| "ndcg_at_100": 0.93362, | |
| "map_at_1": 0.88462, | |
| "map_at_3": 0.90924, | |
| "map_at_5": 0.91309, | |
| "map_at_10": 0.91501, | |
| "map_at_20": 0.91585, | |
| "map_at_50": 0.91603, | |
| "map_at_100": 0.91612, | |
| "recall_at_1": 0.88462, | |
| "recall_at_3": 0.93927, | |
| "recall_at_5": 0.95547, | |
| "recall_at_10": 0.96964, | |
| "recall_at_20": 0.98178, | |
| "recall_at_50": 0.98785, | |
| "recall_at_100": 0.99393, | |
| "precision_at_1": 0.88462, | |
| "precision_at_3": 0.31309, | |
| "precision_at_5": 0.19109, | |
| "precision_at_10": 0.09696, | |
| "precision_at_20": 0.04909, | |
| "precision_at_50": 0.01976, | |
| "precision_at_100": 0.00994, | |
| "mrr_at_1": 0.8846153846153846, | |
| "mrr_at_3": 0.9092442645074225, | |
| "mrr_at_5": 0.9129892037786774, | |
| "mrr_at_10": 0.9147717049032839, | |
| "mrr_at_20": 0.9154333931688503, | |
| "mrr_at_50": 0.9156112743493511, | |
| "mrr_at_100": 0.915698522470477, | |
| "naucs_at_1_max": 0.6002033777762705, | |
| "naucs_at_1_std": -0.07102965572007143, | |
| "naucs_at_1_diff1": 0.9544715151522033, | |
| "naucs_at_3_max": 0.8025453360230541, | |
| "naucs_at_3_std": 0.19885112605813188, | |
| "naucs_at_3_diff1": 0.960819563780572, | |
| "naucs_at_5_max": 0.9121856227472513, | |
| "naucs_at_5_std": 0.3583912666662475, | |
| "naucs_at_5_diff1": 0.9584449918884831, | |
| "naucs_at_10_max": 0.9368484108193146, | |
| "naucs_at_10_std": 0.44097650713388975, | |
| "naucs_at_10_diff1": 0.9651729455827266, | |
| "naucs_at_20_max": 0.9709774546522766, | |
| "naucs_at_20_std": 0.6007213904253922, | |
| "naucs_at_20_diff1": 0.9854887273261383, | |
| "naucs_at_50_max": 0.9782330909892136, | |
| "naucs_at_50_std": 0.7577252323561762, | |
| "naucs_at_50_diff1": 0.9782330909892136, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 0.7075525547215259, | |
| "naucs_at_100_diff1": 0.9564661819784259 | |
| }, | |
| "data_dir/eval_vidore/syntheticDocQA_healthcare_industry_test": { | |
| "ndcg_at_1": 0.95, | |
| "ndcg_at_3": 0.97893, | |
| "ndcg_at_5": 0.97893, | |
| "ndcg_at_10": 0.97893, | |
| "ndcg_at_20": 0.97893, | |
| "ndcg_at_50": 0.97893, | |
| "ndcg_at_100": 0.97893, | |
| "map_at_1": 0.95, | |
| "map_at_3": 0.97167, | |
| "map_at_5": 0.97167, | |
| "map_at_10": 0.97167, | |
| "map_at_20": 0.97167, | |
| "map_at_50": 0.97167, | |
| "map_at_100": 0.97167, | |
| "recall_at_1": 0.95, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.95, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.96, | |
| "mrr_at_3": 0.9783333333333333, | |
| "mrr_at_5": 0.9783333333333333, | |
| "mrr_at_10": 0.9783333333333333, | |
| "mrr_at_20": 0.9783333333333333, | |
| "mrr_at_50": 0.9783333333333333, | |
| "mrr_at_100": 0.9783333333333333, | |
| "naucs_at_1_max": 0.7605042016806716, | |
| "naucs_at_1_std": -0.35732959850606716, | |
| "naucs_at_1_diff1": 0.9738562091503253, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "data_dir/eval_vidore/tabfquad_test_subsampled": { | |
| "ndcg_at_1": 0.85714, | |
| "ndcg_at_3": 0.90523, | |
| "ndcg_at_5": 0.91122, | |
| "ndcg_at_10": 0.91807, | |
| "ndcg_at_20": 0.92163, | |
| "ndcg_at_50": 0.92385, | |
| "ndcg_at_100": 0.92385, | |
| "map_at_1": 0.85714, | |
| "map_at_3": 0.89345, | |
| "map_at_5": 0.89685, | |
| "map_at_10": 0.89962, | |
| "map_at_20": 0.90057, | |
| "map_at_50": 0.90097, | |
| "map_at_100": 0.90097, | |
| "recall_at_1": 0.85714, | |
| "recall_at_3": 0.93929, | |
| "recall_at_5": 0.95357, | |
| "recall_at_10": 0.975, | |
| "recall_at_20": 0.98929, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.85714, | |
| "precision_at_3": 0.3131, | |
| "precision_at_5": 0.19071, | |
| "precision_at_10": 0.0975, | |
| "precision_at_20": 0.04946, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.8571428571428571, | |
| "mrr_at_3": 0.8928571428571429, | |
| "mrr_at_5": 0.8955357142857143, | |
| "mrr_at_10": 0.8989866780045351, | |
| "mrr_at_20": 0.8999608080411652, | |
| "mrr_at_50": 0.9003545809349381, | |
| "mrr_at_100": 0.9003545809349381, | |
| "naucs_at_1_max": 0.44880952380952394, | |
| "naucs_at_1_std": 0.06635082604470473, | |
| "naucs_at_1_diff1": 0.9110301263362479, | |
| "naucs_at_3_max": 0.7531443950129105, | |
| "naucs_at_3_std": 0.4460647003899606, | |
| "naucs_at_3_diff1": 0.9288735101883916, | |
| "naucs_at_5_max": 0.8075127486892179, | |
| "naucs_at_5_std": 0.529124470300943, | |
| "naucs_at_5_diff1": 0.917043740573152, | |
| "naucs_at_10_max": 0.7902494331065706, | |
| "naucs_at_10_std": 0.6329198346005056, | |
| "naucs_at_10_diff1": 0.9626517273576021, | |
| "naucs_at_20_max": 0.9564270152505505, | |
| "naucs_at_20_std": 0.8638344226579515, | |
| "naucs_at_20_diff1": 0.9564270152505505, | |
| "naucs_at_50_max": 1.0, | |
| "naucs_at_50_std": 1.0, | |
| "naucs_at_50_diff1": 1.0, | |
| "naucs_at_100_max": 1.0, | |
| "naucs_at_100_std": 1.0, | |
| "naucs_at_100_diff1": 1.0 | |
| }, | |
| "data_dir/eval_vidore/syntheticDocQA_government_reports_test": { | |
| "ndcg_at_1": 0.92, | |
| "ndcg_at_3": 0.96417, | |
| "ndcg_at_5": 0.96417, | |
| "ndcg_at_10": 0.9675, | |
| "ndcg_at_20": 0.9675, | |
| "ndcg_at_50": 0.9675, | |
| "ndcg_at_100": 0.9675, | |
| "map_at_1": 0.92, | |
| "map_at_3": 0.955, | |
| "map_at_5": 0.955, | |
| "map_at_10": 0.95643, | |
| "map_at_20": 0.95643, | |
| "map_at_50": 0.95643, | |
| "map_at_100": 0.95643, | |
| "recall_at_1": 0.92, | |
| "recall_at_3": 0.99, | |
| "recall_at_5": 0.99, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.92, | |
| "precision_at_3": 0.33, | |
| "precision_at_5": 0.198, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.92, | |
| "mrr_at_3": 0.955, | |
| "mrr_at_5": 0.955, | |
| "mrr_at_10": 0.9564285714285714, | |
| "mrr_at_20": 0.9564285714285714, | |
| "mrr_at_50": 0.9564285714285714, | |
| "mrr_at_100": 0.9564285714285714, | |
| "naucs_at_1_max": 0.8768674136321195, | |
| "naucs_at_1_std": 0.41940943043884304, | |
| "naucs_at_1_diff1": 0.9673202614379083, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 0.8692810457516356, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 0.8692810457516413, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "data_dir/eval_vidore/shiftproject_test": { | |
| "ndcg_at_1": 0.78, | |
| "ndcg_at_3": 0.8644, | |
| "ndcg_at_5": 0.87302, | |
| "ndcg_at_10": 0.8828, | |
| "ndcg_at_20": 0.88795, | |
| "ndcg_at_50": 0.88997, | |
| "ndcg_at_100": 0.88997, | |
| "map_at_1": 0.78, | |
| "map_at_3": 0.845, | |
| "map_at_5": 0.85, | |
| "map_at_10": 0.8541, | |
| "map_at_20": 0.85555, | |
| "map_at_50": 0.85589, | |
| "map_at_100": 0.85589, | |
| "recall_at_1": 0.78, | |
| "recall_at_3": 0.92, | |
| "recall_at_5": 0.94, | |
| "recall_at_10": 0.97, | |
| "recall_at_20": 0.99, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.78, | |
| "precision_at_3": 0.30667, | |
| "precision_at_5": 0.188, | |
| "precision_at_10": 0.097, | |
| "precision_at_20": 0.0495, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.81, | |
| "mrr_at_3": 0.8633333333333333, | |
| "mrr_at_5": 0.8658333333333332, | |
| "mrr_at_10": 0.8699285714285715, | |
| "mrr_at_20": 0.871412087912088, | |
| "mrr_at_50": 0.8717454212454213, | |
| "mrr_at_100": 0.8717454212454213, | |
| "naucs_at_1_max": -0.24755413363008374, | |
| "naucs_at_1_std": -0.6164510594890337, | |
| "naucs_at_1_diff1": 0.8162006769601703, | |
| "naucs_at_3_max": 0.2243814192343627, | |
| "naucs_at_3_std": -0.23225957049486393, | |
| "naucs_at_3_diff1": 0.8544000933706815, | |
| "naucs_at_5_max": -0.012371615312794324, | |
| "naucs_at_5_std": -0.5504201680672273, | |
| "naucs_at_5_diff1": 0.9128540305010897, | |
| "naucs_at_10_max": -0.049486461251166146, | |
| "naucs_at_10_std": -0.27591036414565706, | |
| "naucs_at_10_diff1": 0.9128540305010848, | |
| "naucs_at_20_max": 0.7222222222222276, | |
| "naucs_at_20_std": 0.5541549953314738, | |
| "naucs_at_20_diff1": 0.8692810457516413, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| }, | |
| "data_dir/eval_vidore/syntheticDocQA_artificial_intelligence_test": { | |
| "ndcg_at_1": 0.99, | |
| "ndcg_at_3": 0.99631, | |
| "ndcg_at_5": 0.99631, | |
| "ndcg_at_10": 0.99631, | |
| "ndcg_at_20": 0.99631, | |
| "ndcg_at_50": 0.99631, | |
| "ndcg_at_100": 0.99631, | |
| "map_at_1": 0.99, | |
| "map_at_3": 0.995, | |
| "map_at_5": 0.995, | |
| "map_at_10": 0.995, | |
| "map_at_20": 0.995, | |
| "map_at_50": 0.995, | |
| "map_at_100": 0.995, | |
| "recall_at_1": 0.99, | |
| "recall_at_3": 1.0, | |
| "recall_at_5": 1.0, | |
| "recall_at_10": 1.0, | |
| "recall_at_20": 1.0, | |
| "recall_at_50": 1.0, | |
| "recall_at_100": 1.0, | |
| "precision_at_1": 0.99, | |
| "precision_at_3": 0.33333, | |
| "precision_at_5": 0.2, | |
| "precision_at_10": 0.1, | |
| "precision_at_20": 0.05, | |
| "precision_at_50": 0.02, | |
| "precision_at_100": 0.01, | |
| "mrr_at_1": 0.99, | |
| "mrr_at_3": 0.995, | |
| "mrr_at_5": 0.995, | |
| "mrr_at_10": 0.995, | |
| "mrr_at_20": 0.995, | |
| "mrr_at_50": 0.995, | |
| "mrr_at_100": 0.995, | |
| "naucs_at_1_max": 0.8692810457516276, | |
| "naucs_at_1_std": -0.5634920634920657, | |
| "naucs_at_1_diff1": 1.0, | |
| "naucs_at_3_max": 1.0, | |
| "naucs_at_3_std": 1.0, | |
| "naucs_at_3_diff1": 1.0, | |
| "naucs_at_5_max": 1.0, | |
| "naucs_at_5_std": 1.0, | |
| "naucs_at_5_diff1": 1.0, | |
| "naucs_at_10_max": 1.0, | |
| "naucs_at_10_std": 1.0, | |
| "naucs_at_10_diff1": 1.0, | |
| "naucs_at_20_max": 1.0, | |
| "naucs_at_20_std": 1.0, | |
| "naucs_at_20_diff1": 1.0, | |
| "naucs_at_50_max": null, | |
| "naucs_at_50_std": null, | |
| "naucs_at_50_diff1": null, | |
| "naucs_at_100_max": null, | |
| "naucs_at_100_std": null, | |
| "naucs_at_100_diff1": null | |
| } | |
| } | |
| } |