{ "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered": { "ndcg_at_1": 0.63125, "ndcg_at_3": 0.62606, "ndcg_at_5": 0.64036, "ndcg_at_10": 0.67252, "ndcg_at_20": 0.70051, "ndcg_at_50": 0.7225, "ndcg_at_100": 0.73342, "map_at_1": 0.38694, "map_at_3": 0.5157, "map_at_5": 0.55286, "map_at_10": 0.58663, "map_at_20": 0.60199, "map_at_50": 0.61092, "map_at_100": 0.61334, "recall_at_1": 0.38694, "recall_at_3": 0.58442, "recall_at_5": 0.65538, "recall_at_10": 0.75502, "recall_at_20": 0.83375, "recall_at_50": 0.89435, "recall_at_100": 0.93847, "precision_at_1": 0.63125, "precision_at_3": 0.37917, "precision_at_5": 0.28375, "precision_at_10": 0.17875, "precision_at_20": 0.10594, "precision_at_50": 0.05062, "precision_at_100": 0.02756, "mrr_at_1": 0.6375, "mrr_at_3": 0.7270833333333332, "mrr_at_5": 0.7339583333333333, "mrr_at_10": 0.7424727182539682, "mrr_at_20": 0.7446067387473636, "mrr_at_50": 0.7449988719519969, "mrr_at_100": 0.7452357796521399, "naucs_at_1_max": 0.2860227930855084, "naucs_at_1_std": -0.3138417441324, "naucs_at_1_diff1": 0.42744397436023296, "naucs_at_3_max": 0.1892512065375088, "naucs_at_3_std": -0.09854623333300405, "naucs_at_3_diff1": 0.00578348450006455, "naucs_at_5_max": 0.0985916495531507, "naucs_at_5_std": -0.06582623582128855, "naucs_at_5_diff1": -0.08828438926991423, "naucs_at_10_max": 0.0933821578304943, "naucs_at_10_std": 0.08155017778485431, "naucs_at_10_diff1": -0.17980682481558072, "naucs_at_20_max": 0.0020201283703657888, "naucs_at_20_std": 0.10903975278089359, "naucs_at_20_diff1": -0.2369554821180032, "naucs_at_50_max": -0.05388593736433463, "naucs_at_50_std": 0.21602702406052268, "naucs_at_50_diff1": -0.25488066393739356, "naucs_at_100_max": -0.09001262910229582, "naucs_at_100_std": 0.2278313121002944, "naucs_at_100_diff1": -0.25651206108102814 }, "vidore/synthetic_economics_macro_economy_2024_filtered_v1.0": { "ndcg_at_1": 0.7069, "ndcg_at_3": 0.6531, "ndcg_at_5": 0.63125, "ndcg_at_10": 0.60817, "ndcg_at_20": 0.61254, "ndcg_at_50": 0.67937, "ndcg_at_100": 0.7167, "map_at_1": 0.12782, "map_at_3": 0.21562, "map_at_5": 0.28161, "map_at_10": 0.35497, "map_at_20": 0.40334, "map_at_50": 0.46201, "map_at_100": 0.49032, "recall_at_1": 0.12782, "recall_at_3": 0.23164, "recall_at_5": 0.33225, "recall_at_10": 0.47034, "recall_at_20": 0.59047, "recall_at_50": 0.78316, "recall_at_100": 0.89751, "precision_at_1": 0.7069, "precision_at_3": 0.59195, "precision_at_5": 0.54138, "precision_at_10": 0.42759, "precision_at_20": 0.30345, "precision_at_50": 0.1969, "precision_at_100": 0.12741, "mrr_at_1": 0.7241379310344828, "mrr_at_3": 0.7729885057471265, "mrr_at_5": 0.7850574712643678, "mrr_at_10": 0.8005199781061849, "mrr_at_20": 0.8015975643130815, "mrr_at_50": 0.8015975643130815, "mrr_at_100": 0.8015975643130815, "naucs_at_1_max": 0.5414377044487941, "naucs_at_1_std": 0.2885423494902651, "naucs_at_1_diff1": 0.4517408404026178, "naucs_at_3_max": 0.5103699198828265, "naucs_at_3_std": 0.34891966684046116, "naucs_at_3_diff1": 0.1460156103430603, "naucs_at_5_max": 0.5399440580636293, "naucs_at_5_std": 0.3199440767695221, "naucs_at_5_diff1": 0.18068785350199154, "naucs_at_10_max": 0.5255610151091152, "naucs_at_10_std": 0.3925916787135598, "naucs_at_10_diff1": 0.14785999421779913, "naucs_at_20_max": 0.4516553789306758, "naucs_at_20_std": 0.4790296057633737, "naucs_at_20_diff1": 0.14584034106403623, "naucs_at_50_max": 0.34044111140425515, "naucs_at_50_std": 0.4292008781133996, "naucs_at_50_diff1": 0.0657235966934423, "naucs_at_100_max": 0.296959342986135, "naucs_at_100_std": 0.4138623681388592, "naucs_at_100_diff1": 0.06422150196714925 }, "vidore/synthetic_rse_restaurant_filtered_v1.0": { "ndcg_at_1": 0.40351, "ndcg_at_3": 0.47539, "ndcg_at_5": 0.49152, "ndcg_at_10": 0.553, "ndcg_at_20": 0.59699, "ndcg_at_50": 0.62258, "ndcg_at_100": 0.63184, "map_at_1": 0.2155, "map_at_3": 0.35506, "map_at_5": 0.38442, "map_at_10": 0.43472, "map_at_20": 0.45888, "map_at_50": 0.46993, "map_at_100": 0.47398, "recall_at_1": 0.2155, "recall_at_3": 0.46042, "recall_at_5": 0.5245, "recall_at_10": 0.69217, "recall_at_20": 0.83089, "recall_at_50": 0.9086, "recall_at_100": 0.93039, "precision_at_1": 0.40351, "precision_at_3": 0.32749, "precision_at_5": 0.25263, "precision_at_10": 0.18421, "precision_at_20": 0.12105, "precision_at_50": 0.06105, "precision_at_100": 0.03421, "mrr_at_1": 0.40350877192982454, "mrr_at_3": 0.5555555555555555, "mrr_at_5": 0.5730994152046783, "mrr_at_10": 0.5808966861598441, "mrr_at_20": 0.586509202298676, "mrr_at_50": 0.5877768876410641, "mrr_at_100": 0.5877768876410641, "naucs_at_1_max": 0.29648339825644304, "naucs_at_1_std": 0.06641652874714649, "naucs_at_1_diff1": 0.5045818569761475, "naucs_at_3_max": 0.16818047225254976, "naucs_at_3_std": 0.17028690578931582, "naucs_at_3_diff1": 0.17461140240553258, "naucs_at_5_max": 0.19459542279126757, "naucs_at_5_std": 0.24823743512641647, "naucs_at_5_diff1": 0.19749834680039569, "naucs_at_10_max": 0.07835810840401428, "naucs_at_10_std": 0.19882694823881913, "naucs_at_10_diff1": 0.13857285069040404, "naucs_at_20_max": -0.007433838800528796, "naucs_at_20_std": 0.19538335810174498, "naucs_at_20_diff1": 0.0012071916296247957, "naucs_at_50_max": -0.0902754753027608, "naucs_at_50_std": 0.1592772246389163, "naucs_at_50_diff1": -0.02404431111361124, "naucs_at_100_max": -0.12119070620203119, "naucs_at_100_std": 0.13618966353380924, "naucs_at_100_diff1": -0.04813882401945786 }, "vidore/synthetic_axa_filtered_v1.0": { "ndcg_at_1": 0.72222, "ndcg_at_3": 0.69497, "ndcg_at_5": 0.66273, "ndcg_at_10": 0.67378, "ndcg_at_20": 0.69012, "ndcg_at_50": 0.75042, "ndcg_at_100": 0.77219, "map_at_1": 0.36488, "map_at_3": 0.47229, "map_at_5": 0.49718, "map_at_10": 0.54558, "map_at_20": 0.56749, "map_at_50": 0.60107, "map_at_100": 0.60562, "recall_at_1": 0.36488, "recall_at_3": 0.54457, "recall_at_5": 0.58939, "recall_at_10": 0.68972, "recall_at_20": 0.74479, "recall_at_50": 0.89156, "recall_at_100": 0.98955, "precision_at_1": 0.72222, "precision_at_3": 0.44444, "precision_at_5": 0.33333, "precision_at_10": 0.24444, "precision_at_20": 0.15, "precision_at_50": 0.08667, "precision_at_100": 0.04667, "mrr_at_1": 0.7222222222222222, "mrr_at_3": 0.7962962962962963, "mrr_at_5": 0.8101851851851852, "mrr_at_10": 0.8101851851851852, "mrr_at_20": 0.8101851851851852, "mrr_at_50": 0.8101851851851852, "mrr_at_100": 0.8109903381642511, "naucs_at_1_max": 0.19079787602675777, "naucs_at_1_std": 0.03533531122016283, "naucs_at_1_diff1": 0.632680674681619, "naucs_at_3_max": -0.31385535499287276, "naucs_at_3_std": -0.04655749634420289, "naucs_at_3_diff1": 0.155475737963548, "naucs_at_5_max": -0.2472297825482412, "naucs_at_5_std": 0.03761412944004308, "naucs_at_5_diff1": -0.10924664115489778, "naucs_at_10_max": -0.37765403672620185, "naucs_at_10_std": 0.002000186439662032, "naucs_at_10_diff1": -0.14440595994296845, "naucs_at_20_max": -0.4699503134091192, "naucs_at_20_std": 0.025147844173298965, "naucs_at_20_diff1": -0.16389645563321997, "naucs_at_50_max": -0.5068940056618596, "naucs_at_50_std": -0.019754043372479577, "naucs_at_50_diff1": -0.16850642524469236, "naucs_at_100_max": -0.4767515120911255, "naucs_at_100_std": 0.015763234490211366, "naucs_at_100_diff1": -0.20731314235125625 }, "vidore/synthetic_rse_restaurant_filtered_v1.0_multilingual": { "ndcg_at_1": 0.4386, "ndcg_at_3": 0.48147, "ndcg_at_5": 0.51908, "ndcg_at_10": 0.57159, "ndcg_at_20": 0.60702, "ndcg_at_50": 0.63417, "ndcg_at_100": 0.64411, "map_at_1": 0.23507, "map_at_3": 0.35761, "map_at_5": 0.40257, "map_at_10": 0.44822, "map_at_20": 0.47031, "map_at_50": 0.48211, "map_at_100": 0.48628, "recall_at_1": 0.23507, "recall_at_3": 0.4487, "recall_at_5": 0.56129, "recall_at_10": 0.71123, "recall_at_20": 0.82755, "recall_at_50": 0.90956, "recall_at_100": 0.93439, "precision_at_1": 0.4386, "precision_at_3": 0.32895, "precision_at_5": 0.27018, "precision_at_10": 0.18991, "precision_at_20": 0.11974, "precision_at_50": 0.06123, "precision_at_100": 0.03447, "mrr_at_1": 0.4517543859649123, "mrr_at_3": 0.5796783625730992, "mrr_at_5": 0.6033625730994151, "mrr_at_10": 0.6098318713450293, "mrr_at_20": 0.6143413495606139, "mrr_at_50": 0.6150114081677309, "mrr_at_100": 0.6150114081677309, "naucs_at_1_max": 0.1695609084486461, "naucs_at_1_std": 0.06302260506626338, "naucs_at_1_diff1": 0.34368925726365046, "naucs_at_3_max": 0.07022225247354692, "naucs_at_3_std": 0.10306674677084651, "naucs_at_3_diff1": 0.14995084571990153, "naucs_at_5_max": 0.07354175839710718, "naucs_at_5_std": 0.16198320276362924, "naucs_at_5_diff1": 0.06343317549781861, "naucs_at_10_max": 0.022729360196430883, "naucs_at_10_std": 0.1654593087070378, "naucs_at_10_diff1": 0.04276127936743703, "naucs_at_20_max": -0.031554725803849416, "naucs_at_20_std": 0.1300165212363859, "naucs_at_20_diff1": 0.006021945700144339, "naucs_at_50_max": -0.06313785005592049, "naucs_at_50_std": 0.13595654225029755, "naucs_at_50_diff1": 0.008224218691189635, "naucs_at_100_max": -0.08372981904605119, "naucs_at_100_std": 0.12653852178262695, "naucs_at_100_diff1": -0.012169561956816101 }, "vidore/synthetic_axa_filtered_v1.0_multilingual": { "ndcg_at_1": 0.65278, "ndcg_at_3": 0.61009, "ndcg_at_5": 0.59294, "ndcg_at_10": 0.61008, "ndcg_at_20": 0.63877, "ndcg_at_50": 0.69737, "ndcg_at_100": 0.72109, "map_at_1": 0.3313, "map_at_3": 0.41787, "map_at_5": 0.44884, "map_at_10": 0.49024, "map_at_20": 0.51424, "map_at_50": 0.54276, "map_at_100": 0.54899, "recall_at_1": 0.3313, "recall_at_3": 0.46646, "recall_at_5": 0.52791, "recall_at_10": 0.64008, "recall_at_20": 0.73391, "recall_at_50": 0.8845, "recall_at_100": 0.97673, "precision_at_1": 0.65278, "precision_at_3": 0.39352, "precision_at_5": 0.30833, "precision_at_10": 0.22222, "precision_at_20": 0.14028, "precision_at_50": 0.08111, "precision_at_100": 0.04556, "mrr_at_1": 0.6527777777777778, "mrr_at_3": 0.7175925925925928, "mrr_at_5": 0.7307870370370372, "mrr_at_10": 0.7397156084656086, "mrr_at_20": 0.7417099104599107, "mrr_at_50": 0.7421307858807862, "mrr_at_100": 0.7426557166541077, "naucs_at_1_max": 0.318303386786102, "naucs_at_1_std": 0.23808943823013543, "naucs_at_1_diff1": 0.6508710234226615, "naucs_at_3_max": 0.13214248537450415, "naucs_at_3_std": 0.16151259206691232, "naucs_at_3_diff1": 0.22229988734788014, "naucs_at_5_max": 0.05889008903334795, "naucs_at_5_std": 0.11423669671153379, "naucs_at_5_diff1": 0.11548423013320717, "naucs_at_10_max": -0.037397142604183475, "naucs_at_10_std": 0.09416088185519944, "naucs_at_10_diff1": 0.008856567275955608, "naucs_at_20_max": -0.11441594604958465, "naucs_at_20_std": 0.04973867186679883, "naucs_at_20_diff1": -0.05320557709786541, "naucs_at_50_max": -0.18138441233720995, "naucs_at_50_std": 0.010825689422469654, "naucs_at_50_diff1": -0.10510638193999315, "naucs_at_100_max": -0.202379290531666, "naucs_at_100_std": -0.014979683943287803, "naucs_at_100_diff1": -0.14558601228430124 }, "vidore/synthetic_mit_biomedical_tissue_interactions_unfiltered_multilingual": { "ndcg_at_1": 0.57344, "ndcg_at_3": 0.59236, "ndcg_at_5": 0.61157, "ndcg_at_10": 0.63866, "ndcg_at_20": 0.66879, "ndcg_at_50": 0.69522, "ndcg_at_100": 0.70444, "map_at_1": 0.36007, "map_at_3": 0.48525, "map_at_5": 0.52371, "map_at_10": 0.5533, "map_at_20": 0.5688, "map_at_50": 0.57892, "map_at_100": 0.58096, "recall_at_1": 0.36007, "recall_at_3": 0.55859, "recall_at_5": 0.63871, "recall_at_10": 0.7243, "recall_at_20": 0.81097, "recall_at_50": 0.88961, "recall_at_100": 0.92417, "precision_at_1": 0.57344, "precision_at_3": 0.36302, "precision_at_5": 0.27469, "precision_at_10": 0.17016, "precision_at_20": 0.10219, "precision_at_50": 0.04997, "precision_at_100": 0.02702, "mrr_at_1": 0.5796875, "mrr_at_3": 0.680729166666666, "mrr_at_5": 0.6924479166666659, "mrr_at_10": 0.6992057291666662, "mrr_at_20": 0.7025276038138841, "mrr_at_50": 0.7032994002394296, "mrr_at_100": 0.7035081912324784, "naucs_at_1_max": 0.2618446848882775, "naucs_at_1_std": -0.170161905683693, "naucs_at_1_diff1": 0.4753067384546821, "naucs_at_3_max": 0.17827521235144483, "naucs_at_3_std": -0.060198662170649925, "naucs_at_3_diff1": 0.05418513753300305, "naucs_at_5_max": 0.08757447272019533, "naucs_at_5_std": -0.049911565737325504, "naucs_at_5_diff1": -0.04360643940712858, "naucs_at_10_max": 0.0797198239661374, "naucs_at_10_std": 0.03949138307521818, "naucs_at_10_diff1": -0.09762488661941034, "naucs_at_20_max": 0.01961040143553897, "naucs_at_20_std": 0.10087633582361409, "naucs_at_20_diff1": -0.17144901884144179, "naucs_at_50_max": -0.02239975053336602, "naucs_at_50_std": 0.1699503781904292, "naucs_at_50_diff1": -0.1722611533882454, "naucs_at_100_max": -0.04512586039808127, "naucs_at_100_std": 0.17386318414141583, "naucs_at_100_diff1": -0.1871247841007667 }, "vidore/synthetics_economics_macro_economy_2024_filtered_v1.0_multilingual": { "ndcg_at_1": 0.66379, "ndcg_at_3": 0.60737, "ndcg_at_5": 0.57718, "ndcg_at_10": 0.55599, "ndcg_at_20": 0.57988, "ndcg_at_50": 0.64999, "ndcg_at_100": 0.68674, "map_at_1": 0.12281, "map_at_3": 0.20294, "map_at_5": 0.25378, "map_at_10": 0.31487, "map_at_20": 0.36947, "map_at_50": 0.42823, "map_at_100": 0.45448, "recall_at_1": 0.12281, "recall_at_3": 0.2256, "recall_at_5": 0.30659, "recall_at_10": 0.42269, "recall_at_20": 0.56687, "recall_at_50": 0.76718, "recall_at_100": 0.88492, "precision_at_1": 0.66379, "precision_at_3": 0.5431, "precision_at_5": 0.48362, "precision_at_10": 0.38836, "precision_at_20": 0.29591, "precision_at_50": 0.19345, "precision_at_100": 0.12414, "mrr_at_1": 0.6767241379310345, "mrr_at_3": 0.7399425287356323, "mrr_at_5": 0.7548132183908046, "mrr_at_10": 0.7652247536945812, "mrr_at_20": 0.7667345585449032, "mrr_at_50": 0.7675009343641154, "mrr_at_100": 0.7675472821579604, "naucs_at_1_max": 0.3427760433833983, "naucs_at_1_std": 0.17486041852136283, "naucs_at_1_diff1": 0.3493033195022897, "naucs_at_3_max": 0.29896675454636334, "naucs_at_3_std": 0.2241328326881254, "naucs_at_3_diff1": 0.07681853603265505, "naucs_at_5_max": 0.3541130170379229, "naucs_at_5_std": 0.27779731381034506, "naucs_at_5_diff1": 0.04428119389439255, "naucs_at_10_max": 0.3161718142368522, "naucs_at_10_std": 0.2654928526689307, "naucs_at_10_diff1": 0.014100490157475373, "naucs_at_20_max": 0.2499680124975916, "naucs_at_20_std": 0.2633550881217926, "naucs_at_20_diff1": -0.02686064189005896, "naucs_at_50_max": 0.1951500660706873, "naucs_at_50_std": 0.23487608842256993, "naucs_at_50_diff1": -0.0871206335340013, "naucs_at_100_max": 0.16510434929270115, "naucs_at_100_std": 0.21054693222984838, "naucs_at_100_diff1": -0.10274771435027014 }, "vidore/restaurant_esg_reports_beir": { "ndcg_at_1": 0.64103, "ndcg_at_3": 0.62507, "ndcg_at_5": 0.65665, "ndcg_at_10": 0.67232, "ndcg_at_20": 0.69863, "ndcg_at_50": 0.72704, "ndcg_at_100": 0.73746, "map_at_1": 0.44583, "map_at_3": 0.54321, "map_at_5": 0.58695, "map_at_10": 0.60061, "map_at_20": 0.61379, "map_at_50": 0.62209, "map_at_100": 0.62409, "recall_at_1": 0.44583, "recall_at_3": 0.60335, "recall_at_5": 0.69232, "recall_at_10": 0.74186, "recall_at_20": 0.81674, "recall_at_50": 0.92301, "recall_at_100": 0.9696, "precision_at_1": 0.65385, "precision_at_3": 0.33333, "precision_at_5": 0.25385, "precision_at_10": 0.14231, "precision_at_20": 0.08558, "precision_at_50": 0.04154, "precision_at_100": 0.0225, "mrr_at_1": 0.6538461538461539, "mrr_at_3": 0.7211538461538461, "mrr_at_5": 0.7288461538461538, "mrr_at_10": 0.736935286935287, "mrr_at_20": 0.738003663003663, "mrr_at_50": 0.739894529945878, "mrr_at_100": 0.7402260949326155, "naucs_at_1_max": 0.18675644205833766, "naucs_at_1_std": 0.12089377306440403, "naucs_at_1_diff1": 0.23180938958754124, "naucs_at_3_max": 0.1337570930336171, "naucs_at_3_std": 0.2424272064488133, "naucs_at_3_diff1": -0.21025078361612204, "naucs_at_5_max": 0.005149088005913239, "naucs_at_5_std": 0.30173104877190665, "naucs_at_5_diff1": -0.3297309037027529, "naucs_at_10_max": 0.058274590610789254, "naucs_at_10_std": 0.32737391815331685, "naucs_at_10_diff1": -0.390552006407966, "naucs_at_20_max": -0.04962516887611918, "naucs_at_20_std": 0.33292517052358606, "naucs_at_20_diff1": -0.435159500418056, "naucs_at_50_max": -0.0972258593819507, "naucs_at_50_std": 0.3213589716904649, "naucs_at_50_diff1": -0.40791385861358653, "naucs_at_100_max": -0.16245636826248977, "naucs_at_100_std": 0.261806614637912, "naucs_at_100_diff1": -0.4069117709772787 }, "vidore/arxivqa_test_subsampled": { "ndcg_at_1": 0.82, "ndcg_at_3": 0.87317, "ndcg_at_5": 0.88134, "ndcg_at_10": 0.88917, "ndcg_at_20": 0.89577, "ndcg_at_50": 0.89856, "ndcg_at_100": 0.89953, "map_at_1": 0.82, "map_at_3": 0.861, "map_at_5": 0.8655, "map_at_10": 0.86878, "map_at_20": 0.8706, "map_at_50": 0.87105, "map_at_100": 0.87113, "recall_at_1": 0.82, "recall_at_3": 0.908, "recall_at_5": 0.928, "recall_at_10": 0.952, "recall_at_20": 0.978, "recall_at_50": 0.992, "recall_at_100": 0.998, "precision_at_1": 0.82, "precision_at_3": 0.30267, "precision_at_5": 0.1856, "precision_at_10": 0.0952, "precision_at_20": 0.0489, "precision_at_50": 0.01984, "precision_at_100": 0.00998, "mrr_at_1": 0.818, "mrr_at_3": 0.8596666666666666, "mrr_at_5": 0.8641666666666665, "mrr_at_10": 0.8674912698412698, "mrr_at_20": 0.8692320429570428, "mrr_at_50": 0.8697783157643562, "mrr_at_100": 0.8698595552455958, "naucs_at_1_max": 0.8542625839850205, "naucs_at_1_std": -0.15451591584976296, "naucs_at_1_diff1": 0.9423945368432642, "naucs_at_3_max": 0.8243413307351934, "naucs_at_3_std": -0.019983355661106324, "naucs_at_3_diff1": 0.8796330126253413, "naucs_at_5_max": 0.8361733582321816, "naucs_at_5_std": 0.04758014316837583, "naucs_at_5_diff1": 0.8694755680049819, "naucs_at_10_max": 0.8481948334889503, "naucs_at_10_std": 0.00797541238717203, "naucs_at_10_diff1": 0.8792989417989403, "naucs_at_20_max": 0.9123588829471215, "naucs_at_20_std": 0.13352007469653804, "naucs_at_20_diff1": 0.9019607843137273, "naucs_at_50_max": 0.9673202614378978, "naucs_at_50_std": 0.24684873949577324, "naucs_at_50_diff1": 0.9305555555555584, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/docvqa_test_subsampled": { "ndcg_at_1": 0.51663, "ndcg_at_3": 0.59278, "ndcg_at_5": 0.61529, "ndcg_at_10": 0.63354, "ndcg_at_20": 0.64688, "ndcg_at_50": 0.66214, "ndcg_at_100": 0.67157, "map_at_1": 0.51663, "map_at_3": 0.57391, "map_at_5": 0.58622, "map_at_10": 0.59393, "map_at_20": 0.59753, "map_at_50": 0.60011, "map_at_100": 0.60096, "recall_at_1": 0.51663, "recall_at_3": 0.64745, "recall_at_5": 0.70288, "recall_at_10": 0.75831, "recall_at_20": 0.81153, "recall_at_50": 0.88692, "recall_at_100": 0.94457, "precision_at_1": 0.51663, "precision_at_3": 0.21582, "precision_at_5": 0.14058, "precision_at_10": 0.07583, "precision_at_20": 0.04058, "precision_at_50": 0.01774, "precision_at_100": 0.00945, "mrr_at_1": 0.5144124168514412, "mrr_at_3": 0.5731707317073172, "mrr_at_5": 0.586031042128603, "mrr_at_10": 0.5933419561468339, "mrr_at_20": 0.5967957449683643, "mrr_at_50": 0.599471176191027, "mrr_at_100": 0.600328115684025, "naucs_at_1_max": 0.6741589793839163, "naucs_at_1_std": -0.16205001702916214, "naucs_at_1_diff1": 0.8162930079294569, "naucs_at_3_max": 0.7465129968490366, "naucs_at_3_std": -0.09544436940181997, "naucs_at_3_diff1": 0.7350663423580653, "naucs_at_5_max": 0.7629182852125305, "naucs_at_5_std": -0.050579687432907604, "naucs_at_5_diff1": 0.7012350138123477, "naucs_at_10_max": 0.7889385100226093, "naucs_at_10_std": 0.04757918573005526, "naucs_at_10_diff1": 0.6915024085662332, "naucs_at_20_max": 0.8147748548808955, "naucs_at_20_std": 0.10152398884541157, "naucs_at_20_diff1": 0.704970590799652, "naucs_at_50_max": 0.8657229702927914, "naucs_at_50_std": 0.3246550245384335, "naucs_at_50_diff1": 0.6989609170252572, "naucs_at_100_max": 0.9055488370856275, "naucs_at_100_std": 0.49971793394423536, "naucs_at_100_diff1": 0.6936703826495957 }, "vidore/infovqa_test_subsampled": { "ndcg_at_1": 0.86842, "ndcg_at_3": 0.90307, "ndcg_at_5": 0.91204, "ndcg_at_10": 0.91773, "ndcg_at_20": 0.92031, "ndcg_at_50": 0.92201, "ndcg_at_100": 0.92265, "map_at_1": 0.86842, "map_at_3": 0.89474, "map_at_5": 0.8996, "map_at_10": 0.90182, "map_at_20": 0.90254, "map_at_50": 0.90285, "map_at_100": 0.90291, "recall_at_1": 0.86842, "recall_at_3": 0.92713, "recall_at_5": 0.94939, "recall_at_10": 0.96761, "recall_at_20": 0.97773, "recall_at_50": 0.98583, "recall_at_100": 0.98988, "precision_at_1": 0.86842, "precision_at_3": 0.30904, "precision_at_5": 0.18988, "precision_at_10": 0.09676, "precision_at_20": 0.04889, "precision_at_50": 0.01972, "precision_at_100": 0.0099, "mrr_at_1": 0.868421052631579, "mrr_at_3": 0.894736842105263, "mrr_at_5": 0.8996963562753034, "mrr_at_10": 0.9018740762161812, "mrr_at_20": 0.9025973885538661, "mrr_at_50": 0.9029005381313663, "mrr_at_100": 0.9029517941275169, "naucs_at_1_max": 0.6108584962344086, "naucs_at_1_std": -0.1359018313249994, "naucs_at_1_diff1": 0.9136811065149164, "naucs_at_3_max": 0.6840687823130331, "naucs_at_3_std": 0.03370197684645164, "naucs_at_3_diff1": 0.886059733600837, "naucs_at_5_max": 0.7765209527928315, "naucs_at_5_std": 0.2228524148946178, "naucs_at_5_diff1": 0.8620463071981626, "naucs_at_10_max": 0.8392814749886633, "naucs_at_10_std": 0.5798638053062866, "naucs_at_10_diff1": 0.8728108205086554, "naucs_at_20_max": 1.0, "naucs_at_20_std": 0.8268704165639359, "naucs_at_20_diff1": 0.8986141894622774, "naucs_at_50_max": 1.0, "naucs_at_50_std": 0.8102686931758233, "naucs_at_50_diff1": 0.8593367911642417, "naucs_at_100_max": 1.0, "naucs_at_100_std": 0.7899231179977639, "naucs_at_100_diff1": 0.9183327616354471 }, "vidore/tabfquad_test_subsampled": { "ndcg_at_1": 0.89286, "ndcg_at_3": 0.92836, "ndcg_at_5": 0.93112, "ndcg_at_10": 0.93804, "ndcg_at_20": 0.94089, "ndcg_at_50": 0.94232, "ndcg_at_100": 0.94232, "map_at_1": 0.89286, "map_at_3": 0.91964, "map_at_5": 0.92107, "map_at_10": 0.92391, "map_at_20": 0.92477, "map_at_50": 0.925, "map_at_100": 0.925, "recall_at_1": 0.89286, "recall_at_3": 0.95357, "recall_at_5": 0.96071, "recall_at_10": 0.98214, "recall_at_20": 0.99286, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.89286, "precision_at_3": 0.31786, "precision_at_5": 0.19214, "precision_at_10": 0.09821, "precision_at_20": 0.04964, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.8928571428571429, "mrr_at_3": 0.9196428571428571, "mrr_at_5": 0.9210714285714285, "mrr_at_10": 0.9239129818594106, "mrr_at_20": 0.9247733714698002, "mrr_at_50": 0.9250078037042323, "mrr_at_100": 0.9250078037042323, "naucs_at_1_max": 0.8354945054945048, "naucs_at_1_std": 0.37345368916797433, "naucs_at_1_diff1": 0.9218210361067499, "naucs_at_3_max": 0.9657042304101127, "naucs_at_3_std": 0.7062773827479698, "naucs_at_3_diff1": 0.9396681749622927, "naucs_at_5_max": 0.9594686359392222, "naucs_at_5_std": 0.6647568118156384, "naucs_at_5_diff1": 0.9286987522281656, "naucs_at_10_max": 1.0, "naucs_at_10_std": 0.9738562091503306, "naucs_at_10_diff1": 0.9477124183006519, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": 1.0, "naucs_at_50_std": 1.0, "naucs_at_50_diff1": 1.0, "naucs_at_100_max": 1.0, "naucs_at_100_std": 1.0, "naucs_at_100_diff1": 1.0 }, "vidore/tatdqa_test": { "ndcg_at_1": 0.60571, "ndcg_at_3": 0.71371, "ndcg_at_5": 0.73981, "ndcg_at_10": 0.7601, "ndcg_at_20": 0.76893, "ndcg_at_50": 0.7733, "ndcg_at_100": 0.77517, "map_at_1": 0.60571, "map_at_3": 0.68702, "map_at_5": 0.70154, "map_at_10": 0.71006, "map_at_20": 0.71253, "map_at_50": 0.71329, "map_at_100": 0.71345, "recall_at_1": 0.60571, "recall_at_3": 0.79101, "recall_at_5": 0.85419, "recall_at_10": 0.91616, "recall_at_20": 0.95079, "recall_at_50": 0.97205, "recall_at_100": 0.9836, "precision_at_1": 0.60571, "precision_at_3": 0.26367, "precision_at_5": 0.17084, "precision_at_10": 0.09162, "precision_at_20": 0.04754, "precision_at_50": 0.01944, "precision_at_100": 0.00984, "mrr_at_1": 0.6038882138517618, "mrr_at_3": 0.6857027136492516, "mrr_at_5": 0.7003138922640758, "mrr_at_10": 0.7090893749156212, "mrr_at_20": 0.7114126192954364, "mrr_at_50": 0.7121742514967643, "mrr_at_100": 0.7123364657648046, "naucs_at_1_max": 0.3821214020038102, "naucs_at_1_std": -0.13489407631885159, "naucs_at_1_diff1": 0.7298383859421139, "naucs_at_3_max": 0.45115702284602416, "naucs_at_3_std": -0.04154958812127681, "naucs_at_3_diff1": 0.6171010959328003, "naucs_at_5_max": 0.4834741295717716, "naucs_at_5_std": -0.03652161720284896, "naucs_at_5_diff1": 0.6133480640518033, "naucs_at_10_max": 0.5818884435678058, "naucs_at_10_std": 0.1493431453981439, "naucs_at_10_diff1": 0.6125109504507562, "naucs_at_20_max": 0.7030343359941152, "naucs_at_20_std": 0.3539243062200548, "naucs_at_20_diff1": 0.6751362244435348, "naucs_at_50_max": 0.7344845360513886, "naucs_at_50_std": 0.5609137914009904, "naucs_at_50_diff1": 0.6435577411690651, "naucs_at_100_max": 0.7291790977931831, "naucs_at_100_std": 0.7412505638179597, "naucs_at_100_diff1": 0.6088543824874393 }, "vidore/shiftproject_test": { "ndcg_at_1": 0.74, "ndcg_at_3": 0.85333, "ndcg_at_5": 0.85764, "ndcg_at_10": 0.87455, "ndcg_at_20": 0.8769, "ndcg_at_50": 0.8769, "ndcg_at_100": 0.8769, "map_at_1": 0.74, "map_at_3": 0.82667, "map_at_5": 0.82917, "map_at_10": 0.8366, "map_at_20": 0.83715, "map_at_50": 0.83715, "map_at_100": 0.83715, "recall_at_1": 0.74, "recall_at_3": 0.93, "recall_at_5": 0.94, "recall_at_10": 0.99, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.74, "precision_at_3": 0.31, "precision_at_5": 0.188, "precision_at_10": 0.099, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.74, "mrr_at_3": 0.8266666666666665, "mrr_at_5": 0.8291666666666666, "mrr_at_10": 0.8365952380952381, "mrr_at_20": 0.8371834733893557, "mrr_at_50": 0.8371834733893557, "mrr_at_100": 0.8371834733893557, "naucs_at_1_max": 0.36659097360204357, "naucs_at_1_std": -0.13924820566886958, "naucs_at_1_diff1": 0.7847208142411088, "naucs_at_3_max": 0.7129518474056293, "naucs_at_3_std": -0.039482459650526366, "naucs_at_3_diff1": 0.8102574363078549, "naucs_at_5_max": 0.8603330220977291, "naucs_at_5_std": 0.24393090569561146, "naucs_at_5_diff1": 0.8004201680672299, "naucs_at_10_max": 0.5541549953314738, "naucs_at_10_std": 0.12278244631185926, "naucs_at_10_diff1": 0.35807656395891135, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_artificial_intelligence_test": { "ndcg_at_1": 0.96, "ndcg_at_3": 0.98393, "ndcg_at_5": 0.98393, "ndcg_at_10": 0.98393, "ndcg_at_20": 0.98393, "ndcg_at_50": 0.98393, "ndcg_at_100": 0.98393, "map_at_1": 0.96, "map_at_3": 0.97833, "map_at_5": 0.97833, "map_at_10": 0.97833, "map_at_20": 0.97833, "map_at_50": 0.97833, "map_at_100": 0.97833, "recall_at_1": 0.96, "recall_at_3": 1.0, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.96, "precision_at_3": 0.33333, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.9783333333333333, "mrr_at_5": 0.9783333333333333, "mrr_at_10": 0.9783333333333333, "mrr_at_20": 0.9783333333333333, "mrr_at_50": 0.9783333333333333, "mrr_at_100": 0.9783333333333333, "naucs_at_1_max": 0.7373949579831944, "naucs_at_1_std": -0.32784780578897976, "naucs_at_1_diff1": 0.9346405228758167, "naucs_at_3_max": 1.0, "naucs_at_3_std": 1.0, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_energy_test": { "ndcg_at_1": 0.94, "ndcg_at_3": 0.95262, "ndcg_at_5": 0.95693, "ndcg_at_10": 0.96008, "ndcg_at_20": 0.96287, "ndcg_at_50": 0.96511, "ndcg_at_100": 0.96511, "map_at_1": 0.94, "map_at_3": 0.95, "map_at_5": 0.9525, "map_at_10": 0.95375, "map_at_20": 0.95466, "map_at_50": 0.95514, "map_at_100": 0.95514, "recall_at_1": 0.94, "recall_at_3": 0.96, "recall_at_5": 0.97, "recall_at_10": 0.98, "recall_at_20": 0.99, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.94, "precision_at_3": 0.32, "precision_at_5": 0.194, "precision_at_10": 0.098, "precision_at_20": 0.0495, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.94, "mrr_at_3": 0.95, "mrr_at_5": 0.9525, "mrr_at_10": 0.95375, "mrr_at_20": 0.9551590909090909, "mrr_at_50": 0.9551590909090909, "mrr_at_100": 0.9551590909090909, "naucs_at_1_max": 0.16293183940242884, "naucs_at_1_std": -1.0354808590102687, "naucs_at_1_diff1": 0.9782135076252712, "naucs_at_3_max": 0.31500933706816353, "naucs_at_3_std": -0.9773576097105482, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 0.08667911609088073, "naucs_at_5_std": -1.5438842203547984, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": -0.3699813258636707, "naucs_at_10_std": -1.7399626517273414, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": -1.7399626517273863, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_government_reports_test": { "ndcg_at_1": 0.9, "ndcg_at_3": 0.95678, "ndcg_at_5": 0.95678, "ndcg_at_10": 0.95994, "ndcg_at_20": 0.95994, "ndcg_at_50": 0.95994, "ndcg_at_100": 0.95994, "map_at_1": 0.9, "map_at_3": 0.945, "map_at_5": 0.945, "map_at_10": 0.94625, "map_at_20": 0.94625, "map_at_50": 0.94625, "map_at_100": 0.94625, "recall_at_1": 0.9, "recall_at_3": 0.99, "recall_at_5": 0.99, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.9, "precision_at_3": 0.33, "precision_at_5": 0.198, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.9, "mrr_at_3": 0.945, "mrr_at_5": 0.945, "mrr_at_10": 0.9464285714285714, "mrr_at_20": 0.9464285714285714, "mrr_at_50": 0.9464285714285714, "mrr_at_100": 0.9464285714285714, "naucs_at_1_max": 0.5815126050420164, "naucs_at_1_std": -0.42105508870214864, "naucs_at_1_diff1": 0.9330065359477123, "naucs_at_3_max": 0.5541549953314585, "naucs_at_3_std": -1.7399626517274398, "naucs_at_3_diff1": 0.8692810457516356, "naucs_at_5_max": 0.5541549953314738, "naucs_at_5_std": -1.7399626517273863, "naucs_at_5_diff1": 0.8692810457516413, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null }, "vidore/syntheticDocQA_healthcare_industry_test": { "ndcg_at_1": 0.96, "ndcg_at_3": 0.97893, "ndcg_at_5": 0.98323, "ndcg_at_10": 0.98323, "ndcg_at_20": 0.98323, "ndcg_at_50": 0.98323, "ndcg_at_100": 0.98323, "map_at_1": 0.96, "map_at_3": 0.975, "map_at_5": 0.9775, "map_at_10": 0.9775, "map_at_20": 0.9775, "map_at_50": 0.9775, "map_at_100": 0.9775, "recall_at_1": 0.96, "recall_at_3": 0.99, "recall_at_5": 1.0, "recall_at_10": 1.0, "recall_at_20": 1.0, "recall_at_50": 1.0, "recall_at_100": 1.0, "precision_at_1": 0.96, "precision_at_3": 0.33, "precision_at_5": 0.2, "precision_at_10": 0.1, "precision_at_20": 0.05, "precision_at_50": 0.02, "precision_at_100": 0.01, "mrr_at_1": 0.96, "mrr_at_3": 0.975, "mrr_at_5": 0.9775, "mrr_at_10": 0.9775, "mrr_at_20": 0.9775, "mrr_at_50": 0.9775, "mrr_at_100": 0.9775, "naucs_at_1_max": 0.7006302521008423, "naucs_at_1_std": -0.7350606909430405, "naucs_at_1_diff1": 0.9305555555555578, "naucs_at_3_max": 0.7222222222222157, "naucs_at_3_std": -1.1517273576097802, "naucs_at_3_diff1": 1.0, "naucs_at_5_max": 1.0, "naucs_at_5_std": 1.0, "naucs_at_5_diff1": 1.0, "naucs_at_10_max": 1.0, "naucs_at_10_std": 1.0, "naucs_at_10_diff1": 1.0, "naucs_at_20_max": 1.0, "naucs_at_20_std": 1.0, "naucs_at_20_diff1": 1.0, "naucs_at_50_max": null, "naucs_at_50_std": null, "naucs_at_50_diff1": null, "naucs_at_100_max": null, "naucs_at_100_std": null, "naucs_at_100_diff1": null } }